From 6c67004e2406c0953e1f6692a381d7bf80ea0ec3 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 12 Jul 2023 20:26:26 -0400 Subject: [PATCH 0001/1141] Begin scaffolding new Selection data structure --- mathesar_ui/.eslintrc.cjs | 1 + mathesar_ui/src/components/sheet/cellIds.ts | 24 ++ .../components/sheet/selection/IdSequence.ts | 50 ++++ .../src/components/sheet/selection/Plane.ts | 72 ++++++ .../components/sheet/selection/Selection.ts | 243 ++++++++++++++++++ .../components/sheet/tests/cellIds.test.ts | 27 ++ .../src/utils/__tests__/iterUtils.test.ts | 21 ++ mathesar_ui/src/utils/iterUtils.ts | 14 + 8 files changed, 452 insertions(+) create mode 100644 mathesar_ui/src/components/sheet/cellIds.ts create mode 100644 mathesar_ui/src/components/sheet/selection/IdSequence.ts create mode 100644 mathesar_ui/src/components/sheet/selection/Plane.ts create mode 100644 mathesar_ui/src/components/sheet/selection/Selection.ts create mode 100644 mathesar_ui/src/components/sheet/tests/cellIds.test.ts create mode 100644 mathesar_ui/src/utils/__tests__/iterUtils.test.ts create mode 100644 mathesar_ui/src/utils/iterUtils.ts diff --git a/mathesar_ui/.eslintrc.cjs b/mathesar_ui/.eslintrc.cjs index 5487252656..7de5aaf976 100644 --- a/mathesar_ui/.eslintrc.cjs +++ b/mathesar_ui/.eslintrc.cjs @@ -21,6 +21,7 @@ module.exports = { rules: { 'import/no-extraneous-dependencies': ['error', { devDependencies: true }], 'no-console': ['warn', { allow: ['error'] }], + 'generator-star-spacing': 'off', '@typescript-eslint/explicit-member-accessibility': 'off', '@typescript-eslint/ban-ts-comment': [ 'error', diff --git a/mathesar_ui/src/components/sheet/cellIds.ts b/mathesar_ui/src/components/sheet/cellIds.ts new file mode 100644 index 0000000000..40c8297dea --- /dev/null +++ b/mathesar_ui/src/components/sheet/cellIds.ts @@ -0,0 +1,24 @@ +const CELL_ID_DELIMITER = '-'; + +/** + * We can serialize a cell id this way only because we're confident that the + * rowId will never contain the delimiter. Some columnIds _do_ contain + * delimiters (e.g. in the Data Explorer), but that's okay because we can still + * separate the values based on the first delimiter. + */ +export function makeCellId(rowId: string, columnId: string): string { + return `${rowId}${CELL_ID_DELIMITER}${columnId}`; +} + +export function parseCellId(cellId: string): { + rowId: string; + columnId: string; +} { + const delimiterIndex = cellId.indexOf(CELL_ID_DELIMITER); + if (delimiterIndex === -1) { + throw new Error(`Unable to parse cell id without a delimiter: ${cellId}.`); + } + const rowId = cellId.slice(0, delimiterIndex); + const columnId = cellId.slice(delimiterIndex + 1); + return { rowId, columnId }; +} diff --git a/mathesar_ui/src/components/sheet/selection/IdSequence.ts b/mathesar_ui/src/components/sheet/selection/IdSequence.ts new file mode 100644 index 0000000000..85eb73ebe6 --- /dev/null +++ b/mathesar_ui/src/components/sheet/selection/IdSequence.ts @@ -0,0 +1,50 @@ +import { ImmutableMap } from '@mathesar/component-library'; + +export default class IdSequence { + private readonly values: Id[]; + + /** Maps the id value to its index */ + private readonly indexLookup: ImmutableMap; + + /** + * @throws Error if duplicate values are provided + */ + constructor(values: Id[]) { + this.values = values; + this.indexLookup = new ImmutableMap( + values.map((value, index) => [value, index]), + ); + if (new Set(values).size !== values.length) { + throw new Error('Duplicate values are not allowed within an IdSequence.'); + } + } + + get length(): number { + return this.values.length; + } + + /** + * Return an iterator of all values between the two provided values, + * inclusive. Iteration occurs in the order stored. The two provided values + * may be present in any order. + * + * @throws an Error if either value is not present in the sequence + */ + range(a: Id, b: Id): Iterable { + const aIndex = this.indexLookup.get(a); + const bIndex = this.indexLookup.get(b); + + if (aIndex === undefined || bIndex === undefined) { + throw new Error('Id value not found within sequence.'); + } + + const startIndex = Math.min(aIndex, bIndex); + const endIndex = Math.max(aIndex, bIndex); + + return this.values.slice(startIndex, endIndex + 1); + } + + [Symbol.iterator](): Iterator { + return this.values[Symbol.iterator](); + } +} diff --git a/mathesar_ui/src/components/sheet/selection/Plane.ts b/mathesar_ui/src/components/sheet/selection/Plane.ts new file mode 100644 index 0000000000..cec6abe732 --- /dev/null +++ b/mathesar_ui/src/components/sheet/selection/Plane.ts @@ -0,0 +1,72 @@ +import { map } from 'iter-tools'; + +import { cartesianProduct } from '@mathesar/utils/iterUtils'; +import type IdSequence from './IdSequence'; +import { makeCellId } from '../cellIds'; + +function makeCells( + rowIds: Iterable, + columnIds: Iterable, +): Iterable { + return map( + ([rowId, columnId]) => makeCellId(rowId, columnId), + cartesianProduct(rowIds, columnIds), + ); +} + +export default class Plane { + readonly rowIds: IdSequence; + + readonly columnIds: IdSequence; + + readonly placeholderRowId: string | undefined; + + constructor( + rowIds: IdSequence, + columnIds: IdSequence, + placeholderRowId: string | undefined, + ) { + this.rowIds = rowIds; + this.columnIds = columnIds; + this.placeholderRowId = placeholderRowId; + } + + /** + * @returns an iterable of all the data cells in the plane. This does not + * include header cells or placeholder cells. + */ + allDataCells(): Iterable { + return makeCells(this.rowIds, this.columnIds); + } + + /** + * @returns an iterable of all the data cells in the plane that are in or + * between the two given rows. This does not include header cells. + * + * @throws Error if the id of the placeholder row is specified because the + * placeholder row is not a data row. + */ + dataCellsInRowRange(rowIdA: string, rowIdB: string): Iterable { + return makeCells(this.rowIds.range(rowIdA, rowIdB), this.columnIds); + } + + /** + * @returns an iterable of all the data cells in the plane that are in or + * between the two given columns. This does not include header cells or + * placeholder cells. + */ + dataCellsInColumnRange( + columnIdA: string, + columnIdB: string, + ): Iterable { + return makeCells(this.rowIds, this.columnIds.range(columnIdA, columnIdB)); + } + + get hasResultRows(): boolean { + return this.rowIds.length > 0; + } + + get hasPlaceholder(): boolean { + return this.placeholderRowId !== undefined; + } +} diff --git a/mathesar_ui/src/components/sheet/selection/Selection.ts b/mathesar_ui/src/components/sheet/selection/Selection.ts new file mode 100644 index 0000000000..44f70d27c4 --- /dev/null +++ b/mathesar_ui/src/components/sheet/selection/Selection.ts @@ -0,0 +1,243 @@ +import { first } from 'iter-tools'; + +import { ImmutableSet } from '@mathesar/component-library'; +import type Plane from './Plane'; +import { parseCellId } from '../cellIds'; + +export enum Direction { + Up = 'up', + Down = 'down', + Left = 'left', + Right = 'right', +} + +interface Basis { + /** + * - `'dataCells'` means that the selection contains data cells. This is by + * far the most common type of selection basis. + * + * - `'emptyColumns'` is used when the sheet has no rows. In this case we + * still want to allow the user to select columns, so we use this basis. + * + * - `'placeholderCell'` is used when the user is selecting a cell in the + * placeholder row. This is a special case because we don't want to allow + * the user to select multiple cells in the placeholder row, and we also + * don't want to allow selections that include cells in data rows _and_ the + * placeholder row. + */ + readonly type: 'dataCells' | 'emptyColumns' | 'placeholderCell'; + readonly activeCellId: string | undefined; + readonly cellIds: ImmutableSet; + readonly rowIds: ImmutableSet; + readonly columnIds: ImmutableSet; +} + +function basisFromDataCells( + cellIds: Iterable, + activeCellId?: string, +): Basis { + const parsedCells = [...cellIds].map(parseCellId); + return { + type: 'dataCells', + activeCellId: activeCellId ?? first(cellIds), + cellIds: new ImmutableSet(cellIds), + columnIds: new ImmutableSet(parsedCells.map((cellId) => cellId.columnId)), + rowIds: new ImmutableSet(parsedCells.map((cellId) => cellId.rowId)), + }; +} + +function basisFromEmptyColumns(columnIds: Iterable): Basis { + return { + type: 'emptyColumns', + activeCellId: undefined, + cellIds: new ImmutableSet(), + columnIds: new ImmutableSet(columnIds), + rowIds: new ImmutableSet(), + }; +} + +function basisFromZeroEmptyColumns(): Basis { + return basisFromEmptyColumns([]); +} + +function basisFromPlaceholderCell(activeCellId: string): Basis { + return { + type: 'placeholderCell', + activeCellId, + cellIds: new ImmutableSet([activeCellId]), + columnIds: new ImmutableSet([parseCellId(activeCellId).columnId]), + rowIds: new ImmutableSet(), + }; +} + +export default class Selection { + private readonly plane: Plane; + + private readonly basis: Basis; + + constructor(plane: Plane, basis: Basis) { + this.plane = plane; + this.basis = basis; + } + + get activeCellId() { + return this.basis.activeCellId; + } + + get cellIds() { + return this.basis.cellIds; + } + + get rowIds() { + return this.basis.rowIds; + } + + get columnIds() { + return this.basis.columnIds; + } + + private withBasis(basis: Basis): Selection { + return new Selection(this.plane, basis); + } + + /** + * @returns a new selection with all cells selected. The active cell will be the + * cell in the first row and first column. + */ + ofAllDataCells(): Selection { + if (!this.plane.hasResultRows) { + return this.withBasis(basisFromZeroEmptyColumns()); + } + return this.withBasis(basisFromDataCells(this.plane.allDataCells())); + } + + /** + * @returns a new selection with the cell in the first row and first column + * selected. + */ + ofFirstDataCell(): Selection { + const firstCellId = first(this.plane.allDataCells()); + if (firstCellId === undefined) { + return this.withBasis(basisFromZeroEmptyColumns()); + } + return this.withBasis(basisFromDataCells([firstCellId])); + } + + /** + * @returns a new selection with all rows selected between (and including) the + * provided rows. + * + * If either of the provided rows are placeholder rows, then the last data row + * will be used in their place. This ensures that the selection is made only + * of data rows, and will never include the placeholder row, even if a user + * drags to select it. + */ + ofRowRange(rowIdA: string, rowIdB: string): Selection { + throw new Error('Not implemented'); + } + + /** + * @returns a new selection of all data cells in all columns between the + * provided columnIds, inclusive. + */ + ofColumnRange(columnIdA: string, columnIdB: string): Selection { + if (!this.plane.hasResultRows) { + throw new Error('Not implemented'); + } + throw new Error('Not implemented'); + } + + /** + * @returns a new selection formed by the rectangle between the provided + * cells, inclusive. + * + * If either of the provided cells are in the placeholder row, then the cell + * in the last data row will be used in its place. This ensures that the + * selection is made only of data cells, and will never include cells in the + * placeholder row, even if a user drags to select a cell in it. + */ + ofCellRange(cellIdA: string, cellIdB: string): Selection { + throw new Error('Not implemented'); + } + + /** + * @returns a new selection formed from one cell within the placeholder row. + * Note that we do not support selections of multiple cells in the placeholder + * row. + */ + atPlaceholderCell(cellId: string): Selection { + throw new Error('Not implemented'); + } + + /** + * @returns a new selection that fits within the provided plane. This is + * useful when a column is deleted, reordered, or inserted. + */ + forNewPlane(plane: Plane): Selection { + throw new Error('Not implemented'); + } + + /** + * @returns a new selection formed by the rectangle between the currently + * active cell and provided cell, inclusive. + * + * This operation is designed to mimic the behavior of Google Sheets when + * shift-clicking a specific cell, or when dragging to create a new selection. + * A new selection is created that contains all cells in a rectangle bounded + * by the active cell (also the first cell selected when dragging) and the + * provided cell. + */ + drawnToCell(cellId: string): Selection { + throw new Error('Not implemented'); + } + + /** + * @returns a new selection formed by the cells in all the rows between the + * active cell and the provided row, inclusive. + */ + drawnToRow(rowId: string): Selection { + throw new Error('Not implemented'); + } + + drawnToColumn(columnId: string): Selection { + throw new Error('Not implemented'); + } + + /** + * @returns a new selection that mimics the behavior of arrow keys in + * spreadsheets. If the active cell can be moved in the provided direction, + * then a new selection is created with only that one cell selected. + */ + collapsedAndMoved(direction: Direction): Selection { + throw new Error('Not implemented'); + } + + /** + * @returns a new selection with the active cell moved within the selection, + * left to right, top to bottom. + * + * This is to handle the `Tab` and `Shift+Tab` keys. + */ + withActiveCellAdvanced(direction: 'forward' | 'back' = 'forward'): Selection { + throw new Error('Not implemented'); + } + + /** + * @returns a new selection that is grown or shrunk to mimic the behavior of + * Google Sheets when manipulating selections via keyboard shortcuts like + * `Shift+Down`. The selection is deformed in the direction of the provided + * argument. The selection is _shrunk_ if doing so will keep the active cell + * within the selection. Otherwise, the selection is _grown_. + * + * Note that other spreadsheet applications have slightly different behavior + * for Shift + arrow keys. For example, LibreOffice Calc maintains state for + * the origin of the selection separate from the active cell. The two cells + * may be different if the user presses `Tab` after making a selection. In + * this case, the selection will be resized with respect to the origin, not + * the active cell. We chose to mimic Google Sheets behavior here because it + * is simpler. + */ + resized(direction: Direction): Selection { + throw new Error('Not implemented'); + } +} diff --git a/mathesar_ui/src/components/sheet/tests/cellIds.test.ts b/mathesar_ui/src/components/sheet/tests/cellIds.test.ts new file mode 100644 index 0000000000..a4c8702b66 --- /dev/null +++ b/mathesar_ui/src/components/sheet/tests/cellIds.test.ts @@ -0,0 +1,27 @@ +import { parseCellId } from '../cellIds'; + +test.each( + // prettier-ignore + [ + // cellId , rowId , columnId + ['a-b' , 'a' , 'b' ], + ['a-b-c' , 'a' , 'b-c' ], + ['a--b' , 'a' , '-b' ], + [' a - b ' , ' a ' , ' b ' ], + ['-' , '' , '' ], + [' - ' , ' ' , ' ' ], + ['--' , '' , '-' ], + ['-a-b' , '' , 'a-b' ], + ['a-' , 'a' , '' ], + ], +)('parseCellId success %#', (cellId, rowId, columnId) => { + const result = parseCellId(cellId); + expect(result.rowId).toBe(rowId); + expect(result.columnId).toBe(columnId); +}); + +test.each([[''], ['foobar']])('parseCellId failure %#', (cellId) => { + expect(() => { + parseCellId(cellId); + }).toThrow(); +}); diff --git a/mathesar_ui/src/utils/__tests__/iterUtils.test.ts b/mathesar_ui/src/utils/__tests__/iterUtils.test.ts new file mode 100644 index 0000000000..a87d5dc5ef --- /dev/null +++ b/mathesar_ui/src/utils/__tests__/iterUtils.test.ts @@ -0,0 +1,21 @@ +import { cartesianProduct } from '../iterUtils'; + +test.each([ + [[], [], []], + [[], ['a', 'b'], []], + [['a', 'b'], [], []], + [ + ['a', 'b'], + ['x', 'y', 'z'], + [ + ['a', 'x'], + ['a', 'y'], + ['a', 'z'], + ['b', 'x'], + ['b', 'y'], + ['b', 'z'], + ], + ], +])('cartesianProduct', (a, b, result) => { + expect([...cartesianProduct(a, b)]).toStrictEqual(result); +}); diff --git a/mathesar_ui/src/utils/iterUtils.ts b/mathesar_ui/src/utils/iterUtils.ts new file mode 100644 index 0000000000..6a8b915719 --- /dev/null +++ b/mathesar_ui/src/utils/iterUtils.ts @@ -0,0 +1,14 @@ +export function cartesianProduct( + a: Iterable, + b: Iterable, +): Iterable<[T, U]> { + return { + *[Symbol.iterator]() { + for (const x of a) { + for (const y of b) { + yield [x, y]; + } + } + }, + }; +} From be561de892212d04c87f3415d1c81375a49e1b61 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jul 2023 12:35:46 -0400 Subject: [PATCH 0002/1141] Upgrade iter-tools package --- mathesar_ui/package-lock.json | 6 +++--- mathesar_ui/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mathesar_ui/package-lock.json b/mathesar_ui/package-lock.json index bd4d9d4ed3..6e78aa6ad4 100644 --- a/mathesar_ui/package-lock.json +++ b/mathesar_ui/package-lock.json @@ -18511,9 +18511,9 @@ } }, "iter-tools": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/iter-tools/-/iter-tools-7.4.0.tgz", - "integrity": "sha512-twcHD87GOpbnfFmOtEQMcZuJzBCViNaYx4//70KkkjFcQTkKcUfcuni5G5dfO4Uyb80KIsWnVGZ4vvkpmYCNQQ==", + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/iter-tools/-/iter-tools-7.5.3.tgz", + "integrity": "sha512-iEcHpgM9cn6tsI5MewqxyEega9KPbIDytQTEnu6c0MtlQQhQFofssYuRqxCarZgUdzliepRZPwwwflE4wAIjaA==", "requires": { "@babel/runtime": "^7.12.1" } diff --git a/mathesar_ui/package.json b/mathesar_ui/package.json index b092a7c033..9a25e205b9 100644 --- a/mathesar_ui/package.json +++ b/mathesar_ui/package.json @@ -63,7 +63,7 @@ "dayjs": "^1.11.5", "fast-diff": "^1.2.0", "flatpickr": "^4.6.13", - "iter-tools": "^7.4.0", + "iter-tools": "^7.5.3", "js-cookie": "^3.0.1", "papaparse": "^5.4.1", "perfect-scrollbar": "^1.5.5" From 29aedab6ae6e605adc8a4c4630442cb7b571a790 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jul 2023 15:15:31 -0400 Subject: [PATCH 0003/1141] Fill in more logic within Selection scaffolding --- .../components/sheet/selection/Direction.ts | 50 +++++++ .../components/sheet/selection/IdSequence.ts | 82 +++++++++- .../src/components/sheet/selection/Plane.ts | 141 +++++++++++++++++- .../components/sheet/selection/Selection.ts | 122 ++++++++++----- .../selection/__tests__/IdSequence.test.ts | 44 ++++++ .../sheet/selection/__tests__/Plane.test.ts | 58 +++++++ 6 files changed, 456 insertions(+), 41 deletions(-) create mode 100644 mathesar_ui/src/components/sheet/selection/Direction.ts create mode 100644 mathesar_ui/src/components/sheet/selection/__tests__/IdSequence.test.ts create mode 100644 mathesar_ui/src/components/sheet/selection/__tests__/Plane.test.ts diff --git a/mathesar_ui/src/components/sheet/selection/Direction.ts b/mathesar_ui/src/components/sheet/selection/Direction.ts new file mode 100644 index 0000000000..2693dd9000 --- /dev/null +++ b/mathesar_ui/src/components/sheet/selection/Direction.ts @@ -0,0 +1,50 @@ +export enum Direction { + Up = 'up', + Down = 'down', + Left = 'left', + Right = 'right', +} + +export function getDirection(event: KeyboardEvent): Direction | undefined { + const { key } = event; + const shift = event.shiftKey; + switch (true) { + case shift && key === 'Tab': + return Direction.Left; + case shift: + return undefined; + case key === 'ArrowUp': + return Direction.Up; + case key === 'ArrowDown': + return Direction.Down; + case key === 'ArrowLeft': + return Direction.Left; + case key === 'ArrowRight': + case key === 'Tab': + return Direction.Right; + default: + return undefined; + } +} + +export function getColumnOffset(direction: Direction): number { + switch (direction) { + case Direction.Left: + return -1; + case Direction.Right: + return 1; + default: + return 0; + } +} + +export function getRowOffset(direction: Direction): number { + switch (direction) { + case Direction.Up: + return -1; + case Direction.Down: + return 1; + default: + return 0; + } +} diff --git a/mathesar_ui/src/components/sheet/selection/IdSequence.ts b/mathesar_ui/src/components/sheet/selection/IdSequence.ts index 85eb73ebe6..ea0396f621 100644 --- a/mathesar_ui/src/components/sheet/selection/IdSequence.ts +++ b/mathesar_ui/src/components/sheet/selection/IdSequence.ts @@ -1,4 +1,5 @@ import { ImmutableMap } from '@mathesar/component-library'; +import { filter, findBest, firstHighest, firstLowest } from 'iter-tools'; export default class IdSequence { private readonly values: Id[]; @@ -19,6 +20,10 @@ export default class IdSequence { } } + private getIndex(value: Id): number | undefined { + return this.indexLookup.get(value); + } + get length(): number { return this.values.length; } @@ -31,8 +36,8 @@ export default class IdSequence { * @throws an Error if either value is not present in the sequence */ range(a: Id, b: Id): Iterable { - const aIndex = this.indexLookup.get(a); - const bIndex = this.indexLookup.get(b); + const aIndex = this.getIndex(a); + const bIndex = this.getIndex(b); if (aIndex === undefined || bIndex === undefined) { throw new Error('Id value not found within sequence.'); @@ -44,6 +49,79 @@ export default class IdSequence { return this.values.slice(startIndex, endIndex + 1); } + get first(): Id | undefined { + return this.values[0]; + } + + get last(): Id | undefined { + return this.values[this.values.length - 1]; + } + + has(value: Id): boolean { + return this.getIndex(value) !== undefined; + } + + /** + * This method is used for `min` and `max`, but could potentially be used for + * other things too. + * + * @param comparator Corresponds to the [iter-tools comparators][1] + * + * [1]: + * https://github.com/iter-tools/iter-tools/blob/d7.5/API.md#compare-values-and-return-true-or-false + */ + best( + values: Iterable, + comparator: (best: number, v: number) => boolean, + ): Id | undefined { + const validValues = filter((v) => this.has(v), values); + return findBest(comparator, (v) => this.getIndex(v) ?? 0, validValues); + } + + min(values: Iterable): Id | undefined { + return this.best(values, firstLowest); + } + + max(values: Iterable): Id | undefined { + return this.best(values, firstHighest); + } + + /** + * @returns the value positioned relative to the given value by the given + * offset. If the offset is 0, the given value will be returned. If the offset + * is positive, the returned value will be that many positions _after_ the + * given value. If no such value is present, then `undefined` will be + * returned. + */ + offset(value: Id, offset: number): Id | undefined { + if (offset === 0) { + return this.has(value) ? value : undefined; + } + const index = this.getIndex(value); + if (index === undefined) { + return undefined; + } + return this.values[index + offset]; + } + + /** + * This is similar to `offset`, but accepts an iterable of values and treats + * them as a unified block to be collapsed into one value. When `offset` is + * positive, the returned value will be that many positions _after_ the last + * value in the block. If no such value is present, then `undefined` will be + * returned. If offset is zero, then `undefined` will be returned. + */ + collapsedOffset(values: Iterable, offset: number): Id | undefined { + if (offset === 0) { + return undefined; + } + const outerValue = offset > 0 ? this.max(values) : this.min(values); + if (outerValue === undefined) { + return undefined; + } + return this.offset(outerValue, offset); + } + [Symbol.iterator](): Iterator { return this.values[Symbol.iterator](); } diff --git a/mathesar_ui/src/components/sheet/selection/Plane.ts b/mathesar_ui/src/components/sheet/selection/Plane.ts index cec6abe732..7b70dd6cb4 100644 --- a/mathesar_ui/src/components/sheet/selection/Plane.ts +++ b/mathesar_ui/src/components/sheet/selection/Plane.ts @@ -1,8 +1,9 @@ import { map } from 'iter-tools'; import { cartesianProduct } from '@mathesar/utils/iterUtils'; +import { makeCellId, parseCellId } from '../cellIds'; +import { Direction, getColumnOffset, getRowOffset } from './Direction'; import type IdSequence from './IdSequence'; -import { makeCellId } from '../cellIds'; function makeCells( rowIds: Iterable, @@ -14,6 +15,35 @@ function makeCells( ); } +/** + * This describes the different kinds of cells that can be adjacent to a given + * cell in a particular direction. + */ +export type AdjacentCell = + | { + type: 'dataCell'; + cellId: string; + } + | { + type: 'placeholderCell'; + cellId: string; + } + | { + type: 'none'; + }; + +function noAdjacentCell(): AdjacentCell { + return { type: 'none' }; +} + +function adjacentDataCell(cellId: string): AdjacentCell { + return { type: 'dataCell', cellId }; +} + +function adjacentPlaceholderCell(cellId: string): AdjacentCell { + return { type: 'placeholderCell', cellId }; +} + export default class Plane { readonly rowIds: IdSequence; @@ -31,6 +61,19 @@ export default class Plane { this.placeholderRowId = placeholderRowId; } + /** + * @returns the row id that should be used to represent the given row id in + * the plane. If the given row id is the placeholder row id, then the last + * row id in the plane will be returned. Otherwise, the given row id will be + * returned. + */ + private normalizeFlexibleRowId(rowId: string): string | undefined { + if (rowId === this.placeholderRowId) { + return this.rowIds.last; + } + return rowId; + } + /** * @returns an iterable of all the data cells in the plane. This does not * include header cells or placeholder cells. @@ -43,11 +86,24 @@ export default class Plane { * @returns an iterable of all the data cells in the plane that are in or * between the two given rows. This does not include header cells. * - * @throws Error if the id of the placeholder row is specified because the - * placeholder row is not a data row. + * If either of the provided rows are placeholder rows, then the last data row + * will be used in their place. This ensures that the selection is made only + * of data rows, and will never include the placeholder row, even if a user + * drags to select it. */ - dataCellsInRowRange(rowIdA: string, rowIdB: string): Iterable { - return makeCells(this.rowIds.range(rowIdA, rowIdB), this.columnIds); + dataCellsInFlexibleRowRange( + rowIdA: string, + rowIdB: string, + ): Iterable { + const a = this.normalizeFlexibleRowId(rowIdA); + if (a === undefined) { + return []; + } + const b = this.normalizeFlexibleRowId(rowIdB); + if (b === undefined) { + return []; + } + return makeCells(this.rowIds.range(a, b), this.columnIds); } /** @@ -62,6 +118,81 @@ export default class Plane { return makeCells(this.rowIds, this.columnIds.range(columnIdA, columnIdB)); } + /** + * @returns an iterable of all the data cells in the plane that are in or + * between the two given cell. This does not include header cells. + * + * If either of the provided cells are placeholder cells, then cells in the + * last row and last column will be used in their place. This ensures that the + * selection is made only of data cells, and will never include the + * placeholder cell, even if a user drags to select it. + */ + dataCellsInFlexibleCellRange( + cellIdA: string, + cellIdB: string, + ): Iterable { + const cellA = parseCellId(cellIdA); + const cellB = parseCellId(cellIdB); + const rowIdA = this.normalizeFlexibleRowId(cellA.rowId); + if (rowIdA === undefined) { + return []; + } + const rowIdB = this.normalizeFlexibleRowId(cellB.rowId); + if (rowIdB === undefined) { + return []; + } + const rowIds = this.rowIds.range(rowIdA, rowIdB); + const columnIds = this.columnIds.range(cellA.columnId, cellB.columnId); + return makeCells(rowIds, columnIds); + } + + getAdjacentCell(cellId: string, direction: Direction): AdjacentCell { + const cell = parseCellId(cellId); + + const columnOffset = getColumnOffset(direction); + const newColumnId = this.columnIds.offset(cell.columnId, columnOffset); + if (newColumnId === undefined) { + return noAdjacentCell(); + } + + if (cell.rowId === this.placeholderRowId) { + if (direction === Direction.Up) { + const lastRowId = this.rowIds.last; + if (lastRowId === undefined) { + // Can't go up from the placeholder row if there are no data rows + return noAdjacentCell(); + } + // Move up from the placeholder row into the last data row + return adjacentDataCell(makeCellId(lastRowId, newColumnId)); + } + if (direction === Direction.Down) { + // Can't go down from the placeholder row + return noAdjacentCell(); + } + // Move laterally within the placeholder row + return adjacentPlaceholderCell(makeCellId(cell.rowId, newColumnId)); + } + + const rowOffset = getRowOffset(direction); + const newRowId = this.rowIds.offset(cell.rowId, rowOffset); + if (newRowId === undefined) { + if (direction === Direction.Down) { + if (this.placeholderRowId === undefined) { + // Can't go down from the last data row if there is no placeholder row + return noAdjacentCell(); + } + const newCellId = makeCellId(this.placeholderRowId, newColumnId); + // Move down from the last data row into the placeholder row + return adjacentPlaceholderCell(newCellId); + } + // Can't move up from the first data row + return noAdjacentCell(); + } + + // Normal movement from one data cell to another data cell + return adjacentDataCell(makeCellId(newRowId, newColumnId)); + } + get hasResultRows(): boolean { return this.rowIds.length > 0; } diff --git a/mathesar_ui/src/components/sheet/selection/Selection.ts b/mathesar_ui/src/components/sheet/selection/Selection.ts index 44f70d27c4..c98e5570dc 100644 --- a/mathesar_ui/src/components/sheet/selection/Selection.ts +++ b/mathesar_ui/src/components/sheet/selection/Selection.ts @@ -1,31 +1,28 @@ import { first } from 'iter-tools'; import { ImmutableSet } from '@mathesar/component-library'; -import type Plane from './Plane'; +import { assertExhaustive } from '@mathesar/utils/typeUtils'; import { parseCellId } from '../cellIds'; +import { Direction, getColumnOffset } from './Direction'; +import type Plane from './Plane'; -export enum Direction { - Up = 'up', - Down = 'down', - Left = 'left', - Right = 'right', -} +/** + * - `'dataCells'` means that the selection contains data cells. This is by + * far the most common type of selection basis. + * + * - `'emptyColumns'` is used when the sheet has no rows. In this case we + * still want to allow the user to select columns, so we use this basis. + * + * - `'placeholderCell'` is used when the user is selecting a cell in the + * placeholder row. This is a special case because we don't want to allow + * the user to select multiple cells in the placeholder row, and we also + * don't want to allow selections that include cells in data rows _and_ the + * placeholder row. + */ +type BasisType = 'dataCells' | 'emptyColumns' | 'placeholderCell'; interface Basis { - /** - * - `'dataCells'` means that the selection contains data cells. This is by - * far the most common type of selection basis. - * - * - `'emptyColumns'` is used when the sheet has no rows. In this case we - * still want to allow the user to select columns, so we use this basis. - * - * - `'placeholderCell'` is used when the user is selecting a cell in the - * placeholder row. This is a special case because we don't want to allow - * the user to select multiple cells in the placeholder row, and we also - * don't want to allow selections that include cells in data rows _and_ the - * placeholder row. - */ - readonly type: 'dataCells' | 'emptyColumns' | 'placeholderCell'; + readonly type: BasisType; readonly activeCellId: string | undefined; readonly cellIds: ImmutableSet; readonly rowIds: ImmutableSet; @@ -101,8 +98,8 @@ export default class Selection { } /** - * @returns a new selection with all cells selected. The active cell will be the - * cell in the first row and first column. + * @returns a new selection with all cells selected. The active cell will be + * the cell in the first row and first column. */ ofAllDataCells(): Selection { if (!this.plane.hasResultRows) { @@ -133,7 +130,11 @@ export default class Selection { * drags to select it. */ ofRowRange(rowIdA: string, rowIdB: string): Selection { - throw new Error('Not implemented'); + return this.withBasis( + basisFromDataCells( + this.plane.dataCellsInFlexibleRowRange(rowIdA, rowIdB), + ), + ); } /** @@ -141,10 +142,12 @@ export default class Selection { * provided columnIds, inclusive. */ ofColumnRange(columnIdA: string, columnIdB: string): Selection { - if (!this.plane.hasResultRows) { - throw new Error('Not implemented'); - } - throw new Error('Not implemented'); + const newBasis = this.plane.hasResultRows + ? basisFromDataCells( + this.plane.dataCellsInColumnRange(columnIdA, columnIdB), + ) + : basisFromEmptyColumns(this.plane.columnIds.range(columnIdA, columnIdB)); + return this.withBasis(newBasis); } /** @@ -157,7 +160,11 @@ export default class Selection { * placeholder row, even if a user drags to select a cell in it. */ ofCellRange(cellIdA: string, cellIdB: string): Selection { - throw new Error('Not implemented'); + return this.withBasis( + basisFromDataCells( + this.plane.dataCellsInFlexibleCellRange(cellIdA, cellIdB), + ), + ); } /** @@ -166,7 +173,14 @@ export default class Selection { * row. */ atPlaceholderCell(cellId: string): Selection { - throw new Error('Not implemented'); + return this.withBasis(basisFromPlaceholderCell(cellId)); + } + + /** + * @returns a new selection formed from one cell within the data rows. + */ + atDataCell(cellId: string): Selection { + return this.withBasis(basisFromDataCells([cellId], cellId)); } /** @@ -188,7 +202,7 @@ export default class Selection { * provided cell. */ drawnToCell(cellId: string): Selection { - throw new Error('Not implemented'); + return this.ofCellRange(this.activeCellId ?? cellId, cellId); } /** @@ -196,11 +210,19 @@ export default class Selection { * active cell and the provided row, inclusive. */ drawnToRow(rowId: string): Selection { - throw new Error('Not implemented'); + const activeRowId = this.activeCellId + ? parseCellId(this.activeCellId).rowId + : rowId; + return this.ofRowRange(activeRowId, rowId); } drawnToColumn(columnId: string): Selection { - throw new Error('Not implemented'); + // TODO improve handling for empty columns + + const activeColumnId = this.activeCellId + ? parseCellId(this.activeCellId).columnId + : columnId; + return this.ofColumnRange(activeColumnId, columnId); } /** @@ -209,7 +231,39 @@ export default class Selection { * then a new selection is created with only that one cell selected. */ collapsedAndMoved(direction: Direction): Selection { - throw new Error('Not implemented'); + if (this.basis.type === 'emptyColumns') { + const offset = getColumnOffset(direction); + const newActiveColumnId = this.plane.columnIds.collapsedOffset( + this.basis.columnIds, + offset, + ); + if (newActiveColumnId === undefined) { + // If we couldn't shift in the direction, then do nothing + return this; + } + return this.withBasis(basisFromEmptyColumns([newActiveColumnId])); + } + + if (this.activeCellId === undefined) { + // If no cells are selected, then select the first data cell + return this.ofFirstDataCell(); + } + + const adjacent = this.plane.getAdjacentCell(this.activeCellId, direction); + + if (adjacent.type === 'none') { + // If we can't move anywhere, then do nothing + return this; + } + if (adjacent.type === 'dataCell') { + // Move to an adjacent data cell + return this.atDataCell(adjacent.cellId); + } + if (adjacent.type === 'placeholderCell') { + // Move to an adjacent placeholder cell + return this.atPlaceholderCell(adjacent.cellId); + } + return assertExhaustive(adjacent); } /** diff --git a/mathesar_ui/src/components/sheet/selection/__tests__/IdSequence.test.ts b/mathesar_ui/src/components/sheet/selection/__tests__/IdSequence.test.ts new file mode 100644 index 0000000000..14da6d0a24 --- /dev/null +++ b/mathesar_ui/src/components/sheet/selection/__tests__/IdSequence.test.ts @@ -0,0 +1,44 @@ +import IdSequence from '../IdSequence'; + +test('IdSequence', () => { + const s = new IdSequence(['h', 'i', 'j', 'k', 'l']); + expect(s.length).toBe(5); + expect(s.first).toBe('h'); + expect(s.last).toBe('l'); + + expect([...s]).toEqual(['h', 'i', 'j', 'k', 'l']); + + expect(s.min(['i', 'j', 'k'])).toBe('i'); + expect(s.min(['k', 'j', 'i'])).toBe('i'); + expect(s.min(['i', 'NOPE'])).toBe('i'); + expect(s.min(['NOPE'])).toBe(undefined); + expect(s.min([])).toBe(undefined); + + expect(s.max(['i', 'j', 'k'])).toBe('k'); + expect(s.max(['k', 'j', 'i'])).toBe('k'); + expect(s.max(['i', 'NOPE'])).toBe('i'); + expect(s.max(['NOPE'])).toBe(undefined); + expect(s.max([])).toBe(undefined); + + expect([...s.range('i', 'k')]).toEqual(['i', 'j', 'k']); + expect([...s.range('k', 'i')]).toEqual(['i', 'j', 'k']); + expect([...s.range('i', 'i')]).toEqual(['i']); + expect(() => s.range('i', 'NOPE')).toThrow(); + + expect(s.offset('i', 0)).toBe('i'); + expect(s.offset('i', 1)).toBe('j'); + expect(s.offset('i', 2)).toBe('k'); + expect(s.offset('i', -1)).toBe('h'); + expect(s.offset('i', -2)).toBe(undefined); + expect(s.offset('NOPE', 0)).toBe(undefined); + expect(s.offset('NOPE', 1)).toBe(undefined); + + expect(s.collapsedOffset(['i', 'k'], 0)).toBe(undefined); + expect(s.collapsedOffset(['i', 'k'], 1)).toBe('l'); + expect(s.collapsedOffset(['i', 'k'], 2)).toBe(undefined); + expect(s.collapsedOffset(['i', 'k'], -1)).toBe('h'); + expect(s.collapsedOffset(['i', 'k'], -2)).toBe(undefined); + expect(s.collapsedOffset(['i', 'NOPE'], 0)).toBe(undefined); + expect(s.collapsedOffset(['i', 'NOPE'], 1)).toBe('j'); + expect(s.collapsedOffset([], 0)).toBe(undefined); +}); diff --git a/mathesar_ui/src/components/sheet/selection/__tests__/Plane.test.ts b/mathesar_ui/src/components/sheet/selection/__tests__/Plane.test.ts new file mode 100644 index 0000000000..6e70548ed2 --- /dev/null +++ b/mathesar_ui/src/components/sheet/selection/__tests__/Plane.test.ts @@ -0,0 +1,58 @@ +import IdSequence from '../IdSequence'; +import Plane from '../Plane'; +import { Direction } from '../Direction'; + +test('Plane with placeholder row', () => { + const p = new Plane( + new IdSequence(['r1', 'r2', 'r3', 'r4']), + new IdSequence(['c1', 'c2', 'c3', 'c4']), + 'PL', + ); + + expect([...p.allDataCells()]).toEqual([ + 'r1-c1', + 'r1-c2', + 'r1-c3', + 'r1-c4', + 'r2-c1', + 'r2-c2', + 'r2-c3', + 'r2-c4', + 'r3-c1', + 'r3-c2', + 'r3-c3', + 'r3-c4', + 'r4-c1', + 'r4-c2', + 'r4-c3', + 'r4-c4', + ]); + + expect([...p.dataCellsInFlexibleRowRange('r1', 'r2')].length).toBe(8); + expect([...p.dataCellsInFlexibleRowRange('r1', 'r3')].length).toBe(12); + expect([...p.dataCellsInFlexibleRowRange('r3', 'r4')].length).toBe(8); + expect([...p.dataCellsInFlexibleRowRange('r3', 'PL')].length).toBe(8); + expect([...p.dataCellsInFlexibleRowRange('PL', 'PL')]).toEqual([ + 'r4-c1', + 'r4-c2', + 'r4-c3', + 'r4-c4', + ]); + + expect([...p.dataCellsInColumnRange('c1', 'c2')].length).toBe(8); + expect([...p.dataCellsInColumnRange('c1', 'c3')].length).toBe(12); + + expect(p.getAdjacentCell('r4-c4', Direction.Up)).toEqual({ + type: 'dataCell', + cellId: 'r3-c4', + }); + expect(p.getAdjacentCell('r4-c4', Direction.Right)).toEqual({ type: 'none' }); + expect(p.getAdjacentCell('r4-c4', Direction.Down)).toEqual({ + type: 'placeholderCell', + cellId: 'PL-c4', + }); + expect(p.getAdjacentCell('r4-c4', Direction.Left)).toEqual({ + type: 'dataCell', + cellId: 'r4-c3', + }); +}); From 8b5ccdcb7a54b59b144c8b99602d987b42200886 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jul 2023 15:17:46 -0400 Subject: [PATCH 0004/1141] Improve some naming --- mathesar_ui/src/components/sheet/selection/Plane.ts | 10 +++++----- .../sheet/selection/{IdSequence.ts => Series.ts} | 4 ++-- .../components/sheet/selection/__tests__/Plane.test.ts | 6 +++--- .../__tests__/{IdSequence.test.ts => Series.test.ts} | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) rename mathesar_ui/src/components/sheet/selection/{IdSequence.ts => Series.ts} (98%) rename mathesar_ui/src/components/sheet/selection/__tests__/{IdSequence.test.ts => Series.test.ts} (92%) diff --git a/mathesar_ui/src/components/sheet/selection/Plane.ts b/mathesar_ui/src/components/sheet/selection/Plane.ts index 7b70dd6cb4..20ed0fccc5 100644 --- a/mathesar_ui/src/components/sheet/selection/Plane.ts +++ b/mathesar_ui/src/components/sheet/selection/Plane.ts @@ -3,7 +3,7 @@ import { map } from 'iter-tools'; import { cartesianProduct } from '@mathesar/utils/iterUtils'; import { makeCellId, parseCellId } from '../cellIds'; import { Direction, getColumnOffset, getRowOffset } from './Direction'; -import type IdSequence from './IdSequence'; +import type Series from './Series'; function makeCells( rowIds: Iterable, @@ -45,15 +45,15 @@ function adjacentPlaceholderCell(cellId: string): AdjacentCell { } export default class Plane { - readonly rowIds: IdSequence; + readonly rowIds: Series; - readonly columnIds: IdSequence; + readonly columnIds: Series; readonly placeholderRowId: string | undefined; constructor( - rowIds: IdSequence, - columnIds: IdSequence, + rowIds: Series, + columnIds: Series, placeholderRowId: string | undefined, ) { this.rowIds = rowIds; diff --git a/mathesar_ui/src/components/sheet/selection/IdSequence.ts b/mathesar_ui/src/components/sheet/selection/Series.ts similarity index 98% rename from mathesar_ui/src/components/sheet/selection/IdSequence.ts rename to mathesar_ui/src/components/sheet/selection/Series.ts index ea0396f621..7b8cd45dcc 100644 --- a/mathesar_ui/src/components/sheet/selection/IdSequence.ts +++ b/mathesar_ui/src/components/sheet/selection/Series.ts @@ -1,7 +1,7 @@ import { ImmutableMap } from '@mathesar/component-library'; import { filter, findBest, firstHighest, firstLowest } from 'iter-tools'; -export default class IdSequence { +export default class Series { private readonly values: Id[]; /** Maps the id value to its index */ @@ -16,7 +16,7 @@ export default class IdSequence { values.map((value, index) => [value, index]), ); if (new Set(values).size !== values.length) { - throw new Error('Duplicate values are not allowed within an IdSequence.'); + throw new Error('Duplicate values are not allowed within a Series.'); } } diff --git a/mathesar_ui/src/components/sheet/selection/__tests__/Plane.test.ts b/mathesar_ui/src/components/sheet/selection/__tests__/Plane.test.ts index 6e70548ed2..dd6b89500c 100644 --- a/mathesar_ui/src/components/sheet/selection/__tests__/Plane.test.ts +++ b/mathesar_ui/src/components/sheet/selection/__tests__/Plane.test.ts @@ -1,11 +1,11 @@ -import IdSequence from '../IdSequence'; +import Series from '../Series'; import Plane from '../Plane'; import { Direction } from '../Direction'; test('Plane with placeholder row', () => { const p = new Plane( - new IdSequence(['r1', 'r2', 'r3', 'r4']), - new IdSequence(['c1', 'c2', 'c3', 'c4']), + new Series(['r1', 'r2', 'r3', 'r4']), + new Series(['c1', 'c2', 'c3', 'c4']), 'PL', ); diff --git a/mathesar_ui/src/components/sheet/selection/__tests__/IdSequence.test.ts b/mathesar_ui/src/components/sheet/selection/__tests__/Series.test.ts similarity index 92% rename from mathesar_ui/src/components/sheet/selection/__tests__/IdSequence.test.ts rename to mathesar_ui/src/components/sheet/selection/__tests__/Series.test.ts index 14da6d0a24..8109932d9f 100644 --- a/mathesar_ui/src/components/sheet/selection/__tests__/IdSequence.test.ts +++ b/mathesar_ui/src/components/sheet/selection/__tests__/Series.test.ts @@ -1,7 +1,7 @@ -import IdSequence from '../IdSequence'; +import Series from '../Series'; -test('IdSequence', () => { - const s = new IdSequence(['h', 'i', 'j', 'k', 'l']); +test('Series', () => { + const s = new Series(['h', 'i', 'j', 'k', 'l']); expect(s.length).toBe(5); expect(s.first).toBe('h'); expect(s.last).toBe('l'); From 66f4e637cc1759af7cf08475ea3389513acfab0e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jul 2023 16:34:44 -0400 Subject: [PATCH 0005/1141] Rename classes to mark old code as deprecated --- ...etSelection.ts => LegacySheetSelection.ts} | 5 ++- .../components/sheet/SheetClipboardHandler.ts | 6 ++-- mathesar_ui/src/components/sheet/index.ts | 4 +-- .../{Selection.ts => SheetSelection.ts} | 36 ++++++++++--------- .../src/stores/table-data/tabularData.ts | 13 ++++--- .../src/systems/data-explorer/QueryRunner.ts | 6 ++-- .../src/systems/table-view/Body.svelte | 2 +- .../src/systems/table-view/StatusPane.svelte | 2 +- .../src/systems/table-view/TableView.svelte | 9 +++-- .../systems/table-view/header/Header.svelte | 2 +- .../src/systems/table-view/row/Row.svelte | 9 +++-- .../table-inspector/TableInspector.svelte | 2 +- .../table-inspector/column/ColumnMode.svelte | 2 +- .../ExtractColumnsModal.svelte | 6 +++- .../table-inspector/record/RecordMode.svelte | 2 +- 15 files changed, 64 insertions(+), 42 deletions(-) rename mathesar_ui/src/components/sheet/{SheetSelection.ts => LegacySheetSelection.ts} (99%) rename mathesar_ui/src/components/sheet/selection/{Selection.ts => SheetSelection.ts} (90%) diff --git a/mathesar_ui/src/components/sheet/SheetSelection.ts b/mathesar_ui/src/components/sheet/LegacySheetSelection.ts similarity index 99% rename from mathesar_ui/src/components/sheet/SheetSelection.ts rename to mathesar_ui/src/components/sheet/LegacySheetSelection.ts index 4c88ef8c50..0aeeeb31c6 100644 --- a/mathesar_ui/src/components/sheet/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/LegacySheetSelection.ts @@ -196,7 +196,10 @@ export function getSelectedRowIndex(selectedCell: string): number { return Number(selectedCell.split(ROW_COLUMN_SEPARATOR)[0]); } -export default class SheetSelection< +/** + * @deprecated + */ +export default class LegacySheetSelection< Row extends SelectionRow, Column extends SelectionColumn, > { diff --git a/mathesar_ui/src/components/sheet/SheetClipboardHandler.ts b/mathesar_ui/src/components/sheet/SheetClipboardHandler.ts index c8e7a8b3d7..2af5549205 100644 --- a/mathesar_ui/src/components/sheet/SheetClipboardHandler.ts +++ b/mathesar_ui/src/components/sheet/SheetClipboardHandler.ts @@ -2,9 +2,9 @@ import * as Papa from 'papaparse'; import { get } from 'svelte/store'; import { ImmutableSet, type MakeToast } from '@mathesar-component-library'; -import SheetSelection, { +import LegacySheetSelection, { isCellSelected, -} from '@mathesar/components/sheet/SheetSelection'; +} from '@mathesar/components/sheet/LegacySheetSelection'; import type { ClipboardHandler } from '@mathesar/stores/clipboard'; import type { ProcessedColumn, @@ -76,7 +76,7 @@ interface SheetClipboardHandlerDeps< Row extends QueryRow | RecordRow, Column extends ProcessedQueryOutputColumn | ProcessedColumn, > { - selection: SheetSelection; + selection: LegacySheetSelection; toast: MakeToast; getRows(): Row[]; getColumnsMap(): ReadableMapLike; diff --git a/mathesar_ui/src/components/sheet/index.ts b/mathesar_ui/src/components/sheet/index.ts index cc67f1275b..0448df86aa 100644 --- a/mathesar_ui/src/components/sheet/index.ts +++ b/mathesar_ui/src/components/sheet/index.ts @@ -5,7 +5,7 @@ export { default as SheetPositionableCell } from './SheetPositionableCell.svelte export { default as SheetCellResizer } from './SheetCellResizer.svelte'; export { default as SheetVirtualRows } from './SheetVirtualRows.svelte'; export { default as SheetRow } from './SheetRow.svelte'; -export { default as SheetSelection } from './SheetSelection'; +export { default as LegacySheetSelection } from './LegacySheetSelection'; export { isColumnSelected, isRowSelected, @@ -14,4 +14,4 @@ export { isCellActive, scrollBasedOnActiveCell, scrollBasedOnSelection, -} from './SheetSelection'; +} from './LegacySheetSelection'; diff --git a/mathesar_ui/src/components/sheet/selection/Selection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts similarity index 90% rename from mathesar_ui/src/components/sheet/selection/Selection.ts rename to mathesar_ui/src/components/sheet/selection/SheetSelection.ts index c98e5570dc..ba8c8b1b8b 100644 --- a/mathesar_ui/src/components/sheet/selection/Selection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -67,7 +67,7 @@ function basisFromPlaceholderCell(activeCellId: string): Basis { }; } -export default class Selection { +export default class SheetSelection { private readonly plane: Plane; private readonly basis: Basis; @@ -93,15 +93,15 @@ export default class Selection { return this.basis.columnIds; } - private withBasis(basis: Basis): Selection { - return new Selection(this.plane, basis); + private withBasis(basis: Basis): SheetSelection { + return new SheetSelection(this.plane, basis); } /** * @returns a new selection with all cells selected. The active cell will be * the cell in the first row and first column. */ - ofAllDataCells(): Selection { + ofAllDataCells(): SheetSelection { if (!this.plane.hasResultRows) { return this.withBasis(basisFromZeroEmptyColumns()); } @@ -112,7 +112,7 @@ export default class Selection { * @returns a new selection with the cell in the first row and first column * selected. */ - ofFirstDataCell(): Selection { + ofFirstDataCell(): SheetSelection { const firstCellId = first(this.plane.allDataCells()); if (firstCellId === undefined) { return this.withBasis(basisFromZeroEmptyColumns()); @@ -129,7 +129,7 @@ export default class Selection { * of data rows, and will never include the placeholder row, even if a user * drags to select it. */ - ofRowRange(rowIdA: string, rowIdB: string): Selection { + ofRowRange(rowIdA: string, rowIdB: string): SheetSelection { return this.withBasis( basisFromDataCells( this.plane.dataCellsInFlexibleRowRange(rowIdA, rowIdB), @@ -141,7 +141,7 @@ export default class Selection { * @returns a new selection of all data cells in all columns between the * provided columnIds, inclusive. */ - ofColumnRange(columnIdA: string, columnIdB: string): Selection { + ofColumnRange(columnIdA: string, columnIdB: string): SheetSelection { const newBasis = this.plane.hasResultRows ? basisFromDataCells( this.plane.dataCellsInColumnRange(columnIdA, columnIdB), @@ -159,7 +159,7 @@ export default class Selection { * selection is made only of data cells, and will never include cells in the * placeholder row, even if a user drags to select a cell in it. */ - ofCellRange(cellIdA: string, cellIdB: string): Selection { + ofCellRange(cellIdA: string, cellIdB: string): SheetSelection { return this.withBasis( basisFromDataCells( this.plane.dataCellsInFlexibleCellRange(cellIdA, cellIdB), @@ -172,14 +172,14 @@ export default class Selection { * Note that we do not support selections of multiple cells in the placeholder * row. */ - atPlaceholderCell(cellId: string): Selection { + atPlaceholderCell(cellId: string): SheetSelection { return this.withBasis(basisFromPlaceholderCell(cellId)); } /** * @returns a new selection formed from one cell within the data rows. */ - atDataCell(cellId: string): Selection { + atDataCell(cellId: string): SheetSelection { return this.withBasis(basisFromDataCells([cellId], cellId)); } @@ -187,7 +187,7 @@ export default class Selection { * @returns a new selection that fits within the provided plane. This is * useful when a column is deleted, reordered, or inserted. */ - forNewPlane(plane: Plane): Selection { + forNewPlane(plane: Plane): SheetSelection { throw new Error('Not implemented'); } @@ -201,7 +201,7 @@ export default class Selection { * by the active cell (also the first cell selected when dragging) and the * provided cell. */ - drawnToCell(cellId: string): Selection { + drawnToCell(cellId: string): SheetSelection { return this.ofCellRange(this.activeCellId ?? cellId, cellId); } @@ -209,14 +209,14 @@ export default class Selection { * @returns a new selection formed by the cells in all the rows between the * active cell and the provided row, inclusive. */ - drawnToRow(rowId: string): Selection { + drawnToRow(rowId: string): SheetSelection { const activeRowId = this.activeCellId ? parseCellId(this.activeCellId).rowId : rowId; return this.ofRowRange(activeRowId, rowId); } - drawnToColumn(columnId: string): Selection { + drawnToColumn(columnId: string): SheetSelection { // TODO improve handling for empty columns const activeColumnId = this.activeCellId @@ -230,7 +230,7 @@ export default class Selection { * spreadsheets. If the active cell can be moved in the provided direction, * then a new selection is created with only that one cell selected. */ - collapsedAndMoved(direction: Direction): Selection { + collapsedAndMoved(direction: Direction): SheetSelection { if (this.basis.type === 'emptyColumns') { const offset = getColumnOffset(direction); const newActiveColumnId = this.plane.columnIds.collapsedOffset( @@ -272,7 +272,9 @@ export default class Selection { * * This is to handle the `Tab` and `Shift+Tab` keys. */ - withActiveCellAdvanced(direction: 'forward' | 'back' = 'forward'): Selection { + withActiveCellAdvanced( + direction: 'forward' | 'back' = 'forward', + ): SheetSelection { throw new Error('Not implemented'); } @@ -291,7 +293,7 @@ export default class Selection { * the active cell. We chose to mimic Google Sheets behavior here because it * is simpler. */ - resized(direction: Direction): Selection { + resized(direction: Direction): SheetSelection { throw new Error('Not implemented'); } } diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index a29e0c9836..543e977ea3 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -11,7 +11,7 @@ import type { TableEntry } from '@mathesar/api/types/tables'; import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; import { States } from '@mathesar/api/utils/requestUtils'; import type { Column } from '@mathesar/api/types/tables/columns'; -import { SheetSelection } from '@mathesar/components/sheet'; +import { LegacySheetSelection } from '@mathesar/components/sheet'; import { getColumnOrder } from '@mathesar/utils/tables'; import { Meta } from './meta'; import { ColumnsDataStore } from './columns'; @@ -44,7 +44,10 @@ export interface TabularDataProps { >[0]['hasEnhancedPrimaryKeyCell']; } -export type TabularDataSelection = SheetSelection; +export type TabularDataSelection = LegacySheetSelection< + RecordRow, + ProcessedColumn +>; export class TabularData { id: DBObjectEntry['id']; @@ -63,7 +66,7 @@ export class TabularData { isLoading: Readable; - selection: TabularDataSelection; + legacySelection: TabularDataSelection; table: TableEntry; @@ -109,7 +112,7 @@ export class TabularData { this.table = props.table; - this.selection = new SheetSelection({ + this.legacySelection = new LegacySheetSelection({ getColumns: () => [...get(this.processedColumns).values()], getColumnOrder: () => getColumnOrder([...get(this.processedColumns).values()], this.table), @@ -218,7 +221,7 @@ export class TabularData { this.recordsData.destroy(); this.constraintsDataStore.destroy(); this.columnsDataStore.destroy(); - this.selection.destroy(); + this.legacySelection.destroy(); } } diff --git a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts index bf3985c226..2b7ee402e2 100644 --- a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts +++ b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts @@ -14,7 +14,7 @@ import type { QueryColumnMetaData, } from '@mathesar/api/types/queries'; import { runQuery } from '@mathesar/stores/queries'; -import { SheetSelection } from '@mathesar/components/sheet'; +import { LegacySheetSelection } from '@mathesar/components/sheet'; import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; import type QueryModel from './QueryModel'; import QueryInspector from './QueryInspector'; @@ -42,7 +42,7 @@ export interface QueryRowsData { rows: QueryRow[]; } -export type QuerySheetSelection = SheetSelection< +export type QuerySheetSelection = LegacySheetSelection< QueryRow, ProcessedQueryOutputColumn >; @@ -81,7 +81,7 @@ export default class QueryRunner< this.query = writable(query); this.speculateProcessedColumns(); void this.run(); - this.selection = new SheetSelection({ + this.selection = new LegacySheetSelection({ getColumns: () => [...get(this.processedColumns).values()], getColumnOrder: () => [...get(this.processedColumns).values()].map((column) => column.id), diff --git a/mathesar_ui/src/systems/table-view/Body.svelte b/mathesar_ui/src/systems/table-view/Body.svelte index bed4c366a6..f917c5dc71 100644 --- a/mathesar_ui/src/systems/table-view/Body.svelte +++ b/mathesar_ui/src/systems/table-view/Body.svelte @@ -31,7 +31,7 @@ export let usesVirtualList = false; - $: ({ id, display, columnsDataStore, selection } = $tabularData); + $: ({ id, display, columnsDataStore } = $tabularData); $: ({ displayableRecords } = display); $: ({ pkColumn } = columnsDataStore); diff --git a/mathesar_ui/src/systems/table-view/StatusPane.svelte b/mathesar_ui/src/systems/table-view/StatusPane.svelte index 36a0672e1c..459e02c56c 100644 --- a/mathesar_ui/src/systems/table-view/StatusPane.svelte +++ b/mathesar_ui/src/systems/table-view/StatusPane.svelte @@ -32,7 +32,7 @@ isLoading, columnsDataStore, constraintsDataStore, - selection, + legacySelection: selection, } = $tabularData); $: ({ pagination } = meta); $: ({ size: pageSize, leftBound, rightBound } = $pagination); diff --git a/mathesar_ui/src/systems/table-view/TableView.svelte b/mathesar_ui/src/systems/table-view/TableView.svelte index 3cb518ae72..8e7e129cff 100644 --- a/mathesar_ui/src/systems/table-view/TableView.svelte +++ b/mathesar_ui/src/systems/table-view/TableView.svelte @@ -40,8 +40,13 @@ $: usesVirtualList = context === 'page'; $: allowsDdlOperations = context === 'page' && canExecuteDDL; $: sheetHasBorder = context === 'widget'; - $: ({ processedColumns, display, isLoading, selection, recordsData } = - $tabularData); + $: ({ + processedColumns, + display, + isLoading, + legacySelection: selection, + recordsData, + } = $tabularData); $: clipboardHandler = new SheetClipboardHandler({ selection, toast, diff --git a/mathesar_ui/src/systems/table-view/header/Header.svelte b/mathesar_ui/src/systems/table-view/header/Header.svelte index e70a5574fd..3b71564078 100644 --- a/mathesar_ui/src/systems/table-view/header/Header.svelte +++ b/mathesar_ui/src/systems/table-view/header/Header.svelte @@ -28,7 +28,7 @@ $: columnOrder = columnOrder ?? []; $: columnOrderString = columnOrder.map(String); - $: ({ selection, processedColumns } = $tabularData); + $: ({ legacySelection: selection, processedColumns } = $tabularData); $: ({ selectedCells, columnsSelectedWhenTheTableIsEmpty, diff --git a/mathesar_ui/src/systems/table-view/row/Row.svelte b/mathesar_ui/src/systems/table-view/row/Row.svelte index eb717f1ba6..967c17b677 100644 --- a/mathesar_ui/src/systems/table-view/row/Row.svelte +++ b/mathesar_ui/src/systems/table-view/row/Row.svelte @@ -29,8 +29,13 @@ const tabularData = getTabularDataStoreFromContext(); - $: ({ recordsData, columnsDataStore, meta, processedColumns, selection } = - $tabularData); + $: ({ + recordsData, + columnsDataStore, + meta, + processedColumns, + legacySelection: selection, + } = $tabularData); $: ({ rowStatus, rowCreationStatus, diff --git a/mathesar_ui/src/systems/table-view/table-inspector/TableInspector.svelte b/mathesar_ui/src/systems/table-view/table-inspector/TableInspector.svelte index 967d289a26..d628a0d124 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/TableInspector.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/TableInspector.svelte @@ -29,7 +29,7 @@ let activeTab: TabItem; const tabularData = getTabularDataStoreFromContext(); - $: ({ selection } = $tabularData); + $: ({ legacySelection: selection } = $tabularData); $: ({ selectedCells } = selection); $: { diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte index cf92780c1b..fe1e01f610 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte @@ -21,7 +21,7 @@ $: database = $currentDatabase; $: schema = $currentSchema; - $: ({ processedColumns, selection } = $tabularData); + $: ({ processedColumns, legacySelection: selection } = $tabularData); $: ({ selectedCells, columnsSelectedWhenTheTableIsEmpty } = selection); $: selectedColumns = (() => { const ids = selection.getSelectedUniqueColumnsId( diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte index cb1d622145..ed5d468f15 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte @@ -48,7 +48,11 @@ export let controller: ExtractColumnsModalController; - $: ({ processedColumns, constraintsDataStore, selection } = $tabularData); + $: ({ + processedColumns, + constraintsDataStore, + legacySelection: selection, + } = $tabularData); $: ({ constraints } = $constraintsDataStore); $: availableProcessedColumns = [...$processedColumns.values()]; $: ({ targetType, columns, isOpen } = controller); diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte index f85100b121..878701fec1 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte @@ -14,7 +14,7 @@ $: database = $currentDatabase; $: schema = $currentSchema; - $: ({ selection, recordsData } = $tabularData); + $: ({ legacySelection: selection, recordsData } = $tabularData); $: ({ selectedCells } = selection); $: selectedRowIndices = $selectedCells .valuesArray() From 8157a8bc27cac20951ecca92edd795444a06911c Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 19 Jul 2023 13:53:22 -0400 Subject: [PATCH 0006/1141] Begin integrating SheetSelection into TabularData --- .../src/components/sheet/selection/Plane.ts | 8 +-- .../src/components/sheet/selection/Series.ts | 5 +- .../sheet/selection/SheetSelection.ts | 37 +++++++++----- mathesar_ui/src/stores/table-data/records.ts | 12 +++++ .../src/stores/table-data/tabularData.ts | 49 ++++++++++++++++--- 5 files changed, 86 insertions(+), 25 deletions(-) diff --git a/mathesar_ui/src/components/sheet/selection/Plane.ts b/mathesar_ui/src/components/sheet/selection/Plane.ts index 20ed0fccc5..7a3185852e 100644 --- a/mathesar_ui/src/components/sheet/selection/Plane.ts +++ b/mathesar_ui/src/components/sheet/selection/Plane.ts @@ -3,7 +3,7 @@ import { map } from 'iter-tools'; import { cartesianProduct } from '@mathesar/utils/iterUtils'; import { makeCellId, parseCellId } from '../cellIds'; import { Direction, getColumnOffset, getRowOffset } from './Direction'; -import type Series from './Series'; +import Series from './Series'; function makeCells( rowIds: Iterable, @@ -52,9 +52,9 @@ export default class Plane { readonly placeholderRowId: string | undefined; constructor( - rowIds: Series, - columnIds: Series, - placeholderRowId: string | undefined, + rowIds: Series = new Series(), + columnIds: Series = new Series(), + placeholderRowId?: string, ) { this.rowIds = rowIds; this.columnIds = columnIds; diff --git a/mathesar_ui/src/components/sheet/selection/Series.ts b/mathesar_ui/src/components/sheet/selection/Series.ts index 7b8cd45dcc..7034fd06b7 100644 --- a/mathesar_ui/src/components/sheet/selection/Series.ts +++ b/mathesar_ui/src/components/sheet/selection/Series.ts @@ -1,6 +1,7 @@ -import { ImmutableMap } from '@mathesar/component-library'; import { filter, findBest, firstHighest, firstLowest } from 'iter-tools'; +import { ImmutableMap } from '@mathesar/component-library'; + export default class Series { private readonly values: Id[]; @@ -10,7 +11,7 @@ export default class Series { /** * @throws Error if duplicate values are provided */ - constructor(values: Id[]) { + constructor(values: Id[] = []) { this.values = values; this.indexLookup = new ImmutableMap( values.map((value, index) => [value, index]), diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index ba8c8b1b8b..a16948d9e0 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -4,22 +4,26 @@ import { ImmutableSet } from '@mathesar/component-library'; import { assertExhaustive } from '@mathesar/utils/typeUtils'; import { parseCellId } from '../cellIds'; import { Direction, getColumnOffset } from './Direction'; -import type Plane from './Plane'; +import Plane from './Plane'; /** - * - `'dataCells'` means that the selection contains data cells. This is by - * far the most common type of selection basis. + * - `'dataCells'` means that the selection contains data cells. This is by far + * the most common type of selection basis. * - * - `'emptyColumns'` is used when the sheet has no rows. In this case we - * still want to allow the user to select columns, so we use this basis. + * - `'emptyColumns'` is used when the sheet has no rows. In this case we still + * want to allow the user to select columns, so we use this basis. * * - `'placeholderCell'` is used when the user is selecting a cell in the - * placeholder row. This is a special case because we don't want to allow - * the user to select multiple cells in the placeholder row, and we also - * don't want to allow selections that include cells in data rows _and_ the + * placeholder row. This is a special case because we don't want to allow the + * user to select multiple cells in the placeholder row, and we also don't + * want to allow selections that include cells in data rows _and_ the * placeholder row. + * + * - `'empty'` is used when no cells are selected. We try to avoid this state, + * but we also allow for it because it makes it easier to construct selection + * instances if we don't already have the full plane data. */ -type BasisType = 'dataCells' | 'emptyColumns' | 'placeholderCell'; +type BasisType = 'dataCells' | 'emptyColumns' | 'placeholderCell' | 'empty'; interface Basis { readonly type: BasisType; @@ -67,12 +71,22 @@ function basisFromPlaceholderCell(activeCellId: string): Basis { }; } +function emptyBasis(): Basis { + return { + type: 'empty', + activeCellId: undefined, + cellIds: new ImmutableSet(), + columnIds: new ImmutableSet(), + rowIds: new ImmutableSet(), + }; +} + export default class SheetSelection { private readonly plane: Plane; private readonly basis: Basis; - constructor(plane: Plane, basis: Basis) { + constructor(plane: Plane = new Plane(), basis: Basis = emptyBasis()) { this.plane = plane; this.basis = basis; } @@ -187,7 +201,8 @@ export default class SheetSelection { * @returns a new selection that fits within the provided plane. This is * useful when a column is deleted, reordered, or inserted. */ - forNewPlane(plane: Plane): SheetSelection { + forNewPlane(newPlane: Plane): SheetSelection { + // TODO_NEXT throw new Error('Not implemented'); } diff --git a/mathesar_ui/src/stores/table-data/records.ts b/mathesar_ui/src/stores/table-data/records.ts index 4719c1ac81..870770ff9e 100644 --- a/mathesar_ui/src/stores/table-data/records.ts +++ b/mathesar_ui/src/stores/table-data/records.ts @@ -29,6 +29,7 @@ import { patchAPI, postAPI, } from '@mathesar/api/utils/requestUtils'; +import Series from '@mathesar/components/sheet/selection/Series'; import type Pagination from '@mathesar/utils/Pagination'; import { getErrorMessage } from '@mathesar/utils/errors'; import { pluralize } from '@mathesar/utils/languageUtils'; @@ -293,6 +294,8 @@ export class RecordsData { error: Writable; + selectableRowIds: Readable>; + private promise: CancellablePromise | undefined; // @ts-ignore: https://github.com/centerofci/mathesar/issues/1055 @@ -339,6 +342,15 @@ export class RecordsData { this.url = `/api/db/v0/tables/${this.parentId}/records/`; void this.fetch(); + this.selectableRowIds = derived( + [this.savedRecords, this.newRecords], + ([savedRecords, newRecords]) => + new Series([ + ...savedRecords.map((r) => r.identifier), + ...newRecords.map((r) => r.identifier), + ]), + ); + // TODO: Create base class to abstract subscriptions and unsubscriptions this.requestParamsUnsubscriber = this.meta.recordsRequestParamsData.subscribe(() => { diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index 543e977ea3..086affd1e3 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -1,30 +1,34 @@ import { getContext, setContext } from 'svelte'; import { derived, - writable, get, + writable, type Readable, type Writable, } from 'svelte/store'; + import type { DBObjectEntry } from '@mathesar/AppTypes'; import type { TableEntry } from '@mathesar/api/types/tables'; -import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; -import { States } from '@mathesar/api/utils/requestUtils'; import type { Column } from '@mathesar/api/types/tables/columns'; +import { States } from '@mathesar/api/utils/requestUtils'; import { LegacySheetSelection } from '@mathesar/components/sheet'; -import { getColumnOrder } from '@mathesar/utils/tables'; -import { Meta } from './meta'; +import Plane from '@mathesar/components/sheet/selection/Plane'; +import Series from '@mathesar/components/sheet/selection/Series'; +import SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; +import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; +import { getColumnOrder, orderProcessedColumns } from '@mathesar/utils/tables'; import { ColumnsDataStore } from './columns'; -import type { RecordRow, TableRecordsData } from './records'; -import { RecordsData } from './records'; -import { Display } from './display'; import type { ConstraintsData } from './constraints'; import { ConstraintsDataStore } from './constraints'; +import { Display } from './display'; +import { Meta } from './meta'; import type { ProcessedColumn, ProcessedColumnsStore, } from './processedColumns'; import { processColumn } from './processedColumns'; +import type { RecordRow, TableRecordsData } from './records'; +import { RecordsData } from './records'; export interface TabularDataProps { id: DBObjectEntry['id']; @@ -56,8 +60,11 @@ export class TabularData { columnsDataStore: ColumnsDataStore; + /** TODO eliminate `processedColumns` in favor of `orderedProcessedColumns` */ processedColumns: ProcessedColumnsStore; + orderedProcessedColumns: ProcessedColumnsStore; + constraintsDataStore: ConstraintsDataStore; recordsData: RecordsData; @@ -68,8 +75,12 @@ export class TabularData { legacySelection: TabularDataSelection; + selection: Writable; + table: TableEntry; + cleanupFunctions: (() => void)[] = []; + constructor(props: TabularDataProps) { const contextualFilters = props.contextualFilters ?? new Map(); @@ -112,6 +123,27 @@ export class TabularData { this.table = props.table; + this.orderedProcessedColumns = derived(this.processedColumns, (p) => + orderProcessedColumns(p, this.table), + ); + + const plane = derived( + [this.recordsData.selectableRowIds, this.orderedProcessedColumns], + ([selectableRowIds, orderedProcessedColumns]) => { + const columnIds = new Series( + [...orderedProcessedColumns.values()].map((c) => String(c.id)), + ); + return new Plane(selectableRowIds, columnIds); + }, + ); + + // TODO add id of placeholder row to selection + this.selection = writable(new SheetSelection()); + + this.cleanupFunctions.push( + plane.subscribe((p) => this.selection.update((s) => s.forNewPlane(p))), + ); + this.legacySelection = new LegacySheetSelection({ getColumns: () => [...get(this.processedColumns).values()], getColumnOrder: () => @@ -222,6 +254,7 @@ export class TabularData { this.constraintsDataStore.destroy(); this.columnsDataStore.destroy(); this.legacySelection.destroy(); + this.cleanupFunctions.forEach((f) => f()); } } From 640eb9b08ba1636005cad928652c13a41de4731e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 25 Aug 2023 09:42:40 -0400 Subject: [PATCH 0007/1141] Some readability improvements --- .../src/components/sheet/selection/Plane.ts | 12 +++++++ .../src/components/sheet/selection/Series.ts | 36 ++++++++++--------- .../sheet/selection/SheetSelection.ts | 17 +++++++++ 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/mathesar_ui/src/components/sheet/selection/Plane.ts b/mathesar_ui/src/components/sheet/selection/Plane.ts index 7a3185852e..2fd4b9df48 100644 --- a/mathesar_ui/src/components/sheet/selection/Plane.ts +++ b/mathesar_ui/src/components/sheet/selection/Plane.ts @@ -44,6 +44,18 @@ function adjacentPlaceholderCell(cellId: string): AdjacentCell { return { type: 'placeholderCell', cellId }; } +/** + * A Plane is like a coordinate system for a sheet. We can query it to find the + * ids of cells within certain bounding rectangles. + * + * The Plane can also have a "placeholder row", which is a row at the bottom of + * the sheet that provides a visual cue to the user that they can add more rows. + * It never contains any data, but we allow the user to move the active cell + * into the placeholder row in order to easily add more rows. + * + * The term "Flexible" is used in methods to indicate that it will gracefully + * handle ids of cells within the placeholder row. + */ export default class Plane { readonly rowIds: Series; diff --git a/mathesar_ui/src/components/sheet/selection/Series.ts b/mathesar_ui/src/components/sheet/selection/Series.ts index 7034fd06b7..cbfd31d16d 100644 --- a/mathesar_ui/src/components/sheet/selection/Series.ts +++ b/mathesar_ui/src/components/sheet/selection/Series.ts @@ -2,16 +2,20 @@ import { filter, findBest, firstHighest, firstLowest } from 'iter-tools'; import { ImmutableMap } from '@mathesar/component-library'; -export default class Series { - private readonly values: Id[]; +/** + * A Series is an immutable ordered collection of values with methods that + * provide efficient access to _ranges_ of those values. + */ +export default class Series { + private readonly values: Value[]; /** Maps the id value to its index */ - private readonly indexLookup: ImmutableMap; + private readonly indexLookup: ImmutableMap; /** * @throws Error if duplicate values are provided */ - constructor(values: Id[] = []) { + constructor(values: Value[] = []) { this.values = values; this.indexLookup = new ImmutableMap( values.map((value, index) => [value, index]), @@ -21,7 +25,7 @@ export default class Series { } } - private getIndex(value: Id): number | undefined { + private getIndex(value: Value): number | undefined { return this.indexLookup.get(value); } @@ -36,7 +40,7 @@ export default class Series { * * @throws an Error if either value is not present in the sequence */ - range(a: Id, b: Id): Iterable { + range(a: Value, b: Value): Iterable { const aIndex = this.getIndex(a); const bIndex = this.getIndex(b); @@ -50,15 +54,15 @@ export default class Series { return this.values.slice(startIndex, endIndex + 1); } - get first(): Id | undefined { + get first(): Value | undefined { return this.values[0]; } - get last(): Id | undefined { + get last(): Value | undefined { return this.values[this.values.length - 1]; } - has(value: Id): boolean { + has(value: Value): boolean { return this.getIndex(value) !== undefined; } @@ -72,18 +76,18 @@ export default class Series { * https://github.com/iter-tools/iter-tools/blob/d7.5/API.md#compare-values-and-return-true-or-false */ best( - values: Iterable, + values: Iterable, comparator: (best: number, v: number) => boolean, - ): Id | undefined { + ): Value | undefined { const validValues = filter((v) => this.has(v), values); return findBest(comparator, (v) => this.getIndex(v) ?? 0, validValues); } - min(values: Iterable): Id | undefined { + min(values: Iterable): Value | undefined { return this.best(values, firstLowest); } - max(values: Iterable): Id | undefined { + max(values: Iterable): Value | undefined { return this.best(values, firstHighest); } @@ -94,7 +98,7 @@ export default class Series { * given value. If no such value is present, then `undefined` will be * returned. */ - offset(value: Id, offset: number): Id | undefined { + offset(value: Value, offset: number): Value | undefined { if (offset === 0) { return this.has(value) ? value : undefined; } @@ -112,7 +116,7 @@ export default class Series { * value in the block. If no such value is present, then `undefined` will be * returned. If offset is zero, then `undefined` will be returned. */ - collapsedOffset(values: Iterable, offset: number): Id | undefined { + collapsedOffset(values: Iterable, offset: number): Value | undefined { if (offset === 0) { return undefined; } @@ -123,7 +127,7 @@ export default class Series { return this.offset(outerValue, offset); } - [Symbol.iterator](): Iterator { + [Symbol.iterator](): Iterator { return this.values[Symbol.iterator](); } } diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index a16948d9e0..d9b7bcd84f 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -25,6 +25,14 @@ import Plane from './Plane'; */ type BasisType = 'dataCells' | 'emptyColumns' | 'placeholderCell' | 'empty'; +/** + * This type stores data about "which cells are selected", with some redundancy + * for efficient and consistent lookup across different kinds of selections. + * + * Due to the redundant nature of some properties on this type, you should be + * sure to only instantiate Basis using the utility functions below. This will + * ensure that the data is always valid. + */ interface Basis { readonly type: BasisType; readonly activeCellId: string | undefined; @@ -81,6 +89,15 @@ function emptyBasis(): Basis { }; } +/** + * This is an immutable data structure which fully represents the state of a + * selection selection of cells along with the Plane in which they were + * selected. + * + * We store the Plane here so to make it possible to provide methods on + * `SheetSelection` instances which return new mutations of the selection that + * are still valid within the Plane. + */ export default class SheetSelection { private readonly plane: Plane; From de875e41130da4d667e0615a04a4d18a852f20f4 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 25 Aug 2023 10:48:46 -0400 Subject: [PATCH 0008/1141] Add `intersect` method to ImmutableSet --- .../component-library/common/utils/ImmutableSet.ts | 9 +++++++++ .../common/utils/__tests__/ImmutableSet.test.ts | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/mathesar_ui/src/component-library/common/utils/ImmutableSet.ts b/mathesar_ui/src/component-library/common/utils/ImmutableSet.ts index 609c156ecd..d442984ad0 100644 --- a/mathesar_ui/src/component-library/common/utils/ImmutableSet.ts +++ b/mathesar_ui/src/component-library/common/utils/ImmutableSet.ts @@ -32,6 +32,15 @@ export default class ImmutableSet { return this.getNewInstance(set); } + /** + * @returns a new ImmutableSet that contains only the items that are present + * in both this set and the other set. The order of the items in the returned + * set is taken from this set (not the supplied set). + */ + intersect(other: { has: (v: T) => boolean }): this { + return this.getNewInstance([...this].filter((v) => other.has(v))); + } + without(itemOrItems: T | T[]): this { const items = Array.isArray(itemOrItems) ? itemOrItems : [itemOrItems]; const set = new Set(this.set); diff --git a/mathesar_ui/src/component-library/common/utils/__tests__/ImmutableSet.test.ts b/mathesar_ui/src/component-library/common/utils/__tests__/ImmutableSet.test.ts index 24938eb957..3d20add0d6 100644 --- a/mathesar_ui/src/component-library/common/utils/__tests__/ImmutableSet.test.ts +++ b/mathesar_ui/src/component-library/common/utils/__tests__/ImmutableSet.test.ts @@ -20,3 +20,15 @@ test('union', () => { expect(a.union(empty).valuesArray()).toEqual([2, 7, 13, 19, 5]); expect(empty.union(a).valuesArray()).toEqual([2, 7, 13, 19, 5]); }); + +test('intersect', () => { + const a = new ImmutableSet([2, 7, 13, 19, 5]); + const b = new ImmutableSet([23, 13, 3, 2]); + const empty = new ImmutableSet(); + expect(a.intersect(a).valuesArray()).toEqual([2, 7, 13, 19, 5]); + expect(a.intersect(b).valuesArray()).toEqual([2, 13]); + expect(b.intersect(a).valuesArray()).toEqual([13, 2]); + expect(a.intersect(empty).valuesArray()).toEqual([]); + expect(empty.intersect(a).valuesArray()).toEqual([]); + expect(empty.intersect(empty).valuesArray()).toEqual([]); +}); From c64e12b87a9f6d81b19770e0c0290fc235309b0f Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 25 Aug 2023 21:57:14 -0400 Subject: [PATCH 0009/1141] Refactor Series tests to use cases --- .../sheet/selection/__tests__/Series.test.ts | 116 +++++++++++------- 1 file changed, 75 insertions(+), 41 deletions(-) diff --git a/mathesar_ui/src/components/sheet/selection/__tests__/Series.test.ts b/mathesar_ui/src/components/sheet/selection/__tests__/Series.test.ts index 8109932d9f..0d485b64a7 100644 --- a/mathesar_ui/src/components/sheet/selection/__tests__/Series.test.ts +++ b/mathesar_ui/src/components/sheet/selection/__tests__/Series.test.ts @@ -1,44 +1,78 @@ import Series from '../Series'; -test('Series', () => { - const s = new Series(['h', 'i', 'j', 'k', 'l']); - expect(s.length).toBe(5); - expect(s.first).toBe('h'); - expect(s.last).toBe('l'); - - expect([...s]).toEqual(['h', 'i', 'j', 'k', 'l']); - - expect(s.min(['i', 'j', 'k'])).toBe('i'); - expect(s.min(['k', 'j', 'i'])).toBe('i'); - expect(s.min(['i', 'NOPE'])).toBe('i'); - expect(s.min(['NOPE'])).toBe(undefined); - expect(s.min([])).toBe(undefined); - - expect(s.max(['i', 'j', 'k'])).toBe('k'); - expect(s.max(['k', 'j', 'i'])).toBe('k'); - expect(s.max(['i', 'NOPE'])).toBe('i'); - expect(s.max(['NOPE'])).toBe(undefined); - expect(s.max([])).toBe(undefined); - - expect([...s.range('i', 'k')]).toEqual(['i', 'j', 'k']); - expect([...s.range('k', 'i')]).toEqual(['i', 'j', 'k']); - expect([...s.range('i', 'i')]).toEqual(['i']); - expect(() => s.range('i', 'NOPE')).toThrow(); - - expect(s.offset('i', 0)).toBe('i'); - expect(s.offset('i', 1)).toBe('j'); - expect(s.offset('i', 2)).toBe('k'); - expect(s.offset('i', -1)).toBe('h'); - expect(s.offset('i', -2)).toBe(undefined); - expect(s.offset('NOPE', 0)).toBe(undefined); - expect(s.offset('NOPE', 1)).toBe(undefined); - - expect(s.collapsedOffset(['i', 'k'], 0)).toBe(undefined); - expect(s.collapsedOffset(['i', 'k'], 1)).toBe('l'); - expect(s.collapsedOffset(['i', 'k'], 2)).toBe(undefined); - expect(s.collapsedOffset(['i', 'k'], -1)).toBe('h'); - expect(s.collapsedOffset(['i', 'k'], -2)).toBe(undefined); - expect(s.collapsedOffset(['i', 'NOPE'], 0)).toBe(undefined); - expect(s.collapsedOffset(['i', 'NOPE'], 1)).toBe('j'); - expect(s.collapsedOffset([], 0)).toBe(undefined); +describe('Series', () => { + const h = 'h'; + const i = 'i'; + const j = 'j'; + const k = 'k'; + const l = 'l'; + const all = [h, i, j, k, l]; + const s = new Series(all); + + test('basics', () => { + expect(s.length).toBe(5); + expect(s.first).toBe(h); + expect(s.last).toBe(l); + expect([...s]).toEqual(all); + }); + + test.each([ + [[i, j, k], i], + [[k, j, i], i], + [[i, 'NOPE'], i], + [['NOPE'], undefined], + [[], undefined], + ])('min %#', (input, expected) => { + expect(s.min(input)).toBe(expected); + }); + + test.each([ + [[i, j, k], k], + [[k, j, i], k], + [[i, 'NOPE'], i], + [['NOPE'], undefined], + [[], undefined], + ])('max %#', (input, expected) => { + expect(s.max(input)).toBe(expected); + }); + + test.each([ + [i, k, [i, j, k]], + [k, i, [i, j, k]], + [i, i, [i]], + ])('range %#', (a, b, expected) => { + expect([...s.range(a, b)]).toEqual(expected); + }); + + test.each([ + [i, 'NOPE'], + ['NOPE', i], + ])('range failures %#', (a, b) => { + expect(() => s.range(a, b)).toThrow(); + }); + + test.each([ + [i, 0, i], + [i, 1, j], + [i, 2, k], + [i, -1, h], + [i, -2, undefined], + ['NOPE', 0, undefined], + ['NOPE', 1, undefined], + ])('offset %#', (value, offset, expected) => { + expect(s.offset(value, offset)).toBe(expected); + }); + + test.each([ + [[i, k], 0, undefined], + [[i, k], 1, l], + [[i, k], 2, undefined], + [[i, k], -1, h], + [[i, k], -2, undefined], + [[i, 'NOPE'], 0, undefined], + [[i, 'NOPE'], 1, j], + [[], 0, undefined], + ])('collapsedOffset %#', (values, offset, expected) => { + expect(s.collapsedOffset(values, offset)).toBe(expected); + }); }); From be749bad54be4996f0a4055d4db818b7f5fed7f1 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 25 Aug 2023 21:57:48 -0400 Subject: [PATCH 0010/1141] Tiny readability improvement --- mathesar_ui/src/components/sheet/selection/Series.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mathesar_ui/src/components/sheet/selection/Series.ts b/mathesar_ui/src/components/sheet/selection/Series.ts index cbfd31d16d..b0e897aab3 100644 --- a/mathesar_ui/src/components/sheet/selection/Series.ts +++ b/mathesar_ui/src/components/sheet/selection/Series.ts @@ -1,10 +1,9 @@ import { filter, findBest, firstHighest, firstLowest } from 'iter-tools'; - -import { ImmutableMap } from '@mathesar/component-library'; +import { ImmutableMap } from '@mathesar-component-library'; /** - * A Series is an immutable ordered collection of values with methods that - * provide efficient access to _ranges_ of those values. + * A Series is an immutable ordered collection of unique values with methods + * that provide efficient access to _ranges_ of those values. */ export default class Series { private readonly values: Value[]; From 0476730ddbfaca8bd3548b926ce8c17d0ce1fbba Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 25 Aug 2023 21:58:20 -0400 Subject: [PATCH 0011/1141] Implement SheetSelection.forNewPlane --- .../src/components/sheet/selection/Plane.ts | 30 ++++++++- .../sheet/selection/SheetSelection.ts | 62 ++++++++++++++++++- 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/mathesar_ui/src/components/sheet/selection/Plane.ts b/mathesar_ui/src/components/sheet/selection/Plane.ts index 2fd4b9df48..3f0d87780f 100644 --- a/mathesar_ui/src/components/sheet/selection/Plane.ts +++ b/mathesar_ui/src/components/sheet/selection/Plane.ts @@ -145,16 +145,40 @@ export default class Plane { ): Iterable { const cellA = parseCellId(cellIdA); const cellB = parseCellId(cellIdB); - const rowIdA = this.normalizeFlexibleRowId(cellA.rowId); + return this.dataCellsInFlexibleRowColumnRange( + cellA.rowId, + cellB.rowId, + cellA.columnId, + cellB.columnId, + ); + } + + /** + * @returns an iterable of all the data cells in the plane that are within the + * rectangle bounded by the given rows and columns. This does not include + * header cells. + * + * If either of the provided rowIds are placeholder cells, then cells in the + * last row and last column will be used in their place. This ensures that the + * selection is made only of data cells, and will never include the + * placeholder cell, even if a user drags to select it. + */ + dataCellsInFlexibleRowColumnRange( + flexibleRowIdA: string, + flexibleRowIdB: string, + columnIdA: string, + columnIdB: string, + ): Iterable { + const rowIdA = this.normalizeFlexibleRowId(flexibleRowIdA); if (rowIdA === undefined) { return []; } - const rowIdB = this.normalizeFlexibleRowId(cellB.rowId); + const rowIdB = this.normalizeFlexibleRowId(flexibleRowIdB); if (rowIdB === undefined) { return []; } const rowIds = this.rowIds.range(rowIdA, rowIdB); - const columnIds = this.columnIds.range(cellA.columnId, cellB.columnId); + const columnIds = this.columnIds.range(columnIdA, columnIdB); return makeCells(rowIds, columnIds); } diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index d9b7bcd84f..9be7602e84 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -219,8 +219,66 @@ export default class SheetSelection { * useful when a column is deleted, reordered, or inserted. */ forNewPlane(newPlane: Plane): SheetSelection { - // TODO_NEXT - throw new Error('Not implemented'); + if (this.basis.type === 'dataCells') { + if (!newPlane.hasResultRows) { + return new SheetSelection(newPlane, basisFromZeroEmptyColumns()); + } + const minColumnId = newPlane.columnIds.min(this.basis.columnIds); + const maxColumnId = newPlane.columnIds.max(this.basis.columnIds); + const minRowId = newPlane.rowIds.min(this.basis.rowIds); + const maxRowId = newPlane.rowIds.max(this.basis.rowIds); + if ( + minColumnId === undefined || + maxColumnId === undefined || + minRowId === undefined || + maxRowId === undefined + ) { + return new SheetSelection(newPlane); + } + const cellIds = newPlane.dataCellsInFlexibleRowColumnRange( + minRowId, + maxRowId, + minColumnId, + maxColumnId, + ); + return new SheetSelection(newPlane, basisFromDataCells(cellIds)); + } + + if (this.basis.type === 'emptyColumns') { + if (newPlane.hasResultRows) { + return new SheetSelection(newPlane); + } + const minColumnId = newPlane.columnIds.min(this.basis.columnIds); + const maxColumnId = newPlane.columnIds.max(this.basis.columnIds); + if (minColumnId === undefined || maxColumnId === undefined) { + return new SheetSelection(newPlane, basisFromZeroEmptyColumns()); + } + const columnIds = newPlane.columnIds.range(minColumnId, maxColumnId); + return new SheetSelection(newPlane, basisFromEmptyColumns(columnIds)); + } + + if (this.basis.type === 'placeholderCell') { + const columnId = first(this.basis.columnIds); + if (columnId === undefined) { + return new SheetSelection(newPlane); + } + const newPlaneHasSelectedCell = + newPlane.columnIds.has(columnId) && + newPlane.placeholderRowId === this.plane.placeholderRowId; + if (newPlaneHasSelectedCell) { + // If we can retain the selected placeholder cell, then do so. + return new SheetSelection(newPlane, basisFromPlaceholderCell(columnId)); + } + // Otherwise, return an empty selection + return new SheetSelection(newPlane); + } + + if (this.basis.type === 'empty') { + // If the selection is empty, we keep it empty. + return new SheetSelection(newPlane); + } + + return assertExhaustive(this.basis.type); } /** From 09cd2198ede775e1d79dad7f05057675e4df9f1e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Sun, 27 Aug 2023 21:15:45 -0400 Subject: [PATCH 0012/1141] Begin incorporating new Selection into table sheet --- .../sheet/selection/SheetSelection.ts | 42 ++++++----- .../src/systems/table-view/row/Row.svelte | 41 +++++------ .../src/systems/table-view/row/RowCell.svelte | 72 ++++++++----------- 3 files changed, 68 insertions(+), 87 deletions(-) diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index 9be7602e84..08f5b67a61 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -55,6 +55,10 @@ function basisFromDataCells( }; } +function basisFromOneDataCell(cellId: string): Basis { + return basisFromDataCells([cellId], cellId); +} + function basisFromEmptyColumns(columnIds: Iterable): Basis { return { type: 'emptyColumns', @@ -106,6 +110,8 @@ export default class SheetSelection { constructor(plane: Plane = new Plane(), basis: Basis = emptyBasis()) { this.plane = plane; this.basis = basis; + // TODO validate that basis is valid within plane. For example, remove + // selected cells from the basis that do not occur within the plane. } get activeCellId() { @@ -199,19 +205,17 @@ export default class SheetSelection { } /** - * @returns a new selection formed from one cell within the placeholder row. - * Note that we do not support selections of multiple cells in the placeholder - * row. + * @returns a new selection formed from one cell within the data rows or the + * placeholder row. */ - atPlaceholderCell(cellId: string): SheetSelection { - return this.withBasis(basisFromPlaceholderCell(cellId)); - } - - /** - * @returns a new selection formed from one cell within the data rows. - */ - atDataCell(cellId: string): SheetSelection { - return this.withBasis(basisFromDataCells([cellId], cellId)); + ofOneCell(cellId: string): SheetSelection { + const { rowId } = parseCellId(cellId); + const { placeholderRowId } = this.plane; + const makeBasis = + rowId === placeholderRowId + ? basisFromPlaceholderCell + : basisFromOneDataCell; + return this.withBasis(makeBasis(cellId)); } /** @@ -233,6 +237,10 @@ export default class SheetSelection { minRowId === undefined || maxRowId === undefined ) { + // TODO: in some cases maybe we can be smarter here. Instead of + // returning an empty selection, we could try to return a selection of + // the same dimensions that is placed as close as possible to the + // original selection. return new SheetSelection(newPlane); } const cellIds = newPlane.dataCellsInFlexibleRowColumnRange( @@ -345,13 +353,9 @@ export default class SheetSelection { // If we can't move anywhere, then do nothing return this; } - if (adjacent.type === 'dataCell') { - // Move to an adjacent data cell - return this.atDataCell(adjacent.cellId); - } - if (adjacent.type === 'placeholderCell') { - // Move to an adjacent placeholder cell - return this.atPlaceholderCell(adjacent.cellId); + if (adjacent.type === 'dataCell' || adjacent.type === 'placeholderCell') { + // Move to an adjacent data cell or adjacent placeholder cell + return this.ofOneCell(adjacent.cellId); } return assertExhaustive(adjacent); } diff --git a/mathesar_ui/src/systems/table-view/row/Row.svelte b/mathesar_ui/src/systems/table-view/row/Row.svelte index 967c17b677..07a1faab61 100644 --- a/mathesar_ui/src/systems/table-view/row/Row.svelte +++ b/mathesar_ui/src/systems/table-view/row/Row.svelte @@ -11,11 +11,7 @@ ID_ROW_CONTROL_COLUMN, type Row, } from '@mathesar/stores/table-data'; - import { - SheetRow, - SheetCell, - isRowSelected, - } from '@mathesar/components/sheet'; + import { SheetRow, SheetCell } from '@mathesar/components/sheet'; import { rowHeightPx } from '@mathesar/geometry'; import { ContextMenu } from '@mathesar/component-library'; import NewRecordMessage from './NewRecordMessage.svelte'; @@ -29,13 +25,8 @@ const tabularData = getTabularDataStoreFromContext(); - $: ({ - recordsData, - columnsDataStore, - meta, - processedColumns, - legacySelection: selection, - } = $tabularData); + $: ({ recordsData, columnsDataStore, meta, processedColumns, selection } = + $tabularData); $: ({ rowStatus, rowCreationStatus, @@ -50,29 +41,31 @@ $: creationStatus = $rowCreationStatus.get(rowKey)?.state; $: status = $rowStatus.get(rowKey); $: wholeRowState = status?.wholeRowState; - $: ({ selectedCells } = selection); - $: isSelected = rowHasRecord(row) && isRowSelected($selectedCells, row); + $: isSelected = $selection.rowIds.has(row.identifier); $: hasWholeRowErrors = wholeRowState === 'failure'; /** Including whole row errors and individual cell errors */ $: hasAnyErrors = !!status?.errorsFromWholeRowAndCells?.length; function checkAndCreateEmptyRow() { - if (isPlaceholderRow(row)) { - void recordsData.addEmptyRecord(); - selection.selectAndActivateFirstDataEntryCellInLastRow(); - } + // // TODO_3037 + // if (isPlaceholderRow(row)) { + // void recordsData.addEmptyRecord(); + // selection.selectAndActivateFirstDataEntryCellInLastRow(); + // } } const handleRowMouseDown = () => { - if (rowHasRecord(row) && !isPlaceholderRow(row)) { - selection.onRowSelectionStart(row); - } + // // TODO_3037 + // if (rowHasRecord(row) && !isPlaceholderRow(row)) { + // selection.onRowSelectionStart(row); + // } }; const handleRowMouseEnter = () => { - if (rowHasRecord(row) && !isPlaceholderRow(row)) { - selection.onMouseEnterRowHeaderWhileSelection(row); - } + // // TODO_3037 + // if (rowHasRecord(row) && !isPlaceholderRow(row)) { + // selection.onMouseEnterRowHeaderWhileSelection(row); + // } }; diff --git a/mathesar_ui/src/systems/table-view/row/RowCell.svelte b/mathesar_ui/src/systems/table-view/row/RowCell.svelte index d91fb0657a..0b9aa43f15 100644 --- a/mathesar_ui/src/systems/table-view/row/RowCell.svelte +++ b/mathesar_ui/src/systems/table-view/row/RowCell.svelte @@ -1,26 +1,27 @@ - - {#key id} {#if usesVirtualList} + import { first } from 'iter-tools'; + + import type { TableEntry } from '@mathesar/api/types/tables'; + import { ContextMenu } from '@mathesar/component-library'; import { - getTabularDataStoreFromContext, - ID_ADD_NEW_COLUMN, - ID_ROW_CONTROL_COLUMN, - } from '@mathesar/stores/table-data'; - import { - SheetHeader, SheetCell, SheetCellResizer, - isColumnSelected, + SheetHeader, } from '@mathesar/components/sheet'; import type { ProcessedColumn } from '@mathesar/stores/table-data'; + import { + getTabularDataStoreFromContext, + ID_ADD_NEW_COLUMN, + ID_ROW_CONTROL_COLUMN, + } from '@mathesar/stores/table-data'; import { saveColumnOrder } from '@mathesar/stores/tables'; - import type { TableEntry } from '@mathesar/api/types/tables'; - import { ContextMenu } from '@mathesar/component-library'; - import HeaderCell from './header-cell/HeaderCell.svelte'; - import NewColumnCell from './new-column-cell/NewColumnCell.svelte'; import { Draggable, Droppable } from './drag-and-drop'; import ColumnHeaderContextMenu from './header-cell/ColumnHeaderContextMenu.svelte'; + import HeaderCell from './header-cell/HeaderCell.svelte'; + import NewColumnCell from './new-column-cell/NewColumnCell.svelte'; const tabularData = getTabularDataStoreFromContext(); @@ -27,17 +28,7 @@ $: columnOrder = columnOrder ?? []; $: columnOrderString = columnOrder.map(String); - - $: ({ legacySelection: selection, processedColumns } = $tabularData); - $: ({ - selectedCells, - columnsSelectedWhenTheTableIsEmpty, - selectionInProgress, - } = selection); - $: selectedColumnIds = selection.getSelectedUniqueColumnsId( - $selectedCells, - $columnsSelectedWhenTheTableIsEmpty, - ); + $: ({ selection, processedColumns } = $tabularData); let locationOfFirstDraggedColumn: number | undefined = undefined; let selectedColumnIdsOrdered: string[] = []; @@ -55,7 +46,7 @@ columnOrderString = columnOrderString; // Remove selected column IDs and keep their order for (const id of columnOrderString) { - if (selectedColumnIds.map(String).includes(id)) { + if ($selection.columnIds.has(id)) { selectedColumnIdsOrdered.push(id); if (!locationOfFirstDraggedColumn) { locationOfFirstDraggedColumn = columnOrderString.indexOf(id); @@ -70,9 +61,8 @@ // Early exit if a column is dropped in the same place. // Should only be done for single column if non-continuous selection is allowed. if ( - selectedColumnIds.length > 0 && columnDroppedOn && - selectedColumnIds[0] === columnDroppedOn.id + first($selection.columnIds) === String(columnDroppedOn.id) ) { // Reset drag information locationOfFirstDraggedColumn = undefined; @@ -122,6 +112,7 @@ {#each [...$processedColumns] as [columnId, processedColumn] (columnId)} + {@const isSelected = $selection.columnIds.has(String(columnId))}
@@ -129,32 +120,27 @@ on:dragstart={() => dragColumn()} column={processedColumn} {selection} - selectionInProgress={$selectionInProgress} > dropColumn(processedColumn)} on:dragover={(e) => e.preventDefault()} {locationOfFirstDraggedColumn} columnLocation={columnOrderString.indexOf(columnId.toString())} - isSelected={isColumnSelected( - $selectedCells, - $columnsSelectedWhenTheTableIsEmpty, - processedColumn, - )} + {isSelected} > - selection.onColumnSelectionStart(processedColumn)} - on:mouseenter={() => - selection.onMouseEnterColumnHeaderWhileSelection( - processedColumn, - )} + {isSelected} + on:mousedown={() => { + // // TODO_3037 + // selection.onColumnSelectionStart(processedColumn) + }} + on:mouseenter={() => { + // // TODO_3037 + // selection.onMouseEnterColumnHeaderWhileSelection( + // processedColumn, + // ) + }} /> diff --git a/mathesar_ui/src/systems/table-view/header/drag-and-drop/Draggable.svelte b/mathesar_ui/src/systems/table-view/header/drag-and-drop/Draggable.svelte index 2dcfe0a5bd..1347bd2f5a 100644 --- a/mathesar_ui/src/systems/table-view/header/drag-and-drop/Draggable.svelte +++ b/mathesar_ui/src/systems/table-view/header/drag-and-drop/Draggable.svelte @@ -1,17 +1,19 @@
Date: Fri, 1 Sep 2023 12:53:56 -0400 Subject: [PATCH 0014/1141] Fix clipboard code to work with new selection code --- .../components/sheet/SheetClipboardHandler.ts | 116 +++++++----------- mathesar_ui/src/stores/table-data/records.ts | 22 ++-- .../src/stores/table-data/tabularData.ts | 12 +- .../src/systems/data-explorer/QueryRunner.ts | 15 ++- .../exploration-inspector/CellTab.svelte | 1 + .../data-explorer/result-pane/Results.svelte | 20 +-- .../src/systems/table-view/TableView.svelte | 86 ++++++------- .../src/systems/table-view/row/Row.svelte | 21 ++-- .../src/systems/table-view/row/RowCell.svelte | 3 +- mathesar_ui/src/utils/collectionUtils.ts | 3 + 10 files changed, 148 insertions(+), 151 deletions(-) create mode 100644 mathesar_ui/src/utils/collectionUtils.ts diff --git a/mathesar_ui/src/components/sheet/SheetClipboardHandler.ts b/mathesar_ui/src/components/sheet/SheetClipboardHandler.ts index 39728674d7..4d55f5aad3 100644 --- a/mathesar_ui/src/components/sheet/SheetClipboardHandler.ts +++ b/mathesar_ui/src/components/sheet/SheetClipboardHandler.ts @@ -1,19 +1,9 @@ import * as Papa from 'papaparse'; -import { get } from 'svelte/store'; -import { ImmutableSet, type MakeToast } from '@mathesar-component-library'; -import LegacySheetSelection, { - isCellSelected, -} from '@mathesar/components/sheet/LegacySheetSelection'; +import type { ImmutableSet } from '@mathesar-component-library'; import type { AbstractTypeCategoryIdentifier } from '@mathesar/stores/abstract-types/types'; import type { ClipboardHandler } from '@mathesar/stores/clipboard'; -import type { - ProcessedColumn, - RecordRow, - RecordSummariesForSheet, -} from '@mathesar/stores/table-data'; -import type { QueryRow } from '@mathesar/systems/data-explorer/QueryRunner'; -import type { ProcessedQueryOutputColumn } from '@mathesar/systems/data-explorer/utils'; +import type { RecordSummariesForSheet } from '@mathesar/stores/table-data'; import type { ReadableMapLike } from '@mathesar/typeUtils'; import { labeledCount } from '@mathesar/utils/languageUtils'; @@ -21,25 +11,34 @@ const MIME_PLAIN_TEXT = 'text/plain'; const MIME_MATHESAR_SHEET_CLIPBOARD = 'application/x-vnd.mathesar-sheet-clipboard'; -/** Keys are row ids, values are records */ -type IndexedRecords = Map>; +/** + * A column which allows the cells in it to be copied. + */ +interface CopyableColumn { + abstractType: { identifier: AbstractTypeCategoryIdentifier }; + formatCellValue: ( + cellValue: unknown, + recordSummaries?: RecordSummariesForSheet, + ) => string | null | undefined; +} -function getRawCellValue< - Column extends ProcessedQueryOutputColumn | ProcessedColumn, ->( - indexedRecords: IndexedRecords, - rowId: number, - columnId: Column['id'], -): unknown { - return indexedRecords.get(rowId)?.[String(columnId)]; +/** + * This is the stuff we need to know from the sheet in order to copy the content + * of cells to the clipboard. + */ +interface CopyingContext { + /** Keys are row ids, values are records */ + rowsMap: Map>; + columnsMap: ReadableMapLike; + recordSummaries: RecordSummariesForSheet; + selectedRowIds: ImmutableSet; + selectedColumnIds: ImmutableSet; } -function getFormattedCellValue< - Column extends ProcessedQueryOutputColumn | ProcessedColumn, ->( +function getFormattedCellValue( rawCellValue: unknown, - columnsMap: ReadableMapLike, - columnId: Column['id'], + columnsMap: CopyingContext['columnsMap'], + columnId: string, recordSummaries: RecordSummariesForSheet, ): string { if (rawCellValue === undefined || rawCellValue === null) { @@ -76,70 +75,37 @@ export interface StructuredCell { formatted: string; } -interface SheetClipboardHandlerDeps< - Row extends QueryRow | RecordRow, - Column extends ProcessedQueryOutputColumn | ProcessedColumn, -> { - selection: LegacySheetSelection; - toast: MakeToast; - getRows(): Row[]; - getColumnsMap(): ReadableMapLike; - getRecordSummaries(): RecordSummariesForSheet; +interface Dependencies { + getCopyingContext(): CopyingContext; + showToastInfo(msg: string): void; } -export class SheetClipboardHandler< - Row extends QueryRow | RecordRow, - Column extends ProcessedQueryOutputColumn | ProcessedColumn, -> implements ClipboardHandler -{ - private readonly deps: SheetClipboardHandlerDeps; +export class SheetClipboardHandler implements ClipboardHandler { + private readonly deps: Dependencies; - constructor(deps: SheetClipboardHandlerDeps) { + constructor(deps: Dependencies) { this.deps = deps; } - private getColumnIds(cells: ImmutableSet) { - return this.deps.selection.getSelectedUniqueColumnsId( - cells, - // We don't care about the columns selected when the table is empty, - // because we only care about cells selected that have content. - new ImmutableSet(), - ); - } - - private getRowIds(cells: ImmutableSet) { - return this.deps.selection.getSelectedUniqueRowsId(cells); - } - private getCopyContent(): { structured: string; tsv: string } { - const cells = get(this.deps.selection.selectedCells); - const indexedRecords = new Map( - this.deps.getRows().map((r) => [r.rowIndex, r.record]), - ); - const columns = this.deps.getColumnsMap(); - const recordSummaries = this.deps.getRecordSummaries(); - + const context = this.deps.getCopyingContext(); const tsvRows: string[][] = []; const structuredRows: StructuredCell[][] = []; - for (const rowId of this.getRowIds(cells)) { + for (const rowId of context.selectedRowIds) { const tsvRow: string[] = []; const structuredRow: StructuredCell[] = []; - for (const columnId of this.getColumnIds(cells)) { - const column = columns.get(columnId); - if (!isCellSelected(cells, { rowIndex: rowId }, { id: columnId })) { - // Ignore cells that are not selected. - continue; - } + for (const columnId of context.selectedColumnIds) { + const column = context.columnsMap.get(columnId); if (!column) { // Ignore cells with no associated column. This should never happen. continue; } - const rawCellValue = getRawCellValue(indexedRecords, rowId, columnId); + const rawCellValue = context.rowsMap.get(rowId)?.[columnId]; const formattedCellValue = getFormattedCellValue( rawCellValue, - columns, + context.columnsMap, columnId, - recordSummaries, + context.recordSummaries, ); const type = column.abstractType.identifier; structuredRow.push({ @@ -152,7 +118,9 @@ export class SheetClipboardHandler< tsvRows.push(tsvRow); structuredRows.push(structuredRow); } - this.deps.toast.info(`Copied ${labeledCount(cells.size, 'cells')}.`); + const cellCount = + context.selectedRowIds.size * context.selectedColumnIds.size; + this.deps.showToastInfo(`Copied ${labeledCount(cellCount, 'cells')}.`); return { structured: JSON.stringify(structuredRows), tsv: serializeTsv(tsvRows), diff --git a/mathesar_ui/src/stores/table-data/records.ts b/mathesar_ui/src/stores/table-data/records.ts index d2fcd34183..da90576fd9 100644 --- a/mathesar_ui/src/stores/table-data/records.ts +++ b/mathesar_ui/src/stores/table-data/records.ts @@ -26,11 +26,10 @@ import { States, deleteAPI, getAPI, + getQueryStringFromParams, patchAPI, postAPI, - getQueryStringFromParams, } from '@mathesar/api/utils/requestUtils'; -import Series from '@mathesar/components/sheet/selection/Series'; import type Pagination from '@mathesar/utils/Pagination'; import { getErrorMessage } from '@mathesar/utils/errors'; import { pluralize } from '@mathesar/utils/languageUtils'; @@ -175,7 +174,6 @@ export function filterRecordRows(rows: Row[]): RecordRow[] { export function rowHasSavedRecord(row: Row): row is RecordRow { return rowHasRecord(row) && Object.entries(row.record).length > 0; } - export interface TableRecordsData { state: States; error?: string; @@ -184,6 +182,10 @@ export interface TableRecordsData { grouping?: RecordGrouping; } +export function getRowSelectionId(row: Row): string { + return row.identifier; +} + export function getRowKey(row: Row, primaryKeyColumnId?: Column['id']): string { if (rowHasRecord(row) && primaryKeyColumnId !== undefined) { const primaryKeyCellValue = row.record[primaryKeyColumnId]; @@ -300,7 +302,8 @@ export class RecordsData { error: Writable; - selectableRowIds: Readable>; + /** Keys are row ids, values are records */ + selectableRowsMap: Readable>>; private promise: CancellablePromise | undefined; @@ -357,13 +360,12 @@ export class RecordsData { this.url = `/api/db/v0/tables/${this.tableId}/records/`; void this.fetch(); - this.selectableRowIds = derived( + this.selectableRowsMap = derived( [this.savedRecords, this.newRecords], - ([savedRecords, newRecords]) => - new Series([ - ...savedRecords.map((r) => r.identifier), - ...newRecords.map((r) => r.identifier), - ]), + ([savedRecords, newRecords]) => { + const records = [...savedRecords, ...newRecords]; + return new Map(records.map((r) => [getRowSelectionId(r), r.record])); + }, ); // TODO: Create base class to abstract subscriptions and unsubscriptions diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index 5029d92376..46d8453809 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -143,12 +143,12 @@ export class TabularData { ); const plane = derived( - [this.recordsData.selectableRowIds, this.orderedProcessedColumns], - ([selectableRowIds, orderedProcessedColumns]) => { - const columnIds = new Series( - [...orderedProcessedColumns.values()].map((c) => String(c.id)), - ); - return new Plane(selectableRowIds, columnIds); + [this.recordsData.selectableRowsMap, this.orderedProcessedColumns], + ([selectableRowsMap, orderedProcessedColumns]) => { + const rowIds = new Series([...selectableRowsMap.keys()]); + const columns = [...orderedProcessedColumns.values()]; + const columnIds = new Series(columns.map((c) => String(c.id))); + return new Plane(rowIds, columnIds); }, ); diff --git a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts index 1a678575b8..ae6c285652 100644 --- a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts +++ b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts @@ -1,5 +1,5 @@ -import { get, writable } from 'svelte/store'; -import type { Writable } from 'svelte/store'; +import { derived, get, writable } from 'svelte/store'; +import type { Readable, Writable } from 'svelte/store'; import type { RequestStatus } from '@mathesar/api/utils/requestUtils'; import { ApiMultiError } from '@mathesar/api/utils/errors'; import { ImmutableMap, CancellablePromise } from '@mathesar-component-library'; @@ -31,6 +31,10 @@ export interface QueryRow { rowIndex: number; } +function getRowSelectionId(row: QueryRow): string { + return String(row.rowIndex); +} + export interface QueryRowsData { totalCount: number; rows: QueryRow[]; @@ -63,6 +67,9 @@ export default class QueryRunner { new ImmutableMap(), ); + /** Keys are row ids, values are records */ + selectableRowsMap: Readable>>; + selection: QuerySheetSelection; inspector: QueryInspector; @@ -100,6 +107,10 @@ export default class QueryRunner { this.shareConsumer = shareConsumer; this.speculateProcessedColumns(); void this.run(); + this.selectableRowsMap = derived( + this.rowsData, + ({ rows }) => new Map(rows.map((r) => [getRowSelectionId(r), r.record])), + ); this.selection = new LegacySheetSelection({ getColumns: () => [...get(this.processedColumns).values()], getColumnOrder: () => diff --git a/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte b/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte index a9490a9ceb..e6fccb7554 100644 --- a/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte +++ b/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte @@ -10,6 +10,7 @@ const cell = $activeCell; if (cell) { const rows = queryHandler.getRows(); + // TODO_3037 change rowIndex to use getRowSelectionId if (rows[cell.rowIndex]) { return rows[cell.rowIndex].record[cell.columnId]; } diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte index 9703ad41a9..da32a65f4d 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte @@ -1,16 +1,16 @@
diff --git a/mathesar_ui/src/systems/table-view/row/Row.svelte b/mathesar_ui/src/systems/table-view/row/Row.svelte index 07a1faab61..a07cbddbc7 100644 --- a/mathesar_ui/src/systems/table-view/row/Row.svelte +++ b/mathesar_ui/src/systems/table-view/row/Row.svelte @@ -1,24 +1,25 @@
diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte index fe1e01f610..7be1b9cd62 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte @@ -21,18 +21,17 @@ $: database = $currentDatabase; $: schema = $currentSchema; - $: ({ processedColumns, legacySelection: selection } = $tabularData); - $: ({ selectedCells, columnsSelectedWhenTheTableIsEmpty } = selection); + $: ({ processedColumns, selection } = $tabularData); + // TODO_3037 verify that table inspector shows selected columns $: selectedColumns = (() => { - const ids = selection.getSelectedUniqueColumnsId( - $selectedCells, - $columnsSelectedWhenTheTableIsEmpty, - ); + const ids = $selection.columnIds; const columns = []; for (const id of ids) { - const c = $processedColumns.get(id); - if (c !== undefined) { - columns.push(c); + // TODO_3037 add code comments explaining why this is necessary + const parsedId = parseInt(id, 10); + const column = $processedColumns.get(parsedId); + if (column !== undefined) { + columns.push(column); } } return columns; diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte index 878701fec1..c342c3eefa 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte @@ -1,6 +1,5 @@
- {#if uniquelySelectedRowIndices.length} - {#if uniquelySelectedRowIndices.length > 1} + {#if selectedRowCount > 0} + {#if selectedRowCount > 1} - {labeledCount(uniquelySelectedRowIndices, 'records')} selected + {labeledCount(selectedRowCount, 'records')} selected {/if}
{:else} - Select one or more cells to view associated record properties. + + Select one or more cells to view associated record properties. + {/if}
diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record/RowActions.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record/RowActions.svelte index 66f343f999..53435f4981 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record/RowActions.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record/RowActions.svelte @@ -12,7 +12,6 @@ import type { ColumnsDataStore, RecordsData, - TabularDataSelection, } from '@mathesar/stores/table-data'; import { getPkValueInRecord } from '@mathesar/stores/table-data/records'; import { toast } from '@mathesar/stores/toast'; @@ -20,7 +19,6 @@ export let selectedRowIndices: number[]; export let recordsData: RecordsData; - export let selection: TabularDataSelection; export let columnsDataStore: ColumnsDataStore; export let canEditTableRecords: boolean; @@ -38,7 +36,8 @@ toast.success({ title: `${confirmationTitle} deleted successfully!`, }); - selection.resetSelection(); + // // TODO_3037 verify that selection behaves okay after deleting records + // selection.resetSelection(); }, }); } From dd7f24232bd9d5b3f10b30f7a85e859a6992d182 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 11 Sep 2023 13:28:10 -0400 Subject: [PATCH 0016/1141] Modify util fn to handle non-rectangular selection --- .../sheet/selection/SheetSelection.ts | 63 ++++++++++++++----- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index b2da58135f..3cf039710e 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -1,4 +1,4 @@ -import { first } from 'iter-tools'; +import { execPipe, filter, first, map } from 'iter-tools'; import { ImmutableSet } from '@mathesar/component-library'; import { assertExhaustive } from '@mathesar/utils/typeUtils'; @@ -98,30 +98,61 @@ function getFullySelectedColumnIds( basis: Basis, ): ImmutableSet { if (basis.type === 'dataCells') { - // The logic here is: if all rows are selected, then every selected column - // FULLY selected. Otherwise, no columns are fully selected. - // - // THIS LOGIC ASSUMES THAT ALL SELECTIONS ARE RECTANGULAR. The application - // enforces this assumption at various levels, but NOT within the Basis data - // structure. That is, it's theoretically possible to have basis data which - // represents a non-rectangular selection, but such data should be - // considered a bug. The logic within this function leverages this - // assumption for the purpose of improving performance. If we know that the - // selection is rectangular, then we can avoid iterating over all cells in - // each selected column to see if the column is fully selected. - return basis.rowIds.size < plane.rowIds.length - ? new ImmutableSet() - : basis.columnIds; + // The logic within this branch is somewhat complex because: + // - We want to suppor non-rectangular selections. + // - For performance, we want to avoid iterating over all the selected + // cells. + + const selctedRowCount = basis.rowIds.size; + const availableRowCount = plane.rowIds.length; + if (selctedRowCount < availableRowCount) { + // Performance heuristic. If the number of selected rows is less than the + // total number of rows, we can assume that no column exist in which all + // rows are selected. + return new ImmutableSet(); + } + + const selectedColumnCount = basis.columnIds.size; + const selectedCellCount = basis.cellIds.size; + const avgCellsSelectedPerColumn = selectedCellCount / selectedColumnCount; + if (avgCellsSelectedPerColumn === availableRowCount) { + // Performance heuristic. We know that no column can have more cells + // selected than the number of rows. Thus, if the average number of cells + // selected per column is equal to the number of rows, then we know that + // all selected columns are fully selected. + return basis.columnIds; + } + + // This is the worst-case scenario, performance-wise, which is why we try to + // return early before hitting this branch. This case will only happen when + // we have a mix of fully selected columns and partially selected columns. + // This case should be rare because most selections are rectangular. + const countSelectedCellsPerColumn = new Map(); + for (const cellId of basis.cellIds) { + const { columnId } = parseCellId(cellId); + const count = countSelectedCellsPerColumn.get(columnId) ?? 0; + countSelectedCellsPerColumn.set(columnId, count + 1); + } + const fullySelectedColumnIds = execPipe( + countSelectedCellsPerColumn, + filter(([, count]) => count === availableRowCount), + map(([id]) => id), + ); + return new ImmutableSet(fullySelectedColumnIds); } + if (basis.type === 'emptyColumns') { return basis.columnIds; } + if (basis.type === 'placeholderCell') { return new ImmutableSet(); } + if (basis.type === 'empty') { return new ImmutableSet(); } + return assertExhaustive(basis.type); } @@ -139,7 +170,7 @@ export default class SheetSelection { private readonly basis: Basis; - /** Ids of columns in which _all_ cells are selected */ + /** Ids of columns in which _all_ data cells are selected */ fullySelectedColumnIds: ImmutableSet; constructor(plane: Plane = new Plane(), basis: Basis = emptyBasis()) { From 70927ff2380a8376755be85347667f8edc48335f Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 11 Sep 2023 13:49:18 -0400 Subject: [PATCH 0017/1141] Refactor ExtractColumnsModal to use new selection --- mathesar_ui/src/components/sheet/cellIds.ts | 14 +++++ .../src/components/sheet/selection/Plane.ts | 15 +----- .../sheet/selection/SheetSelection.ts | 21 +++++++- mathesar_ui/src/stores/table-data/index.ts | 1 - .../src/stores/table-data/tabularData.ts | 54 ++----------------- .../ExtractColumnsModal.svelte | 12 ++--- 6 files changed, 44 insertions(+), 73 deletions(-) diff --git a/mathesar_ui/src/components/sheet/cellIds.ts b/mathesar_ui/src/components/sheet/cellIds.ts index 40c8297dea..fd6a263637 100644 --- a/mathesar_ui/src/components/sheet/cellIds.ts +++ b/mathesar_ui/src/components/sheet/cellIds.ts @@ -1,3 +1,7 @@ +import { map } from 'iter-tools'; + +import { cartesianProduct } from '@mathesar/utils/iterUtils'; + const CELL_ID_DELIMITER = '-'; /** @@ -22,3 +26,13 @@ export function parseCellId(cellId: string): { const columnId = cellId.slice(delimiterIndex + 1); return { rowId, columnId }; } + +export function makeCells( + rowIds: Iterable, + columnIds: Iterable, +): Iterable { + return map( + ([rowId, columnId]) => makeCellId(rowId, columnId), + cartesianProduct(rowIds, columnIds), + ); +} diff --git a/mathesar_ui/src/components/sheet/selection/Plane.ts b/mathesar_ui/src/components/sheet/selection/Plane.ts index 3f0d87780f..0c57de5c5b 100644 --- a/mathesar_ui/src/components/sheet/selection/Plane.ts +++ b/mathesar_ui/src/components/sheet/selection/Plane.ts @@ -1,20 +1,7 @@ -import { map } from 'iter-tools'; - -import { cartesianProduct } from '@mathesar/utils/iterUtils'; -import { makeCellId, parseCellId } from '../cellIds'; +import { makeCellId, makeCells, parseCellId } from '../cellIds'; import { Direction, getColumnOffset, getRowOffset } from './Direction'; import Series from './Series'; -function makeCells( - rowIds: Iterable, - columnIds: Iterable, -): Iterable { - return map( - ([rowId, columnId]) => makeCellId(rowId, columnId), - cartesianProduct(rowIds, columnIds), - ); -} - /** * This describes the different kinds of cells that can be adjacent to a given * cell in a particular direction. diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index 3cf039710e..53318a7af3 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -2,7 +2,7 @@ import { execPipe, filter, first, map } from 'iter-tools'; import { ImmutableSet } from '@mathesar/component-library'; import { assertExhaustive } from '@mathesar/utils/typeUtils'; -import { parseCellId } from '../cellIds'; +import { makeCells, parseCellId } from '../cellIds'; import { Direction, getColumnOffset } from './Direction'; import Plane from './Plane'; @@ -260,6 +260,17 @@ export default class SheetSelection { return this.withBasis(newBasis); } + /** + * @returns a new selection of all data cells in the intersection of the + * provided rows and columns. + */ + ofRowColumnIntersection( + rowIds: Iterable, + columnIds: Iterable, + ): SheetSelection { + return this.withBasis(basisFromDataCells(makeCells(rowIds, columnIds))); + } + /** * @returns a new selection formed by the rectangle between the provided * cells, inclusive. @@ -291,6 +302,14 @@ export default class SheetSelection { return this.withBasis(makeBasis(cellId)); } + ofOneRow(rowId: string): SheetSelection { + return this.ofRowRange(rowId, rowId); + } + + ofOneColumn(columnId: string): SheetSelection { + return this.ofColumnRange(columnId, columnId); + } + /** * @returns a new selection that fits within the provided plane. This is * useful when a column is deleted, reordered, or inserted. diff --git a/mathesar_ui/src/stores/table-data/index.ts b/mathesar_ui/src/stores/table-data/index.ts index 279ba7d931..2d68b467ac 100644 --- a/mathesar_ui/src/stores/table-data/index.ts +++ b/mathesar_ui/src/stores/table-data/index.ts @@ -42,7 +42,6 @@ export { getTabularDataStoreFromContext, TabularData, type TabularDataProps, - type TabularDataSelection, } from './tabularData'; export { type ProcessedColumn, diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index 820db37e0e..0e81eb4775 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -1,34 +1,24 @@ import { getContext, setContext } from 'svelte'; -import { - derived, - get, - writable, - type Readable, - type Writable, -} from 'svelte/store'; +import { derived, writable, type Readable, type Writable } from 'svelte/store'; import type { DBObjectEntry } from '@mathesar/AppTypes'; import type { TableEntry } from '@mathesar/api/types/tables'; import type { Column } from '@mathesar/api/types/tables/columns'; import { States } from '@mathesar/api/utils/requestUtils'; -import { LegacySheetSelection } from '@mathesar/components/sheet'; import Plane from '@mathesar/components/sheet/selection/Plane'; import Series from '@mathesar/components/sheet/selection/Series'; import SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; -import { getColumnOrder, orderProcessedColumns } from '@mathesar/utils/tables'; import type { ShareConsumer } from '@mathesar/utils/shares'; +import { orderProcessedColumns } from '@mathesar/utils/tables'; import { ColumnsDataStore } from './columns'; import type { ConstraintsData } from './constraints'; import { ConstraintsDataStore } from './constraints'; import { Display } from './display'; import { Meta } from './meta'; -import type { - ProcessedColumn, - ProcessedColumnsStore, -} from './processedColumns'; +import type { ProcessedColumnsStore } from './processedColumns'; import { processColumn } from './processedColumns'; -import type { RecordRow, TableRecordsData } from './records'; +import type { TableRecordsData } from './records'; import { RecordsData } from './records'; export interface TabularDataProps { @@ -50,11 +40,6 @@ export interface TabularDataProps { >[0]['hasEnhancedPrimaryKeyCell']; } -export type TabularDataSelection = LegacySheetSelection< - RecordRow, - ProcessedColumn ->; - export class TabularData { id: DBObjectEntry['id']; @@ -75,13 +60,6 @@ export class TabularData { isLoading: Readable; - /** - * TODO_3037 - * - * @deprecated - */ - legacySelection: TabularDataSelection; - selection: Writable; table: TableEntry; @@ -159,29 +137,6 @@ export class TabularData { plane.subscribe((p) => this.selection.update((s) => s.forNewPlane(p))), ); - this.legacySelection = new LegacySheetSelection({ - getColumns: () => [...get(this.processedColumns).values()], - getColumnOrder: () => - getColumnOrder([...get(this.processedColumns).values()], this.table), - getRows: () => this.recordsData.getRecordRows(), - getMaxSelectionRowIndex: () => { - const totalCount = get(this.recordsData.totalCount) ?? 0; - const savedRecords = get(this.recordsData.savedRecords); - const newRecords = get(this.recordsData.newRecords); - const pagination = get(this.meta.pagination); - const { offset } = pagination; - const pageSize = pagination.size; - /** - * We are not subtracting 1 from the below maxRowIndex calculation - * inorder to account for the add-new-record placeholder row - */ - return ( - Math.min(pageSize, totalCount - offset, savedRecords.length) + - newRecords.length - ); - }, - }); - this.isLoading = derived( [ this.columnsDataStore.fetchStatus, @@ -268,7 +223,6 @@ export class TabularData { this.recordsData.destroy(); this.constraintsDataStore.destroy(); this.columnsDataStore.destroy(); - this.legacySelection.destroy(); this.cleanupFunctions.forEach((f) => f()); } } diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte index ed5d468f15..f1fb68cfa3 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte @@ -48,11 +48,7 @@ export let controller: ExtractColumnsModalController; - $: ({ - processedColumns, - constraintsDataStore, - legacySelection: selection, - } = $tabularData); + $: ({ processedColumns, constraintsDataStore, selection } = $tabularData); $: ({ constraints } = $constraintsDataStore); $: availableProcessedColumns = [...$processedColumns.values()]; $: ({ targetType, columns, isOpen } = controller); @@ -110,7 +106,9 @@ // unmounting this component. return; } - selection.intersectSelectedRowsWithGivenColumns(_columns); + // TODO_3037 test to verify that selected columns are updated + const columnIds = _columns.map((c) => String(c.id)); + selection.update((s) => s.ofRowColumnIntersection(s.rowIds, columnIds)); } $: handleColumnsChange($columns); @@ -193,7 +191,7 @@ // will need to modify this logic when we position the new column where // the old columns were. const newFkColumn = allColumns.slice(-1)[0]; - selection.toggleColumnSelection(newFkColumn); + selection.update((s) => s.ofOneColumn(String(newFkColumn.id))); await tick(); scrollBasedOnSelection(); } From 9bc1e5ed2a05b69dadc81567a2350cb7a978389e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 11 Sep 2023 15:03:38 -0400 Subject: [PATCH 0018/1141] Deprecate selection within Data Explorer --- .../src/stores/table-data/tabularData.ts | 2 +- .../src/systems/data-explorer/QueryRunner.ts | 78 +++++++++++++------ .../exploration-inspector/CellTab.svelte | 2 +- .../column-tab/ColumnTab.svelte | 6 +- .../data-explorer/result-pane/Results.svelte | 2 +- 5 files changed, 64 insertions(+), 26 deletions(-) diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index 0e81eb4775..a16572f80f 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -64,7 +64,7 @@ export class TabularData { table: TableEntry; - cleanupFunctions: (() => void)[] = []; + private cleanupFunctions: (() => void)[] = []; shareConsumer?: ShareConsumer; diff --git a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts index ae6c285652..e2089bee6d 100644 --- a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts +++ b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts @@ -1,29 +1,33 @@ -import { derived, get, writable } from 'svelte/store'; import type { Readable, Writable } from 'svelte/store'; -import type { RequestStatus } from '@mathesar/api/utils/requestUtils'; -import { ApiMultiError } from '@mathesar/api/utils/errors'; -import { ImmutableMap, CancellablePromise } from '@mathesar-component-library'; -import Pagination from '@mathesar/utils/Pagination'; +import { derived, get, writable } from 'svelte/store'; + +import { CancellablePromise, ImmutableMap } from '@mathesar-component-library'; import type { + QueryColumnMetaData, QueryResultRecord, - QueryRunResponse, QueryResultsResponse, - QueryColumnMetaData, + QueryRunResponse, } from '@mathesar/api/types/queries'; -import { runQuery, fetchQueryResults } from '@mathesar/stores/queries'; +import { ApiMultiError } from '@mathesar/api/utils/errors'; +import type { RequestStatus } from '@mathesar/api/utils/requestUtils'; import { LegacySheetSelection } from '@mathesar/components/sheet'; +import Plane from '@mathesar/components/sheet/selection/Plane'; +import Series from '@mathesar/components/sheet/selection/Series'; +import SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; +import { fetchQueryResults, runQuery } from '@mathesar/stores/queries'; +import Pagination from '@mathesar/utils/Pagination'; import type { ShareConsumer } from '@mathesar/utils/shares'; -import type QueryModel from './QueryModel'; import QueryInspector from './QueryInspector'; +import type QueryModel from './QueryModel'; import { - processColumnMetaData, getProcessedOutputColumns, + processColumnMetaData, speculateColumnMetaData, + type InputColumnsStoreSubstance, type ProcessedQueryOutputColumn, - type ProcessedQueryResultColumnMap, type ProcessedQueryOutputColumnMap, - type InputColumnsStoreSubstance, + type ProcessedQueryResultColumnMap, } from './utils'; export interface QueryRow { @@ -40,6 +44,7 @@ export interface QueryRowsData { rows: QueryRow[]; } +/** @deprecated TODO_3037 remove */ export type QuerySheetSelection = LegacySheetSelection< QueryRow, ProcessedQueryOutputColumn @@ -70,7 +75,10 @@ export default class QueryRunner { /** Keys are row ids, values are records */ selectableRowsMap: Readable>>; - selection: QuerySheetSelection; + /** @deprecated TODO_3037 remove */ + legacySelection: QuerySheetSelection; + + selection: Writable; inspector: QueryInspector; @@ -84,6 +92,8 @@ export default class QueryRunner { private shareConsumer?: ShareConsumer; + private cleanupFunctions: (() => void)[] = []; + constructor({ query, abstractTypeMap, @@ -111,7 +121,8 @@ export default class QueryRunner { this.rowsData, ({ rows }) => new Map(rows.map((r) => [getRowSelectionId(r), r.record])), ); - this.selection = new LegacySheetSelection({ + + this.legacySelection = new LegacySheetSelection({ getColumns: () => [...get(this.processedColumns).values()], getColumnOrder: () => [...get(this.processedColumns).values()].map((column) => column.id), @@ -125,6 +136,27 @@ export default class QueryRunner { return Math.min(pageSize, totalCount - offset, rowLength) - 1; }, }); + + const plane = derived( + [this.rowsData, this.processedColumns], + ([{ rows }, columnsMap]) => { + const rowIds = new Series(rows.map(getRowSelectionId)); + const columns = [...columnsMap.values()]; + const columnIds = new Series(columns.map((c) => String(c.id))); + return new Plane(rowIds, columnIds); + }, + ); + + this.selection = writable(new SheetSelection()); + + this.cleanupFunctions.push( + plane.subscribe((p) => this.selection.update((s) => s.forNewPlane(p))), + ); + + this.cleanupFunctions.push( + plane.subscribe((p) => this.selection.update((s) => s.forNewPlane(p))), + ); + this.inspector = new QueryInspector(this.query); } @@ -237,7 +269,7 @@ export default class QueryRunner { async setPagination(pagination: Pagination): Promise { this.pagination.set(pagination); await this.run(); - this.selection.activateFirstCellInSelectedColumn(); + this.legacySelection.activateFirstCellInSelectedColumn(); } protected resetPagination(): void { @@ -262,30 +294,31 @@ export default class QueryRunner { protected async resetPaginationAndRun(): Promise { this.resetPagination(); await this.run(); - this.selection.activateFirstCellInSelectedColumn(); + this.legacySelection.activateFirstCellInSelectedColumn(); } selectColumn(alias: QueryColumnMetaData['alias']): void { const processedColumn = get(this.processedColumns).get(alias); if (!processedColumn) { - this.selection.resetSelection(); - this.selection.selectAndActivateFirstCellIfExists(); + this.legacySelection.resetSelection(); + this.legacySelection.selectAndActivateFirstCellIfExists(); this.inspector.selectCellTab(); return; } - const isSelected = this.selection.toggleColumnSelection(processedColumn); + const isSelected = + this.legacySelection.toggleColumnSelection(processedColumn); if (isSelected) { this.inspector.selectColumnTab(); return; } - this.selection.activateFirstCellInSelectedColumn(); + this.legacySelection.activateFirstCellInSelectedColumn(); this.inspector.selectCellTab(); } clearSelection(): void { - this.selection.resetSelection(); + this.legacySelection.resetSelection(); } getRows(): QueryRow[] { @@ -297,7 +330,8 @@ export default class QueryRunner { } destroy(): void { - this.selection.destroy(); + this.legacySelection.destroy(); this.runPromise?.cancel(); + this.cleanupFunctions.forEach((fn) => fn()); } } diff --git a/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte b/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte index e6fccb7554..e90240883c 100644 --- a/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte +++ b/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte @@ -3,7 +3,7 @@ import type QueryRunner from '../QueryRunner'; export let queryHandler: QueryRunner; - $: ({ selection, processedColumns } = queryHandler); + $: ({ legacySelection: selection, processedColumns } = queryHandler); $: ({ activeCell } = selection); $: selectedCellValue = (() => { diff --git a/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/ColumnTab.svelte b/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/ColumnTab.svelte index c7c3d42332..0f1847c6d1 100644 --- a/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/ColumnTab.svelte +++ b/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/ColumnTab.svelte @@ -14,7 +14,11 @@ $: queryManager = queryHandler instanceof QueryManager ? queryHandler : undefined; - $: ({ selection, columnsMetaData, processedColumns } = queryHandler); + $: ({ + legacySelection: selection, + columnsMetaData, + processedColumns, + } = queryHandler); $: ({ selectedCells, columnsSelectedWhenTheTableIsEmpty } = selection); $: selectedColumns = (() => { const ids = selection.getSelectedUniqueColumnsId( diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte index da32a65f4d..8004a6bd0b 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte @@ -37,7 +37,7 @@ selectableRowsMap, pagination, runState, - selection, + legacySelection: selection, inspector, } = queryHandler); $: ({ initial_columns } = $query); From aeae784729b10752f03df40923dbe9a93697c60b Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 11 Sep 2023 20:26:59 -0400 Subject: [PATCH 0019/1141] Use new Selection code in data explorer cell tab --- .../src/systems/data-explorer/QueryRunner.ts | 6 +-- .../exploration-inspector/CellTab.svelte | 49 +++++++------------ 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts index e2089bee6d..7e69371554 100644 --- a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts +++ b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts @@ -35,7 +35,7 @@ export interface QueryRow { rowIndex: number; } -function getRowSelectionId(row: QueryRow): string { +export function getRowSelectionId(row: QueryRow): string { return String(row.rowIndex); } @@ -321,10 +321,6 @@ export default class QueryRunner { this.legacySelection.resetSelection(); } - getRows(): QueryRow[] { - return get(this.rowsData).rows; - } - getQueryModel(): QueryModel { return get(this.query); } diff --git a/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte b/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte index e90240883c..dfc26ed3d4 100644 --- a/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte +++ b/mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte @@ -1,48 +1,33 @@ -
- {#if selectedCellValue !== undefined} +
+ {#if cellValue !== undefined}
Content
- {#if processedQueryColumn} + {#if column} {/if}
From efa0dcb2fc2b71283bd831db94e5e623769bd6cc Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 11 Sep 2023 20:58:09 -0400 Subject: [PATCH 0020/1141] Use new Selection code in DE col tab and results --- .../column-tab/ColumnTab.svelte | 34 +++---- .../result-pane/ResultRowCell.svelte | 96 ++++++++----------- .../data-explorer/result-pane/Results.svelte | 13 +-- 3 files changed, 55 insertions(+), 88 deletions(-) diff --git a/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/ColumnTab.svelte b/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/ColumnTab.svelte index 0f1847c6d1..e37d358fdc 100644 --- a/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/ColumnTab.svelte +++ b/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/ColumnTab.svelte @@ -1,12 +1,15 @@ - +
{#if row || recordRunState === 'processing'} { - if (row) { - selection.activateCell(row, processedQueryColumn); - inspector.selectCellTab(); - } + // // TODO_3037 + // if (row) { + // selection.activateCell(row, processedQueryColumn); + // inspector.selectCellTab(); + // } }} on:onSelectionStart={() => { - if (row) { - selection.onStartSelection(row, processedQueryColumn); - } + // // TODO_3037 + // if (row) { + // selection.onStartSelection(row, processedQueryColumn); + // } }} on:onMouseEnterCellWhileSelection={() => { - if (row) { - // This enables the click + drag to - // select multiple cells - selection.onMouseEnterCellWhileSelection(row, processedQueryColumn); - } + // // TODO_3037 + // if (row) { + // // This enables the click + drag to + // // select multiple cells + // selection.onMouseEnterCellWhileSelection(row, processedQueryColumn); + // } }} /> {/if} diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte index 8004a6bd0b..63370db7c0 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte @@ -10,7 +10,6 @@ SheetHeader, SheetRow, SheetVirtualRows, - isColumnSelected, } from '@mathesar/components/sheet'; import { SheetClipboardHandler } from '@mathesar/components/sheet/SheetClipboardHandler'; import { rowHeaderWidthPx, rowHeightPx } from '@mathesar/geometry'; @@ -37,7 +36,7 @@ selectableRowsMap, pagination, runState, - legacySelection: selection, + selection, inspector, } = queryHandler); $: ({ initial_columns } = $query); @@ -53,7 +52,7 @@ }), showToastInfo: toast.info, }); - $: ({ selectedCells, columnsSelectedWhenTheTableIsEmpty } = selection); + $: ({ columnIds } = $selection); $: recordRunState = $runState?.state; $: errors = $runState?.state === 'failure' ? $runState.errors : undefined; $: columnList = [...$processedColumns.values()]; @@ -102,11 +101,7 @@ {/each} @@ -139,8 +134,8 @@ {#each columnList as processedQueryColumn (processedQueryColumn.id)} Date: Mon, 11 Sep 2023 21:24:54 -0400 Subject: [PATCH 0021/1141] Remove lots of dead code --- .../components/sheet/LegacySheetSelection.ts | 731 ------------------ mathesar_ui/src/components/sheet/index.ts | 8 +- .../components/sheet/sheetScollingUtils.ts | 49 ++ .../systems/data-explorer/ActionsPane.svelte | 1 - .../src/systems/data-explorer/QueryRunner.ts | 53 +- .../column-tab/DeleteColumnAction.svelte | 1 - 6 files changed, 52 insertions(+), 791 deletions(-) delete mode 100644 mathesar_ui/src/components/sheet/LegacySheetSelection.ts create mode 100644 mathesar_ui/src/components/sheet/sheetScollingUtils.ts diff --git a/mathesar_ui/src/components/sheet/LegacySheetSelection.ts b/mathesar_ui/src/components/sheet/LegacySheetSelection.ts deleted file mode 100644 index 0aeeeb31c6..0000000000 --- a/mathesar_ui/src/components/sheet/LegacySheetSelection.ts +++ /dev/null @@ -1,731 +0,0 @@ -import { ImmutableSet, WritableSet } from '@mathesar-component-library'; -import { get, writable, type Unsubscriber, type Writable } from 'svelte/store'; - -export interface SelectionColumn { - id: number | string; -} - -export interface SelectionRow { - rowIndex: number; -} - -// TODO: Select active cell using primary key instead of index -// Checkout scenarios with pk consisting multiple columns -export interface ActiveCell { - rowIndex: number; - columnId: number | string; -} - -enum Direction { - Up = 'up', - Down = 'down', - Left = 'left', - Right = 'right', -} - -function getDirection(event: KeyboardEvent): Direction | undefined { - const { key } = event; - const shift = event.shiftKey; - switch (true) { - case shift && key === 'Tab': - return Direction.Left; - case shift: - return undefined; - case key === 'ArrowUp': - return Direction.Up; - case key === 'ArrowDown': - return Direction.Down; - case key === 'ArrowLeft': - return Direction.Left; - case key === 'ArrowRight': - case key === 'Tab': - return Direction.Right; - default: - return undefined; - } -} - -function getHorizontalDelta(direction: Direction): number { - switch (direction) { - case Direction.Left: - return -1; - case Direction.Right: - return 1; - default: - return 0; - } -} - -function getVerticalDelta(direction: Direction): number { - switch (direction) { - case Direction.Up: - return -1; - case Direction.Down: - return 1; - default: - return 0; - } -} - -export function isCellActive( - activeCell: ActiveCell, - row: SelectionRow, - column: SelectionColumn, -): boolean { - return ( - activeCell && - activeCell?.columnId === column.id && - activeCell.rowIndex === row.rowIndex - ); -} - -// TODO: Create a common utility action to handle active element based scroll -function scrollToElement(htmlElement: HTMLElement | null): void { - const activeRow = htmlElement?.parentElement; - const container = document.querySelector('[data-sheet-body-element="list"]'); - if (!container || !activeRow) { - return; - } - // Vertical scroll - if ( - activeRow.offsetTop + activeRow.clientHeight + 40 > - container.scrollTop + container.clientHeight - ) { - const offsetValue: number = - container.getBoundingClientRect().bottom - - activeRow.getBoundingClientRect().bottom - - 40; - container.scrollTop -= offsetValue; - } else if (activeRow.offsetTop - 30 < container.scrollTop) { - container.scrollTop = activeRow.offsetTop - 30; - } - - // Horizontal scroll - if ( - htmlElement.offsetLeft + activeRow.clientWidth + 30 > - container.scrollLeft + container.clientWidth - ) { - const offsetValue: number = - container.getBoundingClientRect().right - - htmlElement.getBoundingClientRect().right - - 30; - container.scrollLeft -= offsetValue; - } else if (htmlElement.offsetLeft - 30 < container.scrollLeft) { - container.scrollLeft = htmlElement.offsetLeft - 30; - } -} - -export function scrollBasedOnActiveCell(): void { - const activeCell: HTMLElement | null = document.querySelector( - '[data-sheet-element="cell"].is-active', - ); - scrollToElement(activeCell); -} - -export function scrollBasedOnSelection(): void { - const selectedCell: HTMLElement | null = document.querySelector( - '[data-sheet-element="cell"].is-selected', - ); - scrollToElement(selectedCell); -} - -type SelectionBounds = { - startRowIndex: number; - endRowIndex: number; - startColumnId: number | string; - endColumnId: number | string; -}; - -type Cell = [ - Row, - Column, -]; - -const ROW_COLUMN_SEPARATOR = '-'; - -/** - * Creates Unique identifier for a cell using rowIndex and columnId - * Storing this identifier instead of an object {rowIndex: number, columnId: number} - * enables easier usage of the Set data type & faster equality checks - */ -const createSelectedCellIdentifier = ( - { rowIndex }: Pick, - { id }: Pick, -): string => `${rowIndex}${ROW_COLUMN_SEPARATOR}${id}`; - -export const isRowSelected = ( - selectedCells: ImmutableSet, - row: SelectionRow, -): boolean => - selectedCells - .valuesArray() - .some((cell) => cell.startsWith(`${row.rowIndex}-`)); - -export const isColumnSelected = ( - selectedCells: ImmutableSet, - columnsSelectedWhenTheTableIsEmpty: ImmutableSet, - column: Pick, -): boolean => - columnsSelectedWhenTheTableIsEmpty.has(column.id) || - selectedCells.valuesArray().some((cell) => cell.endsWith(`-${column.id}`)); - -export const isCellSelected = ( - selectedCells: ImmutableSet, - row: Pick, - column: Pick, -): boolean => selectedCells.has(createSelectedCellIdentifier(row, column)); - -// The following function is similar to splitting the string with max_split = 1. -// input "a-b-c" => output "b-c" -// input "a-b" => output "b" -function splitWithLimit(str: string): string { - const tokens = str.split(ROW_COLUMN_SEPARATOR); - return tokens.slice(1).join(ROW_COLUMN_SEPARATOR); -} - -function getSelectedColumnId(selectedCell: string): SelectionColumn['id'] { - const columnId = splitWithLimit(selectedCell); - const numericalColumnId = Number(columnId); - if (Number.isNaN(numericalColumnId)) { - return columnId; - } - return numericalColumnId; -} - -export function getSelectedRowIndex(selectedCell: string): number { - return Number(selectedCell.split(ROW_COLUMN_SEPARATOR)[0]); -} - -/** - * @deprecated - */ -export default class LegacySheetSelection< - Row extends SelectionRow, - Column extends SelectionColumn, -> { - private getColumns: () => Column[]; - - private getColumnOrder: () => string[] | number[]; - - private getRows: () => Row[]; - - // max index is inclusive - private getMaxSelectionRowIndex: () => number; - - activeCell: Writable; - - private selectionBounds: SelectionBounds | undefined; - - private activeCellUnsubscriber: Unsubscriber; - - selectedCells: WritableSet; - - /** - * When the table has a non-zero number of rows, we store the user's selection - * in the `selectedCells` store. But when the table has no rows (and thus no - * cells) we still need a way to select columns to configure the data types, - * so we use this store as a workaround. More elegant solutions are being - * discussed in [1732][1]. - * - * [1]: https://github.com/centerofci/mathesar/issues/1732 - */ - columnsSelectedWhenTheTableIsEmpty: WritableSet; - - freezeSelection: boolean; - - selectionInProgress: Writable; - - constructor(args: { - getColumns: () => Column[]; - getColumnOrder: () => string[] | number[]; - getRows: () => Row[]; - getMaxSelectionRowIndex: () => number; - }) { - this.selectedCells = new WritableSet(); - this.columnsSelectedWhenTheTableIsEmpty = new WritableSet(); - this.getColumns = args.getColumns; - this.getColumnOrder = args.getColumnOrder; - this.getRows = args.getRows; - this.getMaxSelectionRowIndex = args.getMaxSelectionRowIndex; - this.freezeSelection = false; - this.selectionInProgress = writable(false); - this.activeCell = writable(undefined); - - /** - * TODO: - * - This adds a document level event listener for each selection - * store instance, and the listener doesn't seem to get removed. - * - Refactor this logic and avoid such listeners within the store instance. - */ - // This event terminates the cell selection process - // specially useful when selecting multiple cells - // Adding this on document to enable boundry cells selection - // when the user drags the mouse out of the table view - document.addEventListener('mouseup', () => { - this.onEndSelection(); - }); - - // Keep active cell and selected cell in sync - this.activeCellUnsubscriber = this.activeCell.subscribe((activeCell) => { - if (activeCell) { - const activeCellRow = this.getRows().find( - (row) => row.rowIndex === activeCell.rowIndex, - ); - const activeCellColumn = this.getColumns().find( - (column) => column.id === activeCell.columnId, - ); - if (activeCellRow && activeCellColumn) { - /** - * This handles the very rare edge case - * when the user starts the selection using mouse - * but before ending(mouseup event) - * she change the active cell using keyboard - */ - this.selectionBounds = undefined; - this.selectMultipleCells([[activeCellRow, activeCellColumn]]); - } else { - // We need to unselect the Selected cells - // when navigating Placeholder cells - this.resetSelection(); - } - } else { - this.resetSelection(); - } - }); - } - - onStartSelection(row: SelectionRow, column: SelectionColumn): void { - if (this.freezeSelection) { - return; - } - - this.selectionInProgress.set(true); - // Initialize the bounds of the selection - this.selectionBounds = { - startColumnId: column.id, - endColumnId: column.id, - startRowIndex: row.rowIndex, - endRowIndex: row.rowIndex, - }; - - const cells = this.getIncludedCells(this.selectionBounds); - this.selectMultipleCells(cells); - } - - onMouseEnterCellWhileSelection( - row: SelectionRow, - column: SelectionColumn, - ): void { - const { rowIndex } = row; - const { id } = column; - - // If there is no selection start cell, - // this means the selection was never initiated - if (!this.selectionBounds || this.freezeSelection) { - return; - } - - this.selectionBounds.endRowIndex = rowIndex; - this.selectionBounds.endColumnId = id; - const cells = this.getIncludedCells(this.selectionBounds); - this.selectMultipleCells(cells); - } - - onEndSelection(): void { - if (this.selectionBounds) { - const cells = this.getIncludedCells(this.selectionBounds); - this.selectMultipleCells(cells); - this.selectionBounds = undefined; - } - this.selectionInProgress.set(false); - } - - selectAndActivateFirstCellIfExists(): void { - const firstRow = this.getRows()[0]; - const firstColumn = this.getColumns()[0]; - if (firstRow && firstColumn) { - this.selectMultipleCells([[firstRow, firstColumn]]); - this.activateCell(firstRow, firstColumn); - } - } - - selectAndActivateFirstDataEntryCellInLastRow(): void { - const currentRows = this.getRows(); - const currentColumns = this.getColumns(); - if (currentRows.length > 0 && currentColumns.length > 1) { - this.activateCell(currentRows[currentRows.length - 1], currentColumns[1]); - } - } - - getIncludedCells(selectionBounds: SelectionBounds): Cell[] { - const { startRowIndex, endRowIndex, startColumnId, endColumnId } = - selectionBounds; - const minRowIndex = Math.min(startRowIndex, endRowIndex); - const maxRowIndex = Math.max(startRowIndex, endRowIndex); - - const columnOrder = this.getColumnOrder(); - - const startOrderIndex = columnOrder.findIndex((id) => id === startColumnId); - const endOrderIndex = columnOrder.findIndex((id) => id === endColumnId); - - const minColumnPosition = Math.min(startOrderIndex, endOrderIndex); - const maxColumnPosition = Math.max(startOrderIndex, endOrderIndex); - const columnOrderSelected = columnOrder.slice( - minColumnPosition, - maxColumnPosition + 1, - ); - - const columns = this.getColumns(); - - const cells: Cell[] = []; - this.getRows().forEach((row) => { - const { rowIndex } = row; - if (rowIndex >= minRowIndex && rowIndex <= maxRowIndex) { - columnOrderSelected.forEach((columnId) => { - const column = columns.find((c) => c.id === columnId); - if (column) { - cells.push([row, column]); - } - }); - } - }); - - return cells; - } - - private selectMultipleCells(cells: Array>) { - const identifiers = cells.map(([row, column]) => - createSelectedCellIdentifier(row, column), - ); - this.selectedCells.reconstruct(identifiers); - } - - resetSelection(): void { - this.selectionBounds = undefined; - this.columnsSelectedWhenTheTableIsEmpty.clear(); - this.selectedCells.clear(); - } - - isCompleteColumnSelected(column: Pick): boolean { - if (this.getRows().length) { - return ( - this.columnsSelectedWhenTheTableIsEmpty.getHas(column.id) || - this.getRows().every((row) => - isCellSelected(get(this.selectedCells), row, column), - ) - ); - } - return this.columnsSelectedWhenTheTableIsEmpty.getHas(column.id); - } - - private isCompleteRowSelected(row: Pick): boolean { - const columns = this.getColumns(); - return ( - columns.length > 0 && - columns.every((column) => - isCellSelected(get(this.selectedCells), row, column), - ) - ); - } - - isAnyColumnCompletelySelected(): boolean { - const selectedCellsArray = get(this.selectedCells).valuesArray(); - const checkedColumns: (number | string)[] = []; - - for (const cell of selectedCellsArray) { - const columnId = getSelectedColumnId(cell); - if (!checkedColumns.includes(columnId)) { - if (this.isCompleteColumnSelected({ id: columnId })) { - return true; - } - checkedColumns.push(columnId); - } - } - - return false; - } - - isAnyRowCompletelySelected(): boolean { - const selectedCellsArray = get(this.selectedCells).valuesArray(); - const checkedRows: number[] = []; - - for (const cell of selectedCellsArray) { - const rowIndex = getSelectedRowIndex(cell); - if (!checkedRows.includes(rowIndex)) { - if (this.isCompleteRowSelected({ rowIndex })) { - return true; - } - checkedRows.push(rowIndex); - } - } - - return false; - } - - /** - * Modifies the selected cells, forming a new selection by maintaining the - * currently selected rows but altering the selected columns to match the - * supplied columns. - */ - intersectSelectedRowsWithGivenColumns(columns: Column[]): void { - const selectedRows = this.getSelectedUniqueRowsId( - new ImmutableSet(this.selectedCells.getValues()), - ); - const cells: Cell[] = []; - columns.forEach((column) => { - selectedRows.forEach((rowIndex) => { - const row = this.getRows()[rowIndex]; - cells.push([row, column]); - }); - }); - - this.selectMultipleCells(cells); - } - - /** - * Use this only for programmatic selection - * - * Prefer: onColumnSelectionStart when - * selection is done using - * user interactions - */ - toggleColumnSelection(column: Column): boolean { - const isCompleteColumnSelected = this.isCompleteColumnSelected(column); - this.activateCell({ rowIndex: 0 }, column); - - if (isCompleteColumnSelected) { - this.resetSelection(); - return false; - } - - const rows = this.getRows(); - - if (rows.length === 0) { - this.resetSelection(); - this.columnsSelectedWhenTheTableIsEmpty.add(column.id); - return true; - } - - const cells: Cell[] = []; - rows.forEach((row) => { - cells.push([row, column]); - }); - - // Clearing the selection - // since we do not have cmd+click to select - // disjointed cells - this.resetSelection(); - this.selectMultipleCells(cells); - return true; - } - - /** - * Use this only for programmatic selection - * - * Prefer: onRowSelectionStart when - * selection is done using - * user interactions - */ - toggleRowSelection(row: Row): void { - const isCompleteRowSelected = this.isCompleteRowSelected(row); - - if (isCompleteRowSelected) { - // Clear the selection - deselect the row - this.resetSelection(); - } else { - const cells: Cell[] = []; - this.getColumns().forEach((column) => { - cells.push([row, column]); - }); - - // Clearing the selection - // since we do not have cmd+click to select - // disjointed cells - this.resetSelection(); - this.selectMultipleCells(cells); - } - } - - onColumnSelectionStart(column: Column): boolean { - if (!this.isCompleteColumnSelected(column)) { - this.activateCell({ rowIndex: 0 }, { id: column.id }); - const rows = this.getRows(); - - if (rows.length === 0) { - this.resetSelection(); - this.columnsSelectedWhenTheTableIsEmpty.add(column.id); - return true; - } - - this.onStartSelection(rows[0], column); - this.onMouseEnterCellWhileSelection(rows[rows.length - 1], column); - } - return true; - } - - onMouseEnterColumnHeaderWhileSelection(column: Column): boolean { - const rows = this.getRows(); - - if (rows.length === 0) { - this.resetSelection(); - this.columnsSelectedWhenTheTableIsEmpty.add(column.id); - return true; - } - - this.onMouseEnterCellWhileSelection(rows[rows.length - 1], column); - return true; - } - - onRowSelectionStart(row: Row): boolean { - const columns = this.getColumns(); - const columnOrder = this.getColumnOrder(); - - if (!columns.length) { - // Not possible to have tables without columns - } - - const startColumnId = columnOrder[0]; - const endColumnId = columnOrder[columnOrder.length - 1]; - const startColumn = columns.find((c) => c.id === startColumnId); - const endColumn = columns.find((c) => c.id === endColumnId); - - if (startColumn && endColumn) { - this.activateCell(row, startColumn); - this.onStartSelection(row, startColumn); - this.onMouseEnterCellWhileSelection(row, endColumn); - } - - return true; - } - - onMouseEnterRowHeaderWhileSelection(row: Row): boolean { - const columns = this.getColumns(); - - if (!columns.length) { - // Not possible to have tables without columns - } - - const endColumn = columns[columns.length - 1]; - this.onMouseEnterCellWhileSelection(row, endColumn); - return true; - } - - resetActiveCell(): void { - this.activeCell.set(undefined); - } - - activateCell(row: Pick, column: Pick): void { - this.activeCell.set({ - rowIndex: row.rowIndex, - columnId: column.id, - }); - } - - focusCell(row: Pick, column: Pick): void { - const cellsInTheColumn = document.querySelectorAll( - `[data-column-identifier="${column.id}"]`, - ); - const targetCell = cellsInTheColumn.item(row.rowIndex); - (targetCell?.querySelector('.cell-wrapper') as HTMLElement)?.focus(); - } - - private getAdjacentCell( - activeCell: ActiveCell, - direction: Direction, - ): ActiveCell | undefined { - const columnOrder = this.getColumnOrder(); - - const rowIndex = (() => { - const delta = getVerticalDelta(direction); - if (delta === 0) { - return activeCell.rowIndex; - } - const minRowIndex = 0; - const maxRowIndex = this.getMaxSelectionRowIndex(); - const newRowIndex = activeCell.rowIndex + delta; - if (newRowIndex < minRowIndex || newRowIndex > maxRowIndex) { - return undefined; - } - return newRowIndex; - })(); - if (rowIndex === undefined) { - return undefined; - } - - const columnId = (() => { - const delta = getHorizontalDelta(direction); - if (delta === 0) { - return activeCell.columnId; - } - if (activeCell.columnId) { - const index = columnOrder.findIndex((id) => id === activeCell.columnId); - const targetId = columnOrder[index + delta]; - return targetId; - } - return ''; - })(); - if (!columnId) { - return undefined; - } - return { rowIndex, columnId }; - } - - handleKeyEventsOnActiveCell(key: KeyboardEvent): 'moved' | undefined { - const direction = getDirection(key); - if (!direction) { - return undefined; - } - let moved = false; - this.activeCell.update((activeCell) => { - if (!activeCell) { - return undefined; - } - const adjacentCell = this.getAdjacentCell(activeCell, direction); - if (adjacentCell) { - moved = true; - return adjacentCell; - } - return activeCell; - }); - - return moved ? 'moved' : undefined; - } - - activateFirstCellInSelectedColumn() { - const activeCell = get(this.activeCell); - if (activeCell) { - this.activateCell({ rowIndex: 0 }, { id: activeCell.columnId }); - } - } - - /** - * This method does not utilize class properties inorder - * to make it reactive during component usage. - * - * It is placed within the class inorder to make use of - * class types - */ - getSelectedUniqueColumnsId( - selectedCells: ImmutableSet, - columnsSelectedWhenTheTableIsEmpty: ImmutableSet, - ): Column['id'][] { - const setOfUniqueColumnIds = new Set([ - ...[...selectedCells].map(getSelectedColumnId), - ...columnsSelectedWhenTheTableIsEmpty, - ]); - return Array.from(setOfUniqueColumnIds); - } - - getSelectedUniqueRowsId( - selectedCells: ImmutableSet, - ): Row['rowIndex'][] { - const setOfUniqueRowIndex = new Set([ - ...[...selectedCells].map(getSelectedRowIndex), - ]); - return Array.from(setOfUniqueRowIndex); - } - - destroy(): void { - this.activeCellUnsubscriber(); - } -} diff --git a/mathesar_ui/src/components/sheet/index.ts b/mathesar_ui/src/components/sheet/index.ts index 0448df86aa..cae20e3d07 100644 --- a/mathesar_ui/src/components/sheet/index.ts +++ b/mathesar_ui/src/components/sheet/index.ts @@ -5,13 +5,7 @@ export { default as SheetPositionableCell } from './SheetPositionableCell.svelte export { default as SheetCellResizer } from './SheetCellResizer.svelte'; export { default as SheetVirtualRows } from './SheetVirtualRows.svelte'; export { default as SheetRow } from './SheetRow.svelte'; -export { default as LegacySheetSelection } from './LegacySheetSelection'; export { - isColumnSelected, - isRowSelected, - isCellSelected, - getSelectedRowIndex, - isCellActive, scrollBasedOnActiveCell, scrollBasedOnSelection, -} from './LegacySheetSelection'; +} from './sheetScollingUtils'; diff --git a/mathesar_ui/src/components/sheet/sheetScollingUtils.ts b/mathesar_ui/src/components/sheet/sheetScollingUtils.ts new file mode 100644 index 0000000000..130f87cefd --- /dev/null +++ b/mathesar_ui/src/components/sheet/sheetScollingUtils.ts @@ -0,0 +1,49 @@ +// TODO: Create a common utility action to handle active element based scroll +function scrollToElement(htmlElement: HTMLElement | null): void { + const activeRow = htmlElement?.parentElement; + const container = document.querySelector('[data-sheet-body-element="list"]'); + if (!container || !activeRow) { + return; + } + // Vertical scroll + if ( + activeRow.offsetTop + activeRow.clientHeight + 40 > + container.scrollTop + container.clientHeight + ) { + const offsetValue: number = + container.getBoundingClientRect().bottom - + activeRow.getBoundingClientRect().bottom - + 40; + container.scrollTop -= offsetValue; + } else if (activeRow.offsetTop - 30 < container.scrollTop) { + container.scrollTop = activeRow.offsetTop - 30; + } + + // Horizontal scroll + if ( + htmlElement.offsetLeft + activeRow.clientWidth + 30 > + container.scrollLeft + container.clientWidth + ) { + const offsetValue: number = + container.getBoundingClientRect().right - + htmlElement.getBoundingClientRect().right - + 30; + container.scrollLeft -= offsetValue; + } else if (htmlElement.offsetLeft - 30 < container.scrollLeft) { + container.scrollLeft = htmlElement.offsetLeft - 30; + } +} + +export function scrollBasedOnActiveCell(): void { + const activeCell: HTMLElement | null = document.querySelector( + '[data-sheet-element="cell"].is-active', + ); + scrollToElement(activeCell); +} + +export function scrollBasedOnSelection(): void { + const selectedCell: HTMLElement | null = document.querySelector( + '[data-sheet-element="cell"].is-selected', + ); + scrollToElement(selectedCell); +} diff --git a/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte b/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte index 77d06c808a..b13fc93274 100644 --- a/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte +++ b/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte @@ -50,7 +50,6 @@ void queryManager.update((q) => q.withBaseTable(tableEntry ? tableEntry.id : undefined), ); - queryManager.clearSelection(); linkCollapsibleOpenState = {}; } diff --git a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts index 7e69371554..63a4cb6db0 100644 --- a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts +++ b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts @@ -10,7 +10,6 @@ import type { } from '@mathesar/api/types/queries'; import { ApiMultiError } from '@mathesar/api/utils/errors'; import type { RequestStatus } from '@mathesar/api/utils/requestUtils'; -import { LegacySheetSelection } from '@mathesar/components/sheet'; import Plane from '@mathesar/components/sheet/selection/Plane'; import Series from '@mathesar/components/sheet/selection/Series'; import SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; @@ -25,7 +24,6 @@ import { processColumnMetaData, speculateColumnMetaData, type InputColumnsStoreSubstance, - type ProcessedQueryOutputColumn, type ProcessedQueryOutputColumnMap, type ProcessedQueryResultColumnMap, } from './utils'; @@ -44,12 +42,6 @@ export interface QueryRowsData { rows: QueryRow[]; } -/** @deprecated TODO_3037 remove */ -export type QuerySheetSelection = LegacySheetSelection< - QueryRow, - ProcessedQueryOutputColumn ->; - type QueryRunMode = 'queryId' | 'queryObject'; export default class QueryRunner { @@ -75,9 +67,6 @@ export default class QueryRunner { /** Keys are row ids, values are records */ selectableRowsMap: Readable>>; - /** @deprecated TODO_3037 remove */ - legacySelection: QuerySheetSelection; - selection: Writable; inspector: QueryInspector; @@ -122,21 +111,6 @@ export default class QueryRunner { ({ rows }) => new Map(rows.map((r) => [getRowSelectionId(r), r.record])), ); - this.legacySelection = new LegacySheetSelection({ - getColumns: () => [...get(this.processedColumns).values()], - getColumnOrder: () => - [...get(this.processedColumns).values()].map((column) => column.id), - getRows: () => get(this.rowsData).rows, - getMaxSelectionRowIndex: () => { - const rowLength = get(this.rowsData).rows.length; - const totalCount = get(this.rowsData).totalCount ?? 0; - const pagination = get(this.pagination); - const { offset } = pagination; - const pageSize = pagination.size; - return Math.min(pageSize, totalCount - offset, rowLength) - 1; - }, - }); - const plane = derived( [this.rowsData, this.processedColumns], ([{ rows }, columnsMap]) => { @@ -153,10 +127,6 @@ export default class QueryRunner { plane.subscribe((p) => this.selection.update((s) => s.forNewPlane(p))), ); - this.cleanupFunctions.push( - plane.subscribe((p) => this.selection.update((s) => s.forNewPlane(p))), - ); - this.inspector = new QueryInspector(this.query); } @@ -269,7 +239,6 @@ export default class QueryRunner { async setPagination(pagination: Pagination): Promise { this.pagination.set(pagination); await this.run(); - this.legacySelection.activateFirstCellInSelectedColumn(); } protected resetPagination(): void { @@ -283,7 +252,6 @@ export default class QueryRunner { } protected resetResults(): void { - this.clearSelection(); this.runPromise?.cancel(); this.resetPagination(); this.rowsData.set({ totalCount: 0, rows: [] }); @@ -294,31 +262,15 @@ export default class QueryRunner { protected async resetPaginationAndRun(): Promise { this.resetPagination(); await this.run(); - this.legacySelection.activateFirstCellInSelectedColumn(); } selectColumn(alias: QueryColumnMetaData['alias']): void { const processedColumn = get(this.processedColumns).get(alias); if (!processedColumn) { - this.legacySelection.resetSelection(); - this.legacySelection.selectAndActivateFirstCellIfExists(); - this.inspector.selectCellTab(); - return; - } - - const isSelected = - this.legacySelection.toggleColumnSelection(processedColumn); - if (isSelected) { - this.inspector.selectColumnTab(); return; } - - this.legacySelection.activateFirstCellInSelectedColumn(); - this.inspector.selectCellTab(); - } - - clearSelection(): void { - this.legacySelection.resetSelection(); + this.selection.update((s) => s.ofOneColumn(processedColumn.id)); + this.inspector.selectColumnTab(); } getQueryModel(): QueryModel { @@ -326,7 +278,6 @@ export default class QueryRunner { } destroy(): void { - this.legacySelection.destroy(); this.runPromise?.cancel(); this.cleanupFunctions.forEach((fn) => fn()); } diff --git a/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/DeleteColumnAction.svelte b/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/DeleteColumnAction.svelte index 1b1c124bdb..2beb400e82 100644 --- a/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/DeleteColumnAction.svelte +++ b/mathesar_ui/src/systems/data-explorer/exploration-inspector/column-tab/DeleteColumnAction.svelte @@ -50,7 +50,6 @@ void queryManager.update((q) => q.withoutInitialColumns(selectedColumnAliases), ); - queryManager.clearSelection(); } From 9a9dbd8da4fc69c84ac2b355388b30efe137a6d4 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 11 Sep 2023 21:27:00 -0400 Subject: [PATCH 0022/1141] Rename `isSelectedInRange` to `isSelected` --- mathesar_ui/src/components/cell-fabric/CellFabric.svelte | 4 ++-- .../cell-fabric/data-types/components/CellWrapper.svelte | 4 ++-- .../data-types/components/SteppedInputCell.svelte | 4 ++-- .../components/__tests__/SteppedInputCell.test.ts | 2 +- .../data-types/components/array/ArrayCell.svelte | 4 ++-- .../data-types/components/checkbox/CheckboxCell.svelte | 4 ++-- .../data-types/components/date-time/DateTimeCell.svelte | 4 ++-- .../components/formatted-input/FormattedInputCell.svelte | 4 ++-- .../components/linked-record/LinkedRecordCell.svelte | 4 ++-- .../components/linked-record/LinkedRecordInput.svelte | 2 +- .../data-types/components/money/MoneyCell.svelte | 4 ++-- .../data-types/components/number/NumberCell.svelte | 4 ++-- .../data-types/components/primary-key/PrimaryKeyCell.svelte | 4 ++-- .../data-types/components/select/SingleSelectCell.svelte | 4 ++-- .../data-types/components/textarea/TextAreaCell.svelte | 4 ++-- .../data-types/components/textbox/TextBoxCell.svelte | 4 ++-- .../cell-fabric/data-types/components/typeDefinitions.ts | 2 +- .../cell-fabric/data-types/components/uri/UriCell.svelte | 4 ++-- .../systems/data-explorer/result-pane/ResultRowCell.svelte | 2 +- mathesar_ui/src/systems/table-view/row/RowCell.svelte | 6 +++--- 20 files changed, 37 insertions(+), 37 deletions(-) diff --git a/mathesar_ui/src/components/cell-fabric/CellFabric.svelte b/mathesar_ui/src/components/cell-fabric/CellFabric.svelte index 8a405123da..9046a74872 100644 --- a/mathesar_ui/src/components/cell-fabric/CellFabric.svelte +++ b/mathesar_ui/src/components/cell-fabric/CellFabric.svelte @@ -21,7 +21,7 @@ | ((recordId: string, recordSummary: string) => void) | undefined = undefined; export let isActive = false; - export let isSelectedInRange = false; + export let isSelected = false; export let disabled = false; export let showAsSkeleton = false; export let horizontalAlignment: HorizontalAlignment | undefined = undefined; @@ -56,7 +56,7 @@ this={component} {...props} {isActive} - {isSelectedInRange} + {isSelected} {disabled} {isIndependentOfSheet} {horizontalAlignment} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte index a731edd395..ab639cc1d8 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte @@ -6,7 +6,7 @@ export let element: HTMLElement | undefined = undefined; export let isActive = false; - export let isSelectedInRange = false; + export let isSelected = false; export let disabled = false; export let mode: 'edit' | 'default' = 'default'; export let multiLineTruncate = false; @@ -116,7 +116,7 @@ {...$$restProps} > {#if mode !== 'edit'} - + ; export let isActive: Props['isActive']; - export let isSelectedInRange: Props['isSelectedInRange']; + export let isSelected: Props['isSelected']; export let value: Props['value']; export let disabled: Props['disabled']; export let multiLineTruncate = false; @@ -154,7 +154,7 @@ { class?: string; id?: string; diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCell.svelte index 54e9f7a556..ee81c06f06 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCell.svelte @@ -6,7 +6,7 @@ type $$Props = MoneyCellProps; export let isActive: $$Props['isActive']; - export let isSelectedInRange: $$Props['isSelectedInRange']; + export let isSelected: $$Props['isSelected']; export let value: $$Props['value']; export let disabled: $$Props['disabled']; export let searchValue: $$Props['searchValue'] = undefined; @@ -19,7 +19,7 @@ = ( export interface CellTypeProps { value: Value | null | undefined; isActive: boolean; - isSelectedInRange: boolean; + isSelected: boolean; disabled: boolean; searchValue?: unknown; isProcessing: boolean; diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte index 08bfa74608..0ff4e5c8fd 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte @@ -11,7 +11,7 @@ type $$Props = CellTypeProps; export let isActive: $$Props['isActive']; - export let isSelectedInRange: $$Props['isSelectedInRange']; + export let isSelected: $$Props['isSelected']; export let value: $$Props['value'] = undefined; export let disabled: $$Props['disabled']; export let searchValue: $$Props['searchValue'] = undefined; @@ -22,7 +22,7 @@ @@ -146,7 +146,7 @@ Date: Mon, 5 Feb 2024 20:26:58 -0500 Subject: [PATCH 0023/1141] Handle movement keys --- mathesar_ui/src/components/sheet/index.ts | 2 +- .../components/sheet/selection/Direction.ts | 22 ------- .../sheet/selection/SheetSelection.ts | 4 +- .../components/sheet/sheetKeyboardUtils.ts | 47 ++++++++++++++ ...collingUtils.ts => sheetScrollingUtils.ts} | 7 +++ mathesar_ui/src/components/sheet/utils.ts | 3 +- .../result-pane/ResultRowCell.svelte | 31 ++------- .../src/systems/table-view/row/RowCell.svelte | 26 ++------ mathesar_ui/src/utils/KeyboardShortcut.ts | 63 +++++++++++++++++++ 9 files changed, 131 insertions(+), 74 deletions(-) create mode 100644 mathesar_ui/src/components/sheet/sheetKeyboardUtils.ts rename mathesar_ui/src/components/sheet/{sheetScollingUtils.ts => sheetScrollingUtils.ts} (93%) create mode 100644 mathesar_ui/src/utils/KeyboardShortcut.ts diff --git a/mathesar_ui/src/components/sheet/index.ts b/mathesar_ui/src/components/sheet/index.ts index cae20e3d07..b0a1df34f4 100644 --- a/mathesar_ui/src/components/sheet/index.ts +++ b/mathesar_ui/src/components/sheet/index.ts @@ -8,4 +8,4 @@ export { default as SheetRow } from './SheetRow.svelte'; export { scrollBasedOnActiveCell, scrollBasedOnSelection, -} from './sheetScollingUtils'; +} from './sheetScrollingUtils'; diff --git a/mathesar_ui/src/components/sheet/selection/Direction.ts b/mathesar_ui/src/components/sheet/selection/Direction.ts index 2693dd9000..b53704f378 100644 --- a/mathesar_ui/src/components/sheet/selection/Direction.ts +++ b/mathesar_ui/src/components/sheet/selection/Direction.ts @@ -5,28 +5,6 @@ export enum Direction { Right = 'right', } -export function getDirection(event: KeyboardEvent): Direction | undefined { - const { key } = event; - const shift = event.shiftKey; - switch (true) { - case shift && key === 'Tab': - return Direction.Left; - case shift: - return undefined; - case key === 'ArrowUp': - return Direction.Up; - case key === 'ArrowDown': - return Direction.Down; - case key === 'ArrowLeft': - return Direction.Left; - case key === 'ArrowRight': - case key === 'Tab': - return Direction.Right; - default: - return undefined; - } -} - export function getColumnOffset(direction: Direction): number { switch (direction) { case Direction.Left: diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index 53318a7af3..7c11c67476 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -480,6 +480,8 @@ export default class SheetSelection { * is simpler. */ resized(direction: Direction): SheetSelection { - throw new Error('Not implemented'); + // TODO + console.log('Sheet selection resizing is not yet implemented'); + return this; } } diff --git a/mathesar_ui/src/components/sheet/sheetKeyboardUtils.ts b/mathesar_ui/src/components/sheet/sheetKeyboardUtils.ts new file mode 100644 index 0000000000..f2035c747b --- /dev/null +++ b/mathesar_ui/src/components/sheet/sheetKeyboardUtils.ts @@ -0,0 +1,47 @@ +import type { Writable } from 'svelte/store'; + +import { KeyboardShortcut } from '@mathesar/utils/KeyboardShortcut'; +import { Direction } from './selection/Direction'; +import type SheetSelection from './selection/SheetSelection'; +import { autoScroll } from './sheetScrollingUtils'; + +function move(selection: Writable, direction: Direction) { + selection.update((s) => s.collapsedAndMoved(direction)); + void autoScroll(); +} + +function resize(selection: Writable, direction: Direction) { + selection.update((s) => s.resized(direction)); + void autoScroll(); +} + +function key(...args: Parameters) { + return KeyboardShortcut.fromKey(...args).toString(); +} + +const shortcutMapData: [string, (s: Writable) => void][] = [ + [key('ArrowUp'), (s) => move(s, Direction.Up)], + [key('ArrowDown'), (s) => move(s, Direction.Down)], + [key('ArrowLeft'), (s) => move(s, Direction.Left)], + [key('ArrowRight'), (s) => move(s, Direction.Right)], + [key('Tab'), (s) => move(s, Direction.Right)], + [key('Tab', ['Shift']), (s) => move(s, Direction.Left)], + [key('ArrowUp', ['Shift']), (s) => resize(s, Direction.Up)], + [key('ArrowDown', ['Shift']), (s) => resize(s, Direction.Down)], + [key('ArrowLeft', ['Shift']), (s) => resize(s, Direction.Left)], + [key('ArrowRight', ['Shift']), (s) => resize(s, Direction.Right)], +]; + +const shortcutMap = new Map(shortcutMapData); + +export function handleKeyboardEventOnCell( + event: KeyboardEvent, + selection: Writable, +): void { + const shortcut = KeyboardShortcut.fromKeyboardEvent(event); + const action = shortcutMap.get(shortcut.toString()); + if (action) { + event.preventDefault(); + action(selection); + } +} diff --git a/mathesar_ui/src/components/sheet/sheetScollingUtils.ts b/mathesar_ui/src/components/sheet/sheetScrollingUtils.ts similarity index 93% rename from mathesar_ui/src/components/sheet/sheetScollingUtils.ts rename to mathesar_ui/src/components/sheet/sheetScrollingUtils.ts index 130f87cefd..1155a4ec7b 100644 --- a/mathesar_ui/src/components/sheet/sheetScollingUtils.ts +++ b/mathesar_ui/src/components/sheet/sheetScrollingUtils.ts @@ -1,3 +1,5 @@ +import { tick } from 'svelte'; + // TODO: Create a common utility action to handle active element based scroll function scrollToElement(htmlElement: HTMLElement | null): void { const activeRow = htmlElement?.parentElement; @@ -47,3 +49,8 @@ export function scrollBasedOnSelection(): void { ); scrollToElement(selectedCell); } + +export async function autoScroll() { + await tick(); + scrollBasedOnActiveCell(); +} diff --git a/mathesar_ui/src/components/sheet/utils.ts b/mathesar_ui/src/components/sheet/utils.ts index c05015864a..4f3915d8d0 100644 --- a/mathesar_ui/src/components/sheet/utils.ts +++ b/mathesar_ui/src/components/sheet/utils.ts @@ -1,5 +1,6 @@ -import { setContext, getContext } from 'svelte'; +import { getContext, setContext } from 'svelte'; import type { Readable } from 'svelte/store'; + import type { ImmutableMap } from '@mathesar-component-library/types'; export interface ColumnPosition { diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte index 50620a2fbd..6010b66a53 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte @@ -1,15 +1,12 @@ @@ -55,7 +31,8 @@ value={row?.record[column.id]} showAsSkeleton={recordRunState === 'processing'} disabled={true} - on:movementKeyDown={moveThroughCells} + on:movementKeyDown={({ detail }) => + handleKeyboardEventOnCell(detail.originalEvent, selection)} on:activate={() => { // // TODO_3037 // if (row) { diff --git a/mathesar_ui/src/systems/table-view/row/RowCell.svelte b/mathesar_ui/src/systems/table-view/row/RowCell.svelte index 168a2bd019..61ce76d658 100644 --- a/mathesar_ui/src/systems/table-view/row/RowCell.svelte +++ b/mathesar_ui/src/systems/table-view/row/RowCell.svelte @@ -1,7 +1,7 @@
diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/SteppedInputCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/SteppedInputCell.svelte index 4544741629..98f1bdcd02 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/SteppedInputCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/SteppedInputCell.svelte @@ -143,12 +143,6 @@ resetEditMode(); } - function handleMouseDown() { - if (!isActive) { - dispatch('activate'); - } - } - onMount(initLastSavedValue); @@ -159,7 +153,6 @@ bind:element={cellRef} on:dblclick={setModeToEdit} on:keydown={handleKeyDown} - on:mousedown={handleMouseDown} on:mouseenter mode={isEditMode ? 'edit' : 'default'} {multiLineTruncate} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/array/ArrayCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/array/ArrayCell.svelte index 1f25b9fc64..66121cef76 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/array/ArrayCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/array/ArrayCell.svelte @@ -34,12 +34,6 @@ break; } } - - function handleMouseDown() { - if (!isActive) { - dispatch('activate'); - } - } {#if isDefinedNonNullable(value)} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/checkbox/CheckboxCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/checkbox/CheckboxCell.svelte index dd31115e7c..a6a469c9d3 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/checkbox/CheckboxCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/checkbox/CheckboxCell.svelte @@ -58,12 +58,15 @@ cellRef?.focus(); } - function handleMouseDown() { - if (!isActive) { - isFirstActivated = true; - dispatch('activate'); - } - } + // // TODO_3037: test checkbox cell thoroughly. The `isFirstActivated` + // // variable is no longer getting set. We need to figure out what to do to + // // handle this. + // function handleMouseDown() { + // if (!isActive) { + // isFirstActivated = true; + // dispatch('activate'); + // } + // } {#if value === undefined} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/date-time/DateTimeCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/date-time/DateTimeCell.svelte index e3a8144221..b50c154cab 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/date-time/DateTimeCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/date-time/DateTimeCell.svelte @@ -31,7 +31,6 @@ let:handleInputKeydown formatValue={formatForDisplay} on:movementKeyDown - on:activate on:mouseenter on:update > diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/formatted-input/FormattedInputCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/formatted-input/FormattedInputCell.svelte index 1738bc4170..42df86179c 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/formatted-input/FormattedInputCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/formatted-input/FormattedInputCell.svelte @@ -32,7 +32,6 @@ let:handleInputKeydown formatValue={formatForDisplay} on:movementKeyDown - on:activate on:mouseenter on:update > diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/LinkedRecordCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/LinkedRecordCell.svelte index 2759539ba1..63de26a0ca 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/LinkedRecordCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/LinkedRecordCell.svelte @@ -80,10 +80,11 @@ } } - function handleMouseDown() { - wasActiveBeforeClick = isActive; - dispatch('activate'); - } + // // TODO_3037: test and see if we need `wasActiveBeforeClick` + // function handleMouseDown() { + // wasActiveBeforeClick = isActive; + // dispatch('activate'); + // } function handleClick() { if (wasActiveBeforeClick) { @@ -99,7 +100,6 @@ {isIndependentOfSheet} on:mouseenter on:keydown={handleWrapperKeyDown} - on:mousedown={handleMouseDown} on:click={handleClick} on:dblclick={launchRecordSelector} hasPadding={false} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCell.svelte index ee81c06f06..6d271559d6 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCell.svelte @@ -30,7 +30,6 @@ let:handleInputKeydown on:movementKeyDown on:mouseenter - on:activate on:update > diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/primary-key/PrimaryKeyCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/primary-key/PrimaryKeyCell.svelte index bb51bc0540..dc01938b39 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/primary-key/PrimaryKeyCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/primary-key/PrimaryKeyCell.svelte @@ -28,10 +28,6 @@ e.stopPropagation(); } - function handleValueMouseDown() { - dispatch('activate'); - } - function handleKeyDown(e: KeyboardEvent) { switch (e.key) { case 'Tab': @@ -63,7 +59,7 @@ class="primary-key-cell" class:is-independent-of-sheet={isIndependentOfSheet} > - + {#if value === undefined} {:else} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/select/SingleSelectCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/select/SingleSelectCell.svelte index 9e2c64e214..886f86c83f 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/select/SingleSelectCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/select/SingleSelectCell.svelte @@ -43,12 +43,13 @@ isInitiallyActivated = false; } - function handleMouseDown() { - if (!isActive) { - isInitiallyActivated = true; - dispatch('activate'); - } - } + // // TODO_3037 test an see how to fix `isInitiallyActivated` logic + // function handleMouseDown() { + // if (!isActive) { + // isInitiallyActivated = true; + // dispatch('activate'); + // } + // } function handleKeyDown( e: KeyboardEvent, @@ -116,7 +117,6 @@ {isSelected} {disabled} {isIndependentOfSheet} - on:mousedown={handleMouseDown} on:mouseenter on:click={() => checkAndToggle(api)} on:keydown={(e) => handleKeyDown(e, api, isOpen)} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/textarea/TextAreaCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/textarea/TextAreaCell.svelte index 5e948ad64e..9ebc8b88a1 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/textarea/TextAreaCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/textarea/TextAreaCell.svelte @@ -42,7 +42,6 @@ let:handleInputBlur let:handleInputKeydown on:movementKeyDown - on:activate on:mouseenter on:update > diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/textbox/TextBoxCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/textbox/TextBoxCell.svelte index 1cecffa707..f560933a6f 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/textbox/TextBoxCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/textbox/TextBoxCell.svelte @@ -28,7 +28,6 @@ let:handleInputBlur let:handleInputKeydown on:movementKeyDown - on:activate on:mouseenter on:update > diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte index 0ff4e5c8fd..2e919190ec 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte @@ -30,7 +30,6 @@ let:handleInputBlur let:handleInputKeydown on:movementKeyDown - on:activate on:mouseenter on:update > diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte index 6010b66a53..75d97df662 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte @@ -33,27 +33,6 @@ disabled={true} on:movementKeyDown={({ detail }) => handleKeyboardEventOnCell(detail.originalEvent, selection)} - on:activate={() => { - // // TODO_3037 - // if (row) { - // selection.activateCell(row, processedQueryColumn); - // inspector.selectCellTab(); - // } - }} - on:onSelectionStart={() => { - // // TODO_3037 - // if (row) { - // selection.onStartSelection(row, processedQueryColumn); - // } - }} - on:onMouseEnterCellWhileSelection={() => { - // // TODO_3037 - // if (row) { - // // This enables the click + drag to - // // select multiple cells - // selection.onMouseEnterCellWhileSelection(row, processedQueryColumn); - // } - }} /> {/if}
diff --git a/mathesar_ui/src/systems/table-view/row/RowCell.svelte b/mathesar_ui/src/systems/table-view/row/RowCell.svelte index 61ce76d658..0316d7d465 100644 --- a/mathesar_ui/src/systems/table-view/row/RowCell.svelte +++ b/mathesar_ui/src/systems/table-view/row/RowCell.svelte @@ -1,5 +1,4 @@ - export let isStatic = false; - export let isControlCell = false; +
+ +
- $: htmlAttributes = { - 'data-sheet-element': 'cell', - 'data-cell-static': isStatic ? true : undefined, - 'data-cell-control': isControlCell ? true : undefined, - }; - + diff --git a/mathesar_ui/src/components/sheet/SheetHeader.svelte b/mathesar_ui/src/components/sheet/SheetHeader.svelte index 7b5e7ac7d8..37203e70f8 100644 --- a/mathesar_ui/src/components/sheet/SheetHeader.svelte +++ b/mathesar_ui/src/components/sheet/SheetHeader.svelte @@ -41,7 +41,7 @@
@@ -50,7 +50,7 @@
diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte index 05841970a8..1a6c80302e 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte @@ -37,7 +37,6 @@ pagination, runState, selection, - inspector, } = queryHandler); $: ({ initial_columns } = $query); $: clipboardHandler = new SheetClipboardHandler({ @@ -87,15 +86,9 @@ > -
- - + /> {#each columnList as processedQueryColumn (processedQueryColumn.id)} -
- - {$pagination.offset + item.index + 1} -
+ + {$pagination.offset + item.index + 1}
{#each columnList as processedQueryColumn (processedQueryColumn.id)} @@ -137,7 +125,6 @@ column={processedQueryColumn} {recordRunState} {selection} - {inspector} /> {/each}
@@ -234,7 +221,7 @@ :global(.column-name-wrapper.selected) { background: var(--slate-200) !important; } - :global([data-sheet-element='cell'].selected) { + :global([data-sheet-element='data-cell'].selected) { background: var(--slate-100); } } diff --git a/mathesar_ui/src/systems/table-view/TableView.svelte b/mathesar_ui/src/systems/table-view/TableView.svelte index d209587b2b..fb048a3460 100644 --- a/mathesar_ui/src/systems/table-view/TableView.svelte +++ b/mathesar_ui/src/systems/table-view/TableView.svelte @@ -94,7 +94,7 @@ // //Better document why we need id. // // const target = e.target as HTMLElement; - // if (!target.closest('[data-sheet-element="cell"')) { + // if (!target.closest('[data-sheet-element="data-cell"')) { // if ($activeCell) { // selection.focusCell( // // TODO make sure to use getRowSelectionId instead of rowIndex diff --git a/mathesar_ui/src/systems/table-view/header/Header.svelte b/mathesar_ui/src/systems/table-view/header/Header.svelte index 01d7fa1d3b..0dd73689d9 100644 --- a/mathesar_ui/src/systems/table-view/header/Header.svelte +++ b/mathesar_ui/src/systems/table-view/header/Header.svelte @@ -94,80 +94,46 @@ - + dropColumn()} on:dragover={(e) => e.preventDefault()} locationOfFirstDraggedColumn={0} columnLocation={-1} - > -
- + /> {#each [...$processedColumns] as [columnId, processedColumn] (columnId)} {@const isSelected = $selection.columnIds.has(String(columnId))} - + +
-
- dragColumn()} - column={processedColumn} - {selection} + dragColumn()} + column={processedColumn} + {selection} + > + dropColumn(processedColumn)} + on:dragover={(e) => e.preventDefault()} + {locationOfFirstDraggedColumn} + columnLocation={columnOrderString.indexOf(columnId.toString())} + {isSelected} > - dropColumn(processedColumn)} - on:dragover={(e) => e.preventDefault()} - {locationOfFirstDraggedColumn} - columnLocation={columnOrderString.indexOf(columnId.toString())} - {isSelected} - > - { - // // TODO_3037 - // selection.onColumnSelectionStart(processedColumn) - }} - on:mouseenter={() => { - // // TODO_3037 - // selection.onMouseEnterColumnHeaderWhileSelection( - // processedColumn, - // ) - }} - /> - - - - - - -
+ + + + + + +
{/each} {#if hasNewColumnButton} - -
- -
+ + {/if} - - diff --git a/mathesar_ui/src/systems/table-view/row/GroupHeader.svelte b/mathesar_ui/src/systems/table-view/row/GroupHeader.svelte index 06a739656c..a71cdfd10f 100644 --- a/mathesar_ui/src/systems/table-view/row/GroupHeader.svelte +++ b/mathesar_ui/src/systems/table-view/row/GroupHeader.svelte @@ -30,13 +30,8 @@ ); - -
+ +
{#each columnIds as columnId, index (columnId)} @@ -60,7 +55,6 @@ From 44dd519b5815e96ecfe774b86f9a8c4354cf2143 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 7 Feb 2024 09:05:21 -0500 Subject: [PATCH 0026/1141] Use dedicated components for sheet cell types --- .../components/cell-fabric/CellFabric.svelte | 5 + mathesar_ui/src/components/sheet/README.md | 25 +++-- .../src/components/sheet/SheetCell.svelte | 104 ------------------ .../cells/SheetColumnCreationCell.svelte | 29 +++++ .../sheet/cells/SheetColumnHeaderCell.svelte | 35 ++++++ .../sheet/cells/SheetDataCell.svelte | 45 ++++++++ .../sheet/cells/SheetOriginCell.svelte | 23 ++++ .../{ => cells}/SheetPositionableCell.svelte | 4 +- .../sheet/cells/SheetRowHeaderCell.svelte | 34 ++++++ .../src/components/sheet/cells/index.ts | 6 + .../components/sheet/cells/sheetCellUtils.ts | 13 +++ mathesar_ui/src/components/sheet/index.ts | 4 +- .../components/sheet/sheetScrollingUtils.ts | 12 +- mathesar_ui/src/components/sheet/types.ts | 39 ------- .../import/preview/ImportPreviewSheet.svelte | 11 +- .../result-pane/ResultHeaderCell.svelte | 18 +-- .../result-pane/ResultRowCell.svelte | 10 +- .../data-explorer/result-pane/Results.svelte | 18 ++- .../systems/table-view/header/Header.svelte | 22 ++-- .../src/systems/table-view/row/Row.svelte | 9 +- .../src/systems/table-view/row/RowCell.svelte | 10 +- 21 files changed, 262 insertions(+), 214 deletions(-) delete mode 100644 mathesar_ui/src/components/sheet/SheetCell.svelte create mode 100644 mathesar_ui/src/components/sheet/cells/SheetColumnCreationCell.svelte create mode 100644 mathesar_ui/src/components/sheet/cells/SheetColumnHeaderCell.svelte create mode 100644 mathesar_ui/src/components/sheet/cells/SheetDataCell.svelte create mode 100644 mathesar_ui/src/components/sheet/cells/SheetOriginCell.svelte rename mathesar_ui/src/components/sheet/{ => cells}/SheetPositionableCell.svelte (90%) create mode 100644 mathesar_ui/src/components/sheet/cells/SheetRowHeaderCell.svelte create mode 100644 mathesar_ui/src/components/sheet/cells/index.ts create mode 100644 mathesar_ui/src/components/sheet/cells/sheetCellUtils.ts diff --git a/mathesar_ui/src/components/cell-fabric/CellFabric.svelte b/mathesar_ui/src/components/cell-fabric/CellFabric.svelte index a5b8a90e60..70f52b80ab 100644 --- a/mathesar_ui/src/components/cell-fabric/CellFabric.svelte +++ b/mathesar_ui/src/components/cell-fabric/CellFabric.svelte @@ -23,6 +23,7 @@ export let isIndependentOfSheet = false; export let showTruncationPopover = false; export let canViewLinkedEntities = true; + export let lightText = false; $: ({ cellComponentAndProps } = columnFabric); $: ({ component } = cellComponentAndProps); @@ -34,6 +35,7 @@ data-column-identifier={columnFabric.id} class:show-as-skeleton={showAsSkeleton} class:is-independent={isIndependentOfSheet} + class:light-text={lightText} > diff --git a/mathesar_ui/src/components/sheet/README.md b/mathesar_ui/src/components/sheet/README.md index 711552011a..ec8340cbe1 100644 --- a/mathesar_ui/src/components/sheet/README.md +++ b/mathesar_ui/src/components/sheet/README.md @@ -1,9 +1,16 @@ -This is a placeholder directory for the Sheet component which is meant to be implemented for use with Tables, Views and Data Explorer. - -- This would be a lower-order component. -- It would encapsulate: - - Rows - - Cells -- It will _not_ include column headers. -- It would be a pure component and will _not_ hardcode requests from within the component. - - They would be made on the parent components utilizing the Sheet component using callbacks, events and/or by exposing a Sheet API through slot. +# Sheet + +The Sheet components help us display things in a spreadsheet-like format for the table page and the data explorer. + +## `data-sheet-element` values + +We use the `data-sheet-element` HTML attribute for CSS styling and JS functionality. The values are: + +- `header-row`: The top most row of the sheet. It contains the column header cells. +- `data-row`: Used for the remaining rows, including (for now) non-standard ones like grouping headers which don't contain data. +- `positionable-cell`: Cells that span multiple columns or are taken out of regular flow e.g. "New records" message, grouping headers, etc. +- `origin-cell`: The cell in the top-left corner of the sheet. +- `column-header-cell`: Contains the column names. +- `new-column-cell`: Contains the `+` button for adding a new column. +- `row-header-cell`: Contains the row numbers. +- `data-cell`: Regular data cells. diff --git a/mathesar_ui/src/components/sheet/SheetCell.svelte b/mathesar_ui/src/components/sheet/SheetCell.svelte deleted file mode 100644 index 9c9ad390b8..0000000000 --- a/mathesar_ui/src/components/sheet/SheetCell.svelte +++ /dev/null @@ -1,104 +0,0 @@ - - -
- -
- - diff --git a/mathesar_ui/src/components/sheet/cells/SheetColumnCreationCell.svelte b/mathesar_ui/src/components/sheet/cells/SheetColumnCreationCell.svelte new file mode 100644 index 0000000000..482f2bf5f0 --- /dev/null +++ b/mathesar_ui/src/components/sheet/cells/SheetColumnCreationCell.svelte @@ -0,0 +1,29 @@ + + +
+ +
+ + diff --git a/mathesar_ui/src/components/sheet/cells/SheetColumnHeaderCell.svelte b/mathesar_ui/src/components/sheet/cells/SheetColumnHeaderCell.svelte new file mode 100644 index 0000000000..0feba78d76 --- /dev/null +++ b/mathesar_ui/src/components/sheet/cells/SheetColumnHeaderCell.svelte @@ -0,0 +1,35 @@ + + +
+ +
+ + diff --git a/mathesar_ui/src/components/sheet/cells/SheetDataCell.svelte b/mathesar_ui/src/components/sheet/cells/SheetDataCell.svelte new file mode 100644 index 0000000000..93feef613d --- /dev/null +++ b/mathesar_ui/src/components/sheet/cells/SheetDataCell.svelte @@ -0,0 +1,45 @@ + + +
+ +
+ + diff --git a/mathesar_ui/src/components/sheet/cells/SheetOriginCell.svelte b/mathesar_ui/src/components/sheet/cells/SheetOriginCell.svelte new file mode 100644 index 0000000000..8f1096d02d --- /dev/null +++ b/mathesar_ui/src/components/sheet/cells/SheetOriginCell.svelte @@ -0,0 +1,23 @@ + + +
+ +
+ + diff --git a/mathesar_ui/src/components/sheet/SheetPositionableCell.svelte b/mathesar_ui/src/components/sheet/cells/SheetPositionableCell.svelte similarity index 90% rename from mathesar_ui/src/components/sheet/SheetPositionableCell.svelte rename to mathesar_ui/src/components/sheet/cells/SheetPositionableCell.svelte index 8c731acda8..27b7806069 100644 --- a/mathesar_ui/src/components/sheet/SheetPositionableCell.svelte +++ b/mathesar_ui/src/components/sheet/cells/SheetPositionableCell.svelte @@ -1,6 +1,6 @@ + +
+ +
+ + diff --git a/mathesar_ui/src/components/sheet/cells/index.ts b/mathesar_ui/src/components/sheet/cells/index.ts new file mode 100644 index 0000000000..4dc087562c --- /dev/null +++ b/mathesar_ui/src/components/sheet/cells/index.ts @@ -0,0 +1,6 @@ +export { default as SheetColumnHeaderCell } from './SheetColumnHeaderCell.svelte'; +export { default as SheetDataCell } from './SheetDataCell.svelte'; +export { default as SheetColumnCreationCell } from './SheetColumnCreationCell.svelte'; +export { default as SheetOriginCell } from './SheetOriginCell.svelte'; +export { default as SheetPositionableCell } from './SheetPositionableCell.svelte'; +export { default as SheetRowHeaderCell } from './SheetRowHeaderCell.svelte'; diff --git a/mathesar_ui/src/components/sheet/cells/sheetCellUtils.ts b/mathesar_ui/src/components/sheet/cells/sheetCellUtils.ts new file mode 100644 index 0000000000..c1312a0adc --- /dev/null +++ b/mathesar_ui/src/components/sheet/cells/sheetCellUtils.ts @@ -0,0 +1,13 @@ +import { derived, type Readable } from 'svelte/store'; +import { getSheetContext } from '../utils'; + +export function getSheetCellStyle( + columnIdentifierKey: ColumnIdentifierKey, +): Readable { + const { stores } = getSheetContext(); + const { columnStyleMap } = stores; + return derived(columnStyleMap, (map) => { + const columnPosition = map.get(columnIdentifierKey); + return columnPosition?.styleString; + }); +} diff --git a/mathesar_ui/src/components/sheet/index.ts b/mathesar_ui/src/components/sheet/index.ts index b0a1df34f4..e541c3f139 100644 --- a/mathesar_ui/src/components/sheet/index.ts +++ b/mathesar_ui/src/components/sheet/index.ts @@ -1,7 +1,5 @@ export { default as Sheet } from './Sheet.svelte'; export { default as SheetHeader } from './SheetHeader.svelte'; -export { default as SheetCell } from './SheetCell.svelte'; -export { default as SheetPositionableCell } from './SheetPositionableCell.svelte'; export { default as SheetCellResizer } from './SheetCellResizer.svelte'; export { default as SheetVirtualRows } from './SheetVirtualRows.svelte'; export { default as SheetRow } from './SheetRow.svelte'; @@ -9,3 +7,5 @@ export { scrollBasedOnActiveCell, scrollBasedOnSelection, } from './sheetScrollingUtils'; + +export * from './cells'; diff --git a/mathesar_ui/src/components/sheet/sheetScrollingUtils.ts b/mathesar_ui/src/components/sheet/sheetScrollingUtils.ts index 16bede15b2..b59cbc2bdf 100644 --- a/mathesar_ui/src/components/sheet/sheetScrollingUtils.ts +++ b/mathesar_ui/src/components/sheet/sheetScrollingUtils.ts @@ -37,17 +37,13 @@ function scrollToElement(htmlElement: HTMLElement | null): void { } export function scrollBasedOnActiveCell(): void { - const activeCell: HTMLElement | null = document.querySelector( - '[data-sheet-element="data-cell"].is-active', - ); - scrollToElement(activeCell); + const cell = document.querySelector('[data-cell-active]'); + scrollToElement(cell); } export function scrollBasedOnSelection(): void { - const selectedCell: HTMLElement | null = document.querySelector( - '[data-sheet-element="data-cell"].is-selected', - ); - scrollToElement(selectedCell); + const cell = document.querySelector('[data-cell-selected]'); + scrollToElement(cell); } export async function autoScroll() { diff --git a/mathesar_ui/src/components/sheet/types.ts b/mathesar_ui/src/components/sheet/types.ts index bbc97588a2..b69d3b24d3 100644 --- a/mathesar_ui/src/components/sheet/types.ts +++ b/mathesar_ui/src/components/sheet/types.ts @@ -4,42 +4,3 @@ export interface SheetVirtualRowsApi { scrollToPosition: (vScrollOffset: number, hScrollOffset: number) => void; recalculateHeightsAfterIndex: (index: number) => void; } - -/** - * These are the different kinds of cells that we can have within a sheet. - * - * - `origin-cell`: The cell in the top-left corner of the sheet. - * - * - `column-header-cell`: Contains the column names. - * - * - `new-column-cell`: Contains the `+` button for adding a new column. - * - * - `row-header-cell`: Contains the row numbers. - * - * - `data-cell`: Regular data cells. - */ -export type SheetCellType = - | 'origin-cell' - | 'column-header-cell' - | 'new-column-cell' - | 'row-header-cell' - | 'data-cell'; - -/** - * These are values used for the `data-sheet-element` attribute on the sheet - * elements. - * - * - `header-row`: The top most row of the sheet. It contains the column header - * cells. - * - * - `data-row`: Used for the remaining rows, including (for now) weird ones - * like grouping headers and such. - * - * - `positionable-cell`: Cells that span multiple columns or are taken out of - * regular flow e.g. "New records" message, grouping headers, etc. - */ -export type SheetElement = - | 'header-row' - | 'data-row' - | 'positionable-cell' - | SheetCellType; diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewSheet.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewSheet.svelte index 3ffd200dad..a775f29ab2 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewSheet.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewSheet.svelte @@ -3,8 +3,9 @@ import CellFabric from '@mathesar/components/cell-fabric/CellFabric.svelte'; import { Sheet, - SheetCell, SheetCellResizer, + SheetColumnHeaderCell, + SheetDataCell, SheetHeader, SheetRow, } from '@mathesar/components/sheet'; @@ -25,7 +26,7 @@ c.id}> {#each columns as column (column.id)} - + - + {/each} {#each records as record (record)} @@ -48,14 +49,14 @@ >
{#each columns as column (column)} - + - + {/each}
diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/ResultHeaderCell.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/ResultHeaderCell.svelte index edc9ccbfc0..ba9855e119 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/ResultHeaderCell.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/ResultHeaderCell.svelte @@ -1,6 +1,9 @@ - + - + diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte index eda7fda532..502c388eee 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte @@ -3,7 +3,7 @@ import type { RequestStatus } from '@mathesar/api/utils/requestUtils'; import CellFabric from '@mathesar/components/cell-fabric/CellFabric.svelte'; - import { SheetCell } from '@mathesar/components/sheet'; + import { SheetDataCell } from '@mathesar/components/sheet'; import { makeCellId } from '@mathesar/components/sheet/cellIds'; import type SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; import { handleKeyboardEventOnCell } from '@mathesar/components/sheet/sheetKeyboardUtils'; @@ -19,7 +19,11 @@ $: isActive = cellId === $selection.activeCellId; - + {#if row || recordRunState === 'processing'} {/if} - + diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte index 1a6c80302e..bb5c7b3d9a 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte @@ -1,15 +1,17 @@ - + dropColumn()} on:dragover={(e) => e.preventDefault()} locationOfFirstDraggedColumn={0} columnLocation={-1} /> - + {#each [...$processedColumns] as [columnId, processedColumn] (columnId)} {@const isSelected = $selection.columnIds.has(String(columnId))} - +
-
+ {/each} {#if hasNewColumnButton} - + - + {/if}
diff --git a/mathesar_ui/src/systems/table-view/row/Row.svelte b/mathesar_ui/src/systems/table-view/row/Row.svelte index 0e08c59f76..cddf79d67c 100644 --- a/mathesar_ui/src/systems/table-view/row/Row.svelte +++ b/mathesar_ui/src/systems/table-view/row/Row.svelte @@ -1,6 +1,6 @@ - @@ -135,6 +134,7 @@ handleKeyboardEventOnCell(detail.originalEvent, selection)} on:update={valueUpdated} horizontalAlignment={column.primary_key ? 'left' : undefined} + lightText={hasError || isProcessing} /> {#if canEditTableRecords || showLinkedRecordHyperLink} @@ -171,4 +171,4 @@ {#if errors.length} {/if} - + From f5448669a3348120184ab63c39ab4cc4bb21f0d2 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 7 Feb 2024 20:35:43 -0500 Subject: [PATCH 0027/1141] Add match utility function for TS pattern matching --- .../utils/__tests__/patternMatching.test.ts | 20 +++++++++++++++++++ mathesar_ui/src/utils/patternMatching.ts | 14 +++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 mathesar_ui/src/utils/__tests__/patternMatching.test.ts create mode 100644 mathesar_ui/src/utils/patternMatching.ts diff --git a/mathesar_ui/src/utils/__tests__/patternMatching.test.ts b/mathesar_ui/src/utils/__tests__/patternMatching.test.ts new file mode 100644 index 0000000000..b00d25a7fc --- /dev/null +++ b/mathesar_ui/src/utils/__tests__/patternMatching.test.ts @@ -0,0 +1,20 @@ +import { match } from '../patternMatching'; + +test('match', () => { + type Shape = + | { kind: 'circle'; radius: number } + | { kind: 'square'; x: number } + | { kind: 'triangle'; x: number; y: number }; + + function area(shape: Shape) { + return match(shape, 'kind', { + circle: ({ radius }) => Math.PI * radius ** 2, + square: ({ x }) => x ** 2, + triangle: ({ x, y }) => (x * y) / 2, + }); + } + + expect(area({ kind: 'circle', radius: 5 })).toBe(78.53981633974483); + expect(area({ kind: 'square', x: 5 })).toBe(25); + expect(area({ kind: 'triangle', x: 5, y: 6 })).toBe(15); +}); diff --git a/mathesar_ui/src/utils/patternMatching.ts b/mathesar_ui/src/utils/patternMatching.ts new file mode 100644 index 0000000000..9f349df973 --- /dev/null +++ b/mathesar_ui/src/utils/patternMatching.ts @@ -0,0 +1,14 @@ +export function match< + /** Discriminant Property name */ + P extends string, + /** The other stuff in the type (besides the discriminant property) */ + O, + /** The union valued type */ + V extends Record & O, + /** The match arms */ + C extends { [K in V[P]]: (v: Extract>) => unknown }, +>(value: V, property: P, cases: C) { + return cases[value[property]]( + value as Extract>, + ) as ReturnType; +} From e8e5e347c1220caf8702224362176db0d8794d8b Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 7 Feb 2024 20:57:06 -0500 Subject: [PATCH 0028/1141] Begin scaffolding to handle drag selection --- mathesar_ui/src/components/sheet/Sheet.svelte | 23 +++++- .../src/components/sheet/cells/index.ts | 1 + .../sheet/selection/SheetSelection.ts | 48 ++++++++++++- .../src/components/sheet/selection/index.ts | 1 + .../sheet/selection/selectionUtils.ts | 71 +++++++++++++++++++ .../src/systems/table-view/TableView.svelte | 13 ++-- 6 files changed, 145 insertions(+), 12 deletions(-) create mode 100644 mathesar_ui/src/components/sheet/selection/index.ts create mode 100644 mathesar_ui/src/components/sheet/selection/selectionUtils.ts diff --git a/mathesar_ui/src/components/sheet/Sheet.svelte b/mathesar_ui/src/components/sheet/Sheet.svelte index 9e01daf36f..92b876876e 100644 --- a/mathesar_ui/src/components/sheet/Sheet.svelte +++ b/mathesar_ui/src/components/sheet/Sheet.svelte @@ -1,12 +1,14 @@
{#if columns.length} diff --git a/mathesar_ui/src/components/sheet/cells/index.ts b/mathesar_ui/src/components/sheet/cells/index.ts index 4dc087562c..2fd6716ac2 100644 --- a/mathesar_ui/src/components/sheet/cells/index.ts +++ b/mathesar_ui/src/components/sheet/cells/index.ts @@ -4,3 +4,4 @@ export { default as SheetColumnCreationCell } from './SheetColumnCreationCell.sv export { default as SheetOriginCell } from './SheetOriginCell.svelte'; export { default as SheetPositionableCell } from './SheetPositionableCell.svelte'; export { default as SheetRowHeaderCell } from './SheetRowHeaderCell.svelte'; +export * from './sheetCellUtils'; diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index 7c11c67476..38dde70bda 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -1,10 +1,12 @@ import { execPipe, filter, first, map } from 'iter-tools'; import { ImmutableSet } from '@mathesar/component-library'; +import { match } from '@mathesar/utils/patternMatching'; import { assertExhaustive } from '@mathesar/utils/typeUtils'; import { makeCells, parseCellId } from '../cellIds'; import { Direction, getColumnOffset } from './Direction'; import Plane from './Plane'; +import type { SheetCellDetails } from './selectionUtils'; /** * - `'dataCells'` means that the selection contains data cells. This is by far @@ -280,7 +282,7 @@ export default class SheetSelection { * selection is made only of data cells, and will never include cells in the * placeholder row, even if a user drags to select a cell in it. */ - ofCellRange(cellIdA: string, cellIdB: string): SheetSelection { + ofDataCellRange(cellIdA: string, cellIdB: string): SheetSelection { return this.withBasis( basisFromDataCells( this.plane.dataCellsInFlexibleCellRange(cellIdA, cellIdB), @@ -288,6 +290,46 @@ export default class SheetSelection { ); } + ofSheetCellRange( + cellA: SheetCellDetails, + cellB: SheetCellDetails, + ): SheetSelection { + // TODO_3037 finish implementation + return match(cellA, 'type', { + 'data-cell': (a) => + match(cellB, 'type', { + 'data-cell': (b) => this.ofDataCellRange(a.cellId, b.cellId), + 'column-header-cell': (b) => { + throw new Error('Not implemented'); + }, + 'row-header-cell': (b) => { + throw new Error('Not implemented'); + }, + }), + 'column-header-cell': (a) => + match(cellB, 'type', { + 'data-cell': (b) => { + throw new Error('Not implemented'); + }, + 'column-header-cell': (b) => + this.ofColumnRange(a.columnId, b.columnId), + 'row-header-cell': (b) => { + throw new Error('Not implemented'); + }, + }), + 'row-header-cell': (a) => + match(cellB, 'type', { + 'data-cell': (b) => { + throw new Error('Not implemented'); + }, + 'column-header-cell': (b) => { + throw new Error('Not implemented'); + }, + 'row-header-cell': (b) => this.ofRowRange(a.rowId, b.rowId), + }), + }); + } + /** * @returns a new selection formed from one cell within the data rows or the * placeholder row. @@ -391,8 +433,8 @@ export default class SheetSelection { * by the active cell (also the first cell selected when dragging) and the * provided cell. */ - drawnToCell(cellId: string): SheetSelection { - return this.ofCellRange(this.activeCellId ?? cellId, cellId); + drawnToDataCell(cellId: string): SheetSelection { + return this.ofDataCellRange(this.activeCellId ?? cellId, cellId); } /** diff --git a/mathesar_ui/src/components/sheet/selection/index.ts b/mathesar_ui/src/components/sheet/selection/index.ts new file mode 100644 index 0000000000..ec81a55c52 --- /dev/null +++ b/mathesar_ui/src/components/sheet/selection/index.ts @@ -0,0 +1 @@ +export * from './selectionUtils'; diff --git a/mathesar_ui/src/components/sheet/selection/selectionUtils.ts b/mathesar_ui/src/components/sheet/selection/selectionUtils.ts new file mode 100644 index 0000000000..32558f0884 --- /dev/null +++ b/mathesar_ui/src/components/sheet/selection/selectionUtils.ts @@ -0,0 +1,71 @@ +import type { Writable } from 'svelte/store'; + +import type SheetSelection from './SheetSelection'; + +export type SheetCellDetails = + | { type: 'data-cell'; cellId: string } + | { type: 'column-header-cell'; columnId: string } + | { type: 'row-header-cell'; rowId: string }; + +export function findContainingSheetCell( + element: HTMLElement, +): SheetCellDetails | undefined { + const containingElement = element.closest('[data-sheet-element]'); + if (!containingElement) return undefined; + + const elementType = containingElement.getAttribute('data-sheet-element'); + if (!elementType) return undefined; + + if (elementType === 'data-cell') { + const cellId = containingElement.getAttribute('data-cell-selection-id'); + if (!cellId) return undefined; + return { type: 'data-cell', cellId }; + } + + if (elementType === 'column-header-cell') { + const columnId = containingElement.getAttribute('data-column-identifier'); + if (!columnId) return undefined; + return { type: 'column-header-cell', columnId }; + } + + if (elementType === 'row-header-cell') { + const rowId = containingElement.getAttribute('data-row-selection-id'); + if (!rowId) return undefined; + return { type: 'row-header-cell', rowId }; + } + + return undefined; +} + +export function beginSelection({ + selection, + sheetElement, + startingCell, +}: { + startingCell: SheetCellDetails; + selection: Writable; + sheetElement: HTMLElement; +}) { + let previousTarget: HTMLElement | undefined; + + function drawToCell(endingCell: SheetCellDetails) { + selection.update((s) => s.ofSheetCellRange(startingCell, endingCell)); + } + + function drawToPoint(e: MouseEvent) { + const target = e.target as HTMLElement; + if (target === previousTarget) return; // For performance + const cell = findContainingSheetCell(target); + if (!cell) return; + drawToCell(cell); + } + + function finish() { + sheetElement.removeEventListener('mousemove', drawToPoint); + window.removeEventListener('mouseup', finish); + } + + drawToCell(startingCell); + sheetElement.addEventListener('mousemove', drawToPoint); + window.addEventListener('mouseup', finish); +} diff --git a/mathesar_ui/src/systems/table-view/TableView.svelte b/mathesar_ui/src/systems/table-view/TableView.svelte index fb048a3460..3f3dad227e 100644 --- a/mathesar_ui/src/systems/table-view/TableView.svelte +++ b/mathesar_ui/src/systems/table-view/TableView.svelte @@ -127,16 +127,17 @@
{#if $processedColumns.size} entry.column.id} - {usesVirtualList} - {columnWidths} {clipboardHandler} - hasBorder={sheetHasBorder} - restrictWidthToRowWidth={!usesVirtualList} + {columnWidths} + {selection} + {usesVirtualList} bind:horizontalScrollOffset={$horizontalScrollOffset} bind:scrollOffset={$scrollOffset} + columns={sheetColumns} + getColumnIdentifier={(entry) => entry.column.id} + hasBorder={sheetHasBorder} hasPaddingRight + restrictWidthToRowWidth={!usesVirtualList} >
From 4f2a253cc8c83a1bd231ae2158b31fbb4c1c4097 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Thu, 8 Feb 2024 20:10:42 -0500 Subject: [PATCH 0029/1141] Document match function --- mathesar_ui/src/utils/patternMatching.ts | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/mathesar_ui/src/utils/patternMatching.ts b/mathesar_ui/src/utils/patternMatching.ts index 9f349df973..8b69cbdaed 100644 --- a/mathesar_ui/src/utils/patternMatching.ts +++ b/mathesar_ui/src/utils/patternMatching.ts @@ -1,3 +1,31 @@ +/** + * Performs branching logic by exhaustively pattern-matching all variants of a + * TypeScript discriminated union. + * + * @param value The value to match + * @param property The name of the discriminant property + * @param cases An object representing the match arms. It should have one entry + * per variant of the union. The key should be the value of the discriminant + * property for that variant, and the value should be a function that takes the + * value of that variant and returns the result of the match arm. + * + * @example + * + * ```ts + * type Shape = + * | { kind: 'circle'; radius: number } + * | { kind: 'square'; x: number } + * | { kind: 'triangle'; x: number; y: number }; + * + * function area(shape: Shape) { + * return match(shape, 'kind', { + * circle: ({ radius }) => Math.PI * radius ** 2, + * square: ({ x }) => x ** 2, + * triangle: ({ x, y }) => (x * y) / 2, + * }); + * } + * ``` + */ export function match< /** Discriminant Property name */ P extends string, From ab4aca0c895d61b26462c91ff9e02de765beaf98 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Thu, 8 Feb 2024 20:12:35 -0500 Subject: [PATCH 0030/1141] Finish implementation of ofSheetCellRange method --- .../sheet/selection/SheetSelection.ts | 71 +++++++++---------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index 38dde70bda..f177794704 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -3,7 +3,7 @@ import { execPipe, filter, first, map } from 'iter-tools'; import { ImmutableSet } from '@mathesar/component-library'; import { match } from '@mathesar/utils/patternMatching'; import { assertExhaustive } from '@mathesar/utils/typeUtils'; -import { makeCells, parseCellId } from '../cellIds'; +import { makeCellId, makeCells, parseCellId } from '../cellIds'; import { Direction, getColumnOffset } from './Direction'; import Plane from './Plane'; import type { SheetCellDetails } from './selectionUtils'; @@ -291,42 +291,41 @@ export default class SheetSelection { } ofSheetCellRange( - cellA: SheetCellDetails, - cellB: SheetCellDetails, + startingCell: SheetCellDetails, + endingCell: SheetCellDetails, ): SheetSelection { - // TODO_3037 finish implementation - return match(cellA, 'type', { - 'data-cell': (a) => - match(cellB, 'type', { - 'data-cell': (b) => this.ofDataCellRange(a.cellId, b.cellId), - 'column-header-cell': (b) => { - throw new Error('Not implemented'); - }, - 'row-header-cell': (b) => { - throw new Error('Not implemented'); - }, - }), - 'column-header-cell': (a) => - match(cellB, 'type', { - 'data-cell': (b) => { - throw new Error('Not implemented'); - }, - 'column-header-cell': (b) => - this.ofColumnRange(a.columnId, b.columnId), - 'row-header-cell': (b) => { - throw new Error('Not implemented'); - }, - }), - 'row-header-cell': (a) => - match(cellB, 'type', { - 'data-cell': (b) => { - throw new Error('Not implemented'); - }, - 'column-header-cell': (b) => { - throw new Error('Not implemented'); - }, - 'row-header-cell': (b) => this.ofRowRange(a.rowId, b.rowId), - }), + // Nullish coalescing is safe here since we know we'll have a first row and + // first column in the cases where we're selecting things. + const firstRow = () => this.plane.rowIds.first ?? ''; + const firstColumn = () => this.plane.columnIds.first ?? ''; + + return match(startingCell, 'type', { + 'data-cell': ({ cellId: startingCellId }) => { + const endingCellId = match(endingCell, 'type', { + 'data-cell': (b) => b.cellId, + 'column-header-cell': (b) => makeCellId(firstRow(), b.columnId), + 'row-header-cell': (b) => makeCellId(b.rowId, firstColumn()), + }); + return this.ofDataCellRange(startingCellId, endingCellId); + }, + + 'column-header-cell': ({ columnId: startingColumnId }) => { + const endingColumnId = match(endingCell, 'type', { + 'data-cell': (b) => parseCellId(b.cellId).columnId, + 'column-header-cell': (b) => b.columnId, + 'row-header-cell': () => firstColumn(), + }); + return this.ofColumnRange(startingColumnId, endingColumnId); + }, + + 'row-header-cell': ({ rowId: startingRowId }) => { + const endingRowId = match(endingCell, 'type', { + 'data-cell': (b) => parseCellId(b.cellId).rowId, + 'column-header-cell': () => firstRow(), + 'row-header-cell': (b) => b.rowId, + }); + return this.ofRowRange(startingRowId, endingRowId); + }, }); } From 4e83d8d3a2814165985220d4e257a60df1129d1f Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Sun, 11 Feb 2024 21:07:34 -0500 Subject: [PATCH 0031/1141] Lift isSelected prop way up Move from being used in CellWrapper to being used in SheetDataCell. --- .../components/cell-fabric/CellFabric.svelte | 2 -- .../data-types/components/CellWrapper.svelte | 2 -- .../components/SteppedInputCell.svelte | 2 -- .../components/array/ArrayCell.svelte | 2 -- .../components/checkbox/CheckboxCell.svelte | 2 -- .../components/date-time/DateTimeCell.svelte | 2 -- .../formatted-input/FormattedInputCell.svelte | 2 -- .../linked-record/LinkedRecordCell.svelte | 2 -- .../components/money/MoneyCell.svelte | 2 -- .../components/number/NumberCell.svelte | 2 -- .../primary-key/PrimaryKeyCell.svelte | 2 -- .../components/select/SingleSelectCell.svelte | 2 -- .../components/textarea/TextAreaCell.svelte | 2 -- .../components/textbox/TextBoxCell.svelte | 2 -- .../data-types/components/typeDefinitions.ts | 1 - .../data-types/components/uri/UriCell.svelte | 2 -- .../sheet/cells/SheetDataCell.svelte | 25 ++++++++++++++++--- .../result-pane/ResultRowCell.svelte | 5 ++-- .../src/systems/table-view/row/RowCell.svelte | 6 ++--- 19 files changed, 26 insertions(+), 41 deletions(-) diff --git a/mathesar_ui/src/components/cell-fabric/CellFabric.svelte b/mathesar_ui/src/components/cell-fabric/CellFabric.svelte index 70f52b80ab..41d9390317 100644 --- a/mathesar_ui/src/components/cell-fabric/CellFabric.svelte +++ b/mathesar_ui/src/components/cell-fabric/CellFabric.svelte @@ -14,7 +14,6 @@ | ((recordId: string, recordSummary: string) => void) | undefined = undefined; export let isActive = false; - export let isSelected = false; export let disabled = false; export let showAsSkeleton = false; export let horizontalAlignment: HorizontalAlignment | undefined = undefined; @@ -42,7 +41,6 @@ {...props} {columnFabric} {isActive} - {isSelected} {disabled} {isIndependentOfSheet} {horizontalAlignment} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte index 8fe7f4bf02..3f4c9ca1d1 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte @@ -6,7 +6,6 @@ export let element: HTMLElement | undefined = undefined; export let isActive = false; - export let isSelected = false; export let disabled = false; export let mode: 'edit' | 'default' = 'default'; export let multiLineTruncate = false; @@ -116,7 +115,6 @@ {...$$restProps} > {#if mode !== 'edit'} - ; export let isActive: Props['isActive']; - export let isSelected: Props['isSelected']; export let value: Props['value']; export let disabled: Props['disabled']; export let multiLineTruncate = false; @@ -148,7 +147,6 @@ = ( export interface CellTypeProps { value: Value | null | undefined; isActive: boolean; - isSelected: boolean; disabled: boolean; searchValue?: unknown; isProcessing: boolean; diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte index 2e919190ec..23782f18cb 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/uri/UriCell.svelte @@ -11,7 +11,6 @@ type $$Props = CellTypeProps; export let isActive: $$Props['isActive']; - export let isSelected: $$Props['isSelected']; export let value: $$Props['value'] = undefined; export let disabled: $$Props['disabled']; export let searchValue: $$Props['searchValue'] = undefined; @@ -22,7 +21,6 @@ + import type SheetSelection from '../selection/SheetSelection'; + + import CellBackground from '@mathesar/components/CellBackground.svelte'; import { getSheetCellStyle } from './sheetCellUtils'; type SheetColumnIdentifierKey = $$Generic; export let columnIdentifierKey: SheetColumnIdentifierKey; export let cellSelectionId: string | undefined = undefined; - export let isActive = false; - export let isSelected = false; + export let selection: SheetSelection | undefined = undefined; $: style = getSheetCellStyle(columnIdentifierKey); + $: ({ isActive, isSelected, hasSelectionBackground } = (() => { + if (!selection || !cellSelectionId) + return { + isActive: false, + isSelected: false, + hasSelectionBackground: false, + }; + const isSelected = selection.cellIds.has(cellSelectionId); + return { + isActive: selection.activeCellId === cellSelectionId, + isSelected: selection.cellIds.has(cellSelectionId), + hasSelectionBackground: isSelected && selection.cellIds.size > 1, + }; + })());
- + {#if hasSelectionBackground} + + {/if} +
diff --git a/mathesar_ui/src/systems/table-view/table-inspector/cell/CellMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/cell/CellMode.svelte index cb7a00bfd4..883a431fc0 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/cell/CellMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/cell/CellMode.svelte @@ -8,73 +8,26 @@ * changes to both files as necessary. */ import { _ } from 'svelte-i18n'; + import { getTabularDataStoreFromContext } from '@mathesar/stores/table-data'; - import CellFabric from '@mathesar/components/cell-fabric/CellFabric.svelte'; + import ActiveCellValue from './ActiveCellValue.svelte'; const tabularData = getTabularDataStoreFromContext(); - $: ({ selection, recordsData, processedColumns } = $tabularData); - $: ({ activeCell } = selection); - $: ({ recordSummaries } = recordsData); - $: cell = $activeCell; - $: selectedCellValue = (() => { - if (cell) { - const rows = recordsData.getRecordRows(); - if (rows[cell.rowIndex]) { - return rows[cell.rowIndex].record[cell.columnId]; - } - } - return undefined; - })(); - $: column = (() => { - if (cell) { - const processedColumn = $processedColumns.get(Number(cell.columnId)); - if (processedColumn) { - return processedColumn; - } - } - return undefined; - })(); - $: recordSummary = - column && - $recordSummaries.get(String(column.id))?.get(String(selectedCellValue)); + $: ({ selection } = $tabularData); + $: ({ activeCellId } = $selection);
- {#if selectedCellValue !== undefined} -
-
{$_('content')}
-
- {#if column} - - {/if} -
-
+ {#if activeCellId} + {:else} {$_('select_cell_view_properties')} {/if}
- From 2608b222c954c361defb99b8bb6992e5cda4318d Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Sun, 18 Feb 2024 11:19:49 -0500 Subject: [PATCH 0033/1141] Remove table-view deps from CellInspector code --- .../inspector/cell/CellInspector.svelte | 69 +++++++++++++++++++ .../inspector/cell/cellInspectorUtils.ts | 16 +++++ .../cell/ActiveCellValue.svelte | 52 -------------- .../table-inspector/cell/CellMode.svelte | 37 +++------- .../table-inspector/cell/cellModeUtils.ts | 35 ++++++++++ 5 files changed, 131 insertions(+), 78 deletions(-) create mode 100644 mathesar_ui/src/components/inspector/cell/CellInspector.svelte create mode 100644 mathesar_ui/src/components/inspector/cell/cellInspectorUtils.ts delete mode 100644 mathesar_ui/src/systems/table-view/table-inspector/cell/ActiveCellValue.svelte create mode 100644 mathesar_ui/src/systems/table-view/table-inspector/cell/cellModeUtils.ts diff --git a/mathesar_ui/src/components/inspector/cell/CellInspector.svelte b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte new file mode 100644 index 0000000000..f01da7a677 --- /dev/null +++ b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte @@ -0,0 +1,69 @@ + + +
+ {#if activeCellData} + {@const { column, value, recordSummary } = activeCellData} +
+
{$_('content')}
+
+ {#if column} + + {/if} +
+
+ {:else} + {$_('select_cell_view_properties')} + {/if} +
+ + diff --git a/mathesar_ui/src/components/inspector/cell/cellInspectorUtils.ts b/mathesar_ui/src/components/inspector/cell/cellInspectorUtils.ts new file mode 100644 index 0000000000..b493f92069 --- /dev/null +++ b/mathesar_ui/src/components/inspector/cell/cellInspectorUtils.ts @@ -0,0 +1,16 @@ +import type { CellColumnFabric } from '@mathesar/components/cell-fabric/types'; + +interface ActiveCellData { + column: CellColumnFabric; + value: unknown; + recordSummary?: string; +} + +interface SelectionData { + cellCount: number; +} + +export interface SelectedCellData { + activeCellData?: ActiveCellData; + selectionData: SelectionData; +} diff --git a/mathesar_ui/src/systems/table-view/table-inspector/cell/ActiveCellValue.svelte b/mathesar_ui/src/systems/table-view/table-inspector/cell/ActiveCellValue.svelte deleted file mode 100644 index 368514e70f..0000000000 --- a/mathesar_ui/src/systems/table-view/table-inspector/cell/ActiveCellValue.svelte +++ /dev/null @@ -1,52 +0,0 @@ - - -
-
{$_('content')}
-
- {#if column} - - {/if} -
-
- - diff --git a/mathesar_ui/src/systems/table-view/table-inspector/cell/CellMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/cell/CellMode.svelte index 883a431fc0..dcdf6b3073 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/cell/CellMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/cell/CellMode.svelte @@ -1,33 +1,18 @@ -
- {#if activeCellId} - - {:else} - {$_('select_cell_view_properties')} - {/if} -
- - + diff --git a/mathesar_ui/src/systems/table-view/table-inspector/cell/cellModeUtils.ts b/mathesar_ui/src/systems/table-view/table-inspector/cell/cellModeUtils.ts new file mode 100644 index 0000000000..870348eb0e --- /dev/null +++ b/mathesar_ui/src/systems/table-view/table-inspector/cell/cellModeUtils.ts @@ -0,0 +1,35 @@ +import type { SelectedCellData } from '@mathesar/components/inspector/cell/cellInspectorUtils'; +import { parseCellId } from '@mathesar/components/sheet/cellIds'; +import type SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; +import type { + ProcessedColumns, + RecordSummariesForSheet, +} from '@mathesar/stores/table-data'; + +export function getSelectedCellData( + selection: SheetSelection, + selectableRowsMap: Map>, + processedColumns: ProcessedColumns, + recordSummaries: RecordSummariesForSheet, +): SelectedCellData { + const { activeCellId } = selection; + const selectionData = { + cellCount: selection.cellIds.size, + }; + if (activeCellId === undefined) { + return { selectionData }; + } + const { rowId, columnId } = parseCellId(activeCellId); + const record = selectableRowsMap.get(rowId) ?? {}; + const value = record[columnId]; + const column = processedColumns.get(Number(columnId)); + const recordSummary = recordSummaries.get(columnId)?.get(String(value)); + return { + activeCellData: column && { + column, + value, + recordSummary, + }, + selectionData, + }; +} From ad9dd5eb78fb0596d5dd9b84413c8f9534a94acb Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Sun, 18 Feb 2024 12:56:15 -0500 Subject: [PATCH 0034/1141] Pass columnIdentifierKey via prop not import This is necessary because the table page and Data Explorer use different values (and types) here. --- .../sheet/cells/SheetColumnCreationCell.svelte | 7 +++++-- .../components/sheet/cells/SheetOriginCell.svelte | 7 +++++-- .../components/sheet/cells/SheetRowHeaderCell.svelte | 6 ++++-- .../systems/data-explorer/result-pane/Results.svelte | 7 +++++-- .../src/systems/table-view/header/Header.svelte | 12 ++++++++---- mathesar_ui/src/systems/table-view/row/Row.svelte | 5 ++++- 6 files changed, 31 insertions(+), 13 deletions(-) diff --git a/mathesar_ui/src/components/sheet/cells/SheetColumnCreationCell.svelte b/mathesar_ui/src/components/sheet/cells/SheetColumnCreationCell.svelte index 482f2bf5f0..9ca66fbe19 100644 --- a/mathesar_ui/src/components/sheet/cells/SheetColumnCreationCell.svelte +++ b/mathesar_ui/src/components/sheet/cells/SheetColumnCreationCell.svelte @@ -1,8 +1,11 @@
diff --git a/mathesar_ui/src/components/sheet/cells/SheetOriginCell.svelte b/mathesar_ui/src/components/sheet/cells/SheetOriginCell.svelte index 8f1096d02d..7af9d9748b 100644 --- a/mathesar_ui/src/components/sheet/cells/SheetOriginCell.svelte +++ b/mathesar_ui/src/components/sheet/cells/SheetOriginCell.svelte @@ -1,8 +1,11 @@
diff --git a/mathesar_ui/src/components/sheet/cells/SheetRowHeaderCell.svelte b/mathesar_ui/src/components/sheet/cells/SheetRowHeaderCell.svelte index b0df3b2a0a..bdf05fecf1 100644 --- a/mathesar_ui/src/components/sheet/cells/SheetRowHeaderCell.svelte +++ b/mathesar_ui/src/components/sheet/cells/SheetRowHeaderCell.svelte @@ -1,10 +1,12 @@
- + {#each columnList as processedQueryColumn (processedQueryColumn.id)} - + {$pagination.offset + item.index + 1} diff --git a/mathesar_ui/src/systems/table-view/header/Header.svelte b/mathesar_ui/src/systems/table-view/header/Header.svelte index df3c19d110..ad28034c9f 100644 --- a/mathesar_ui/src/systems/table-view/header/Header.svelte +++ b/mathesar_ui/src/systems/table-view/header/Header.svelte @@ -5,13 +5,17 @@ import { ContextMenu } from '@mathesar/component-library'; import { SheetCellResizer, + SheetColumnCreationCell, SheetColumnHeaderCell, SheetHeader, - SheetColumnCreationCell, } from '@mathesar/components/sheet'; import SheetOriginCell from '@mathesar/components/sheet/cells/SheetOriginCell.svelte'; import type { ProcessedColumn } from '@mathesar/stores/table-data'; - import { getTabularDataStoreFromContext } from '@mathesar/stores/table-data'; + import { + ID_ADD_NEW_COLUMN, + ID_ROW_CONTROL_COLUMN, + getTabularDataStoreFromContext, + } from '@mathesar/stores/table-data'; import { saveColumnOrder } from '@mathesar/stores/tables'; import { Draggable, Droppable } from './drag-and-drop'; import ColumnHeaderContextMenu from './header-cell/ColumnHeaderContextMenu.svelte'; @@ -92,7 +96,7 @@ - + dropColumn()} on:dragover={(e) => e.preventDefault()} @@ -130,7 +134,7 @@ {/each} {#if hasNewColumnButton} - + {/if} diff --git a/mathesar_ui/src/systems/table-view/row/Row.svelte b/mathesar_ui/src/systems/table-view/row/Row.svelte index cddf79d67c..dc2f228071 100644 --- a/mathesar_ui/src/systems/table-view/row/Row.svelte +++ b/mathesar_ui/src/systems/table-view/row/Row.svelte @@ -70,7 +70,10 @@ style="--cell-height:{rowHeightPx - 1}px;{styleString}" on:mousedown={checkAndCreateEmptyRow} > - + {#if rowHasRecord(row)} Date: Sun, 18 Feb 2024 13:16:38 -0500 Subject: [PATCH 0035/1141] Use common CellInspector component within DE --- .../inspector/cell/CellInspector.svelte | 8 --- .../exploration-inspector/CellTab.svelte | 67 ------------------- .../ExplorationInspector.svelte | 2 +- .../exploration-inspector/cell/CellTab.svelte | 16 +++++ .../cell/cellTabUtils.ts | 25 +++++++ 5 files changed, 42 insertions(+), 76 deletions(-) delete mode 100644 mathesar_ui/src/systems/data-explorer/exploration-inspector/CellTab.svelte create mode 100644 mathesar_ui/src/systems/data-explorer/exploration-inspector/cell/CellTab.svelte create mode 100644 mathesar_ui/src/systems/data-explorer/exploration-inspector/cell/cellTabUtils.ts diff --git a/mathesar_ui/src/components/inspector/cell/CellInspector.svelte b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte index f01da7a677..0e6c3e8ad0 100644 --- a/mathesar_ui/src/components/inspector/cell/CellInspector.svelte +++ b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte @@ -1,12 +1,4 @@ - -
- {#if cellValue !== undefined} -
-
{$_('content')}
-
- {#if column} - - {/if} -
-
- {:else} - {$_('select_cell_view_properties')} - {/if} -
- - diff --git a/mathesar_ui/src/systems/data-explorer/exploration-inspector/ExplorationInspector.svelte b/mathesar_ui/src/systems/data-explorer/exploration-inspector/ExplorationInspector.svelte index 6ef0dd00e8..9c33275db2 100644 --- a/mathesar_ui/src/systems/data-explorer/exploration-inspector/ExplorationInspector.svelte +++ b/mathesar_ui/src/systems/data-explorer/exploration-inspector/ExplorationInspector.svelte @@ -4,7 +4,7 @@ import type QueryManager from '../QueryManager'; import ExplorationTab from './ExplorationTab.svelte'; import ColumnTab from './column-tab/ColumnTab.svelte'; - import CellTab from './CellTab.svelte'; + import CellTab from './cell/CellTab.svelte'; export let queryHandler: QueryRunner | QueryManager; export let canEditMetadata: boolean; diff --git a/mathesar_ui/src/systems/data-explorer/exploration-inspector/cell/CellTab.svelte b/mathesar_ui/src/systems/data-explorer/exploration-inspector/cell/CellTab.svelte new file mode 100644 index 0000000000..8f57fa461b --- /dev/null +++ b/mathesar_ui/src/systems/data-explorer/exploration-inspector/cell/CellTab.svelte @@ -0,0 +1,16 @@ + + + diff --git a/mathesar_ui/src/systems/data-explorer/exploration-inspector/cell/cellTabUtils.ts b/mathesar_ui/src/systems/data-explorer/exploration-inspector/cell/cellTabUtils.ts new file mode 100644 index 0000000000..479a659136 --- /dev/null +++ b/mathesar_ui/src/systems/data-explorer/exploration-inspector/cell/cellTabUtils.ts @@ -0,0 +1,25 @@ +import type { SelectedCellData } from '@mathesar/components/inspector/cell/cellInspectorUtils'; +import { parseCellId } from '@mathesar/components/sheet/cellIds'; +import type SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; +import { getRowSelectionId, type QueryRow } from '../../QueryRunner'; +import type { ProcessedQueryOutputColumnMap } from '../../utils'; + +export function getSelectedCellData( + selection: SheetSelection, + rows: QueryRow[], + processedColumns: ProcessedQueryOutputColumnMap, +): SelectedCellData { + const { activeCellId } = selection; + const selectionData = { cellCount: selection.cellIds.size }; + const fallback = { selectionData }; + if (!activeCellId) return fallback; + const { rowId, columnId } = parseCellId(activeCellId); + // TODO: Usage of `find` is not ideal for perf here. Would be nice to store + // rows in a map for faster lookup. + const row = rows.find((r) => getRowSelectionId(r) === rowId); + if (!row) return fallback; + const value = row.record[columnId]; + const column = processedColumns.get(columnId); + const activeCellData = column && { column, value }; + return { activeCellData, selectionData }; +} From 63b6dd90a0c274e0f37a402a7252784bfb76db84 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Sun, 18 Feb 2024 13:44:11 -0500 Subject: [PATCH 0036/1141] Pass selection into data explorer sheet --- mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte index 323a92279c..a1786e55df 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte @@ -85,6 +85,7 @@ {columnWidths} {clipboardHandler} usesVirtualList + {selection} > From af35b896346e65b6690a332919535316fd1a2d42 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Sun, 18 Feb 2024 13:46:48 -0500 Subject: [PATCH 0037/1141] Re-enable copying data explorer cells --- .../src/systems/data-explorer/result-pane/Results.svelte | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte index a1786e55df..450825a910 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte @@ -46,10 +46,8 @@ rowsMap: get(selectableRowsMap), columnsMap: get(processedColumns), recordSummaries: new ImmutableMap(), - // TODO_3037 - selectedRowIds: new ImmutableSet(), - // TODO_3037 - selectedColumnIds: new ImmutableSet(), + selectedRowIds: get(selection).rowIds, + selectedColumnIds: get(selection).columnIds, }), showToastInfo: toast.info, }); From 748134dcb3932cf39b88950b51304ef6d77c1658 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Sun, 18 Feb 2024 14:27:45 -0500 Subject: [PATCH 0038/1141] Reinstate row header selection in table page --- mathesar_ui/src/systems/table-view/row/Row.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mathesar_ui/src/systems/table-view/row/Row.svelte b/mathesar_ui/src/systems/table-view/row/Row.svelte index dc2f228071..cacc964df4 100644 --- a/mathesar_ui/src/systems/table-view/row/Row.svelte +++ b/mathesar_ui/src/systems/table-view/row/Row.svelte @@ -39,6 +39,7 @@ $: ({ pkColumn } = columnsDataStore); $: primaryKeyColumnId = $pkColumn?.id; $: rowKey = getRowKey(row, primaryKeyColumnId); + $: rowSelectionId = getRowSelectionId(row); $: creationStatus = $rowCreationStatus.get(rowKey)?.state; $: status = $rowStatus.get(rowKey); $: wholeRowState = status?.wholeRowState; @@ -71,7 +72,7 @@ on:mousedown={checkAndCreateEmptyRow} > {#if rowHasRecord(row)} From 7fded1c2107089df475c265d6edb77a4079f0c19 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Sun, 18 Feb 2024 14:40:13 -0500 Subject: [PATCH 0039/1141] Reinstate row header selection in data explorer --- .../result-pane/ResultRowCell.svelte | 5 ++-- .../data-explorer/result-pane/Results.svelte | 23 ++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte index 30f947fcee..30f10f7d2d 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/ResultRowCell.svelte @@ -7,15 +7,16 @@ import { makeCellId } from '@mathesar/components/sheet/cellIds'; import type SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; import { handleKeyboardEventOnCell } from '@mathesar/components/sheet/sheetKeyboardUtils'; - import { getRowSelectionId, type QueryRow } from '../QueryRunner'; + import type { QueryRow } from '../QueryRunner'; import type { ProcessedQueryOutputColumn } from '../utils'; export let column: ProcessedQueryOutputColumn; export let row: QueryRow | undefined; + export let rowSelectionId: string; export let recordRunState: RequestStatus['state'] | undefined; export let selection: Writable; - $: cellId = row && makeCellId(getRowSelectionId(row), column.id); + $: cellId = row && makeCellId(rowSelectionId, column.id);
@@ -103,14 +111,16 @@ let:items > {#each items as item (item.key)} - {#if rows[item.index] || showDummyGhostRow} + {@const row = getRow(item.index)} + {@const rowSelectionId = (row && getRowSelectionId(row)) ?? ''} + {#if row || showDummyGhostRow}
@@ -119,7 +129,8 @@ {#each columnList as processedQueryColumn (processedQueryColumn.id)} Date: Mon, 4 Mar 2024 13:07:07 -0500 Subject: [PATCH 0040/1141] Reinstate deleting records --- mathesar_ui/src/stores/table-data/records.ts | 31 ++++----- .../src/systems/data-explorer/QueryRunner.ts | 6 +- .../data-explorer/result-pane/Results.svelte | 3 +- .../src/systems/table-view/TableView.svelte | 5 +- .../table-view/row/RowContextOptions.svelte | 4 +- .../table-inspector/cell/cellModeUtils.ts | 13 ++-- .../table-inspector/record/RecordMode.svelte | 5 +- .../table-inspector/record/RowActions.svelte | 64 ++++++++----------- mathesar_ui/src/utils/iterUtils.ts | 12 ++++ 9 files changed, 79 insertions(+), 64 deletions(-) diff --git a/mathesar_ui/src/stores/table-data/records.ts b/mathesar_ui/src/stores/table-data/records.ts index da90576fd9..25dd965a4d 100644 --- a/mathesar_ui/src/stores/table-data/records.ts +++ b/mathesar_ui/src/stores/table-data/records.ts @@ -302,8 +302,8 @@ export class RecordsData { error: Writable; - /** Keys are row ids, values are records */ - selectableRowsMap: Readable>>; + /** Keys are row selection ids */ + selectableRowsMap: Readable>; private promise: CancellablePromise | undefined; @@ -364,7 +364,7 @@ export class RecordsData { [this.savedRecords, this.newRecords], ([savedRecords, newRecords]) => { const records = [...savedRecords, ...newRecords]; - return new Map(records.map((r) => [getRowSelectionId(r), r.record])); + return new Map(records.map((r) => [getRowSelectionId(r), r])); }, ); @@ -457,22 +457,23 @@ export class RecordsData { return undefined; } - async deleteSelected(selectedRowIndices: number[]): Promise { - const recordRows = this.getRecordRows(); + async deleteSelected(rowSelectionIds: Iterable): Promise { + const ids = + typeof rowSelectionIds === 'string' ? [rowSelectionIds] : rowSelectionIds; const pkColumn = get(this.columnsDataStore.pkColumn); const primaryKeysOfSavedRows: string[] = []; const identifiersOfUnsavedRows: string[] = []; - selectedRowIndices.forEach((index) => { - const row = recordRows[index]; - if (row) { - const rowKey = getRowKey(row, pkColumn?.id); - if (pkColumn?.id && isDefinedNonNullable(row.record[pkColumn?.id])) { - primaryKeysOfSavedRows.push(rowKey); - } else { - identifiersOfUnsavedRows.push(rowKey); - } + const selectableRows = get(this.selectableRowsMap); + for (const rowId of ids) { + const row = selectableRows.get(rowId); + if (!row) continue; + const rowKey = getRowKey(row, pkColumn?.id); + if (pkColumn?.id && isDefinedNonNullable(row.record[pkColumn?.id])) { + primaryKeysOfSavedRows.push(rowKey); + } else { + identifiersOfUnsavedRows.push(rowKey); } - }); + } const rowKeys = [...primaryKeysOfSavedRows, ...identifiersOfUnsavedRows]; if (rowKeys.length === 0) { diff --git a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts index 63a4cb6db0..baee9cb275 100644 --- a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts +++ b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts @@ -64,8 +64,8 @@ export default class QueryRunner { new ImmutableMap(), ); - /** Keys are row ids, values are records */ - selectableRowsMap: Readable>>; + /** Keys are row selection ids */ + selectableRowsMap: Readable>; selection: Writable; @@ -108,7 +108,7 @@ export default class QueryRunner { void this.run(); this.selectableRowsMap = derived( this.rowsData, - ({ rows }) => new Map(rows.map((r) => [getRowSelectionId(r), r.record])), + ({ rows }) => new Map(rows.map((r) => [getRowSelectionId(r), r])), ); const plane = derived( diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte index ca067b624f..6c0ba4d5c2 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/Results.svelte @@ -1,4 +1,5 @@
- {#if showOpenRecordLink} + {#if recordPageLink}
@@ -90,7 +82,7 @@ {/if} diff --git a/mathesar_ui/src/utils/iterUtils.ts b/mathesar_ui/src/utils/iterUtils.ts index 36553b2d7d..805370b416 100644 --- a/mathesar_ui/src/utils/iterUtils.ts +++ b/mathesar_ui/src/utils/iterUtils.ts @@ -32,3 +32,15 @@ export function mapExactlyOne( } return p.whenMany; } + +/** + * If the iterable contains exactly one element, returns that element. Otherwise + * returns undefined. + */ +export function takeFirstAndOnly(iterable: Iterable): T | undefined { + return mapExactlyOne(iterable, { + whenZero: undefined, + whenOne: (v) => v, + whenMany: undefined, + }); +} From 53b8317454870c960068257342b74fa5d2ddcdd7 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 4 Mar 2024 14:04:58 -0500 Subject: [PATCH 0041/1141] Fix checkbox toggle on click --- .../data-types/components/CellWrapper.svelte | 2 ++ .../components/checkbox/CheckboxCell.svelte | 30 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte index 3f4c9ca1d1..e97c8f8646 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte @@ -106,7 +106,9 @@ on:click on:dblclick on:mousedown + on:mouseup on:mouseenter + on:mouseleave on:keydown on:copy={handleCopy} on:focus={handleFocus} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/checkbox/CheckboxCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/checkbox/CheckboxCell.svelte index d85f256b1f..a5f14189f0 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/checkbox/CheckboxCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/checkbox/CheckboxCell.svelte @@ -17,7 +17,7 @@ export let isIndependentOfSheet: $$Props['isIndependentOfSheet']; let cellRef: HTMLElement; - let isFirstActivated = false; + let shouldToggleOnMouseUp = false; $: valueComparisonOutcome = compareWholeValues(searchValue, value); @@ -48,24 +48,20 @@ } } - function checkAndToggle(e: Event) { - if (!disabled && isActive && e.target === cellRef && !isFirstActivated) { + function handleMouseDown() { + shouldToggleOnMouseUp = isActive; + } + + function handleMouseLeave() { + shouldToggleOnMouseUp = false; + } + + function handleMouseUp() { + if (!disabled && isActive && shouldToggleOnMouseUp) { value = !value; dispatchUpdate(); } - isFirstActivated = false; - cellRef?.focus(); } - - // // TODO_3037: test checkbox cell thoroughly. The `isFirstActivated` - // // variable is no longer getting set. We need to figure out what to do to - // // handle this. - // function handleMouseDown() { - // if (!isActive) { - // isFirstActivated = true; - // dispatch('activate'); - // } - // } {#if value === undefined} From c1fee48fdd18b6978ff8964e1e238b975c300bac Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 4 Mar 2024 14:20:03 -0500 Subject: [PATCH 0042/1141] Fix LinkedRecordCell launch on click --- .../components/linked-record/LinkedRecordCell.svelte | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/LinkedRecordCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/LinkedRecordCell.svelte index 9f71177829..df14de41f1 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/LinkedRecordCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/LinkedRecordCell.svelte @@ -79,11 +79,10 @@ } } - // // TODO_3037: test and see if we need `wasActiveBeforeClick` - // function handleMouseDown() { - // wasActiveBeforeClick = isActive; - // dispatch('activate'); - // } + function handleMouseDown() { + wasActiveBeforeClick = isActive; + dispatch('activate'); + } function handleClick() { if (wasActiveBeforeClick) { @@ -98,6 +97,7 @@ {isIndependentOfSheet} on:mouseenter on:keydown={handleWrapperKeyDown} + on:mousedown={handleMouseDown} on:click={handleClick} on:dblclick={launchRecordSelector} hasPadding={false} From 7bee2ed008c71941bf37bbf5b591294bf6c38f38 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 4 Mar 2024 14:27:09 -0500 Subject: [PATCH 0043/1141] Fix SingleSelectCell opening --- .../components/select/SingleSelectCell.svelte | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/select/SingleSelectCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/select/SingleSelectCell.svelte index 2469775798..c9c70ea850 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/select/SingleSelectCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/select/SingleSelectCell.svelte @@ -42,13 +42,12 @@ isInitiallyActivated = false; } - // // TODO_3037 test an see how to fix `isInitiallyActivated` logic - // function handleMouseDown() { - // if (!isActive) { - // isInitiallyActivated = true; - // dispatch('activate'); - // } - // } + function handleMouseDown() { + if (!isActive) { + isInitiallyActivated = true; + dispatch('activate'); + } + } function handleKeyDown( e: KeyboardEvent, @@ -115,6 +114,7 @@ {isActive} {disabled} {isIndependentOfSheet} + on:mousedown={handleMouseDown} on:mouseenter on:click={() => checkAndToggle(api)} on:keydown={(e) => handleKeyDown(e, api, isOpen)} From 16fdd6372014a3e2797f992e176049384b9b97c1 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 4 Mar 2024 15:25:20 -0500 Subject: [PATCH 0044/1141] Implement shift+click to select range --- mathesar_ui/src/components/sheet/Sheet.svelte | 29 ++++++++++++++---- .../sheet/selection/selectionUtils.ts | 6 ++-- mathesar_ui/src/utils/pointerUtils.ts | 30 +++++++++++++++++++ 3 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 mathesar_ui/src/utils/pointerUtils.ts diff --git a/mathesar_ui/src/components/sheet/Sheet.svelte b/mathesar_ui/src/components/sheet/Sheet.svelte index 92b876876e..0bf2344765 100644 --- a/mathesar_ui/src/components/sheet/Sheet.svelte +++ b/mathesar_ui/src/components/sheet/Sheet.svelte @@ -4,7 +4,12 @@ import { ImmutableMap } from '@mathesar-component-library/types'; import type { ClipboardHandler } from '@mathesar/stores/clipboard'; import { getClipboardHandlerStoreFromContext } from '@mathesar/stores/clipboard'; - import { beginSelection, findContainingSheetCell } from './selection'; + import { getModifierKeyCombo } from '@mathesar/utils/pointerUtils'; + import { + beginSelection, + findContainingSheetCell, + type SheetCellDetails, + } from './selection'; import type SheetSelection from './selection/SheetSelection'; import { calculateColumnStyleMapAndRowWidth, @@ -122,13 +127,25 @@ function handleMouseDown(e: MouseEvent) { if (!selection) return; - // TODO_3037: - // - handle mouse events with other buttons - // - handle Shift/Alt/Ctrl key modifiers + const target = e.target as HTMLElement; - const startingCell = findContainingSheetCell(target); + const targetCell = findContainingSheetCell(target); + if (!targetCell) return; + + const startingCell: SheetCellDetails | undefined = (() => { + const modifierKeyCombo = getModifierKeyCombo(e); + if (modifierKeyCombo === '') return targetCell; + if (modifierKeyCombo === 'Shift') { + if (!$selection) return undefined; + const { activeCellId } = $selection; + if (!activeCellId) return undefined; + return { type: 'data-cell', cellId: activeCellId }; + } + return undefined; + })(); if (!startingCell) return; - beginSelection({ selection, sheetElement, startingCell }); + + beginSelection({ selection, sheetElement, startingCell, targetCell }); } diff --git a/mathesar_ui/src/components/sheet/selection/selectionUtils.ts b/mathesar_ui/src/components/sheet/selection/selectionUtils.ts index 32558f0884..f79f894597 100644 --- a/mathesar_ui/src/components/sheet/selection/selectionUtils.ts +++ b/mathesar_ui/src/components/sheet/selection/selectionUtils.ts @@ -41,10 +41,12 @@ export function beginSelection({ selection, sheetElement, startingCell, + targetCell, }: { - startingCell: SheetCellDetails; selection: Writable; sheetElement: HTMLElement; + startingCell: SheetCellDetails; + targetCell: SheetCellDetails; }) { let previousTarget: HTMLElement | undefined; @@ -65,7 +67,7 @@ export function beginSelection({ window.removeEventListener('mouseup', finish); } - drawToCell(startingCell); + drawToCell(targetCell); sheetElement.addEventListener('mousemove', drawToPoint); window.addEventListener('mouseup', finish); } diff --git a/mathesar_ui/src/utils/pointerUtils.ts b/mathesar_ui/src/utils/pointerUtils.ts new file mode 100644 index 0000000000..82db98c0a7 --- /dev/null +++ b/mathesar_ui/src/utils/pointerUtils.ts @@ -0,0 +1,30 @@ +type ModifierKeyCombo = + | '' + // 1 modifier + | 'Alt' + | 'Ctrl' + | 'Meta' + | 'Shift' + // 2 modifiers + | 'Alt+Ctrl' + | 'Alt+Meta' + | 'Alt+Shift' + | 'Ctrl+Meta' + | 'Ctrl+Shift' + | 'Meta+Shift' + // 3 modifiers + | 'Alt+Ctrl+Meta' + | 'Alt+Ctrl+Shift' + | 'Alt+Meta+Shift' + | 'Ctrl+Meta+Shift' + // 4 modifiers + | 'Alt+Ctrl+Meta+Shift'; + +export function getModifierKeyCombo(e: MouseEvent) { + return [ + ...(e.altKey ? ['Alt'] : []), + ...(e.ctrlKey ? ['Ctrl'] : []), + ...(e.metaKey ? ['Meta'] : []), + ...(e.shiftKey ? ['Shift'] : []), + ].join('+') as ModifierKeyCombo; +} From 610c138a3fd7921456749f236e60baee0f087f7d Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 4 Mar 2024 15:37:30 -0500 Subject: [PATCH 0045/1141] Order a table's processed columns sooner --- .../src/stores/table-data/tabularData.ts | 44 +++++++++---------- .../src/systems/table-view/TableView.svelte | 7 +-- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index a16572f80f..80700931a6 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -47,11 +47,8 @@ export class TabularData { columnsDataStore: ColumnsDataStore; - /** TODO_3037 eliminate `processedColumns` in favor of `orderedProcessedColumns` */ processedColumns: ProcessedColumnsStore; - orderedProcessedColumns: ProcessedColumnsStore; - constraintsDataStore: ConstraintsDataStore; recordsData: RecordsData; @@ -96,35 +93,34 @@ export class TabularData { this.recordsData, ); + this.table = props.table; + this.processedColumns = derived( [this.columnsDataStore.columns, this.constraintsDataStore], ([columns, constraintsData]) => - new Map( - columns.map((column, columnIndex) => [ - column.id, - processColumn({ - tableId: this.id, - column, - columnIndex, - constraints: constraintsData.constraints, - abstractTypeMap: props.abstractTypesMap, - hasEnhancedPrimaryKeyCell: props.hasEnhancedPrimaryKeyCell, - }), - ]), + orderProcessedColumns( + new Map( + columns.map((column, columnIndex) => [ + column.id, + processColumn({ + tableId: this.id, + column, + columnIndex, + constraints: constraintsData.constraints, + abstractTypeMap: props.abstractTypesMap, + hasEnhancedPrimaryKeyCell: props.hasEnhancedPrimaryKeyCell, + }), + ]), + ), + this.table, ), ); - this.table = props.table; - - this.orderedProcessedColumns = derived(this.processedColumns, (p) => - orderProcessedColumns(p, this.table), - ); - const plane = derived( - [this.recordsData.selectableRowsMap, this.orderedProcessedColumns], - ([selectableRowsMap, orderedProcessedColumns]) => { + [this.recordsData.selectableRowsMap, this.processedColumns], + ([selectableRowsMap, processedColumns]) => { const rowIds = new Series([...selectableRowsMap.keys()]); - const columns = [...orderedProcessedColumns.values()]; + const columns = [...processedColumns.values()]; const columnIds = new Series(columns.map((c) => String(c.id))); return new Plane(rowIds, columnIds); }, diff --git a/mathesar_ui/src/systems/table-view/TableView.svelte b/mathesar_ui/src/systems/table-view/TableView.svelte index 55f7822027..cb5052b530 100644 --- a/mathesar_ui/src/systems/table-view/TableView.svelte +++ b/mathesar_ui/src/systems/table-view/TableView.svelte @@ -22,7 +22,6 @@ import { toast } from '@mathesar/stores/toast'; import { getUserProfileStoreFromContext } from '@mathesar/stores/userProfile'; import { stringifyMapKeys } from '@mathesar/utils/collectionUtils'; - import { orderProcessedColumns } from '@mathesar/utils/tables'; import Body from './Body.svelte'; import Header from './header/Header.svelte'; import StatusPane from './StatusPane.svelte'; @@ -72,13 +71,9 @@ */ $: supportsTableInspector = context === 'page'; $: sheetColumns = (() => { - const orderedProcessedColumns = orderProcessedColumns( - $processedColumns, - table, - ); const columns = [ { column: { id: ID_ROW_CONTROL_COLUMN, name: 'ROW_CONTROL' } }, - ...orderedProcessedColumns.values(), + ...$processedColumns.values(), ]; if (hasNewColumnButton) { columns.push({ column: { id: ID_ADD_NEW_COLUMN, name: 'ADD_NEW' } }); From a8be26d442ed8e611ca25a46124b2275338754c0 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 20 Mar 2024 21:37:08 -0400 Subject: [PATCH 0046/1141] Clean up some dead code --- .../table-view/table-inspector/record/RecordMode.svelte | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte index bdc8fa956a..5f6f4967ed 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte @@ -20,11 +20,6 @@ ); $: selectedRowIds = $selection.rowIds; $: selectedRowCount = selectedRowIds.size; - - // TODO_3037 Need to calculate selectedRowIndices. This might be a deeper - // problem. Seems like we might need access to the row index here instead of - // the row identifier. - $: selectedRowIndices = [];
From 6c18123a8513493f8dc8eeecfdcf2d8a553ab5a9 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 20 Mar 2024 21:44:48 -0400 Subject: [PATCH 0047/1141] Remove TODO code comment --- .../column/column-extraction/ExtractColumnsModal.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte index b8dff19c08..7b482c9342 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte @@ -119,7 +119,6 @@ // unmounting this component. return; } - // TODO_3037 test to verify that selected columns are updated const columnIds = _columns.map((c) => String(c.id)); selection.update((s) => s.ofRowColumnIntersection(s.rowIds, columnIds)); } From 0f8cf9da299d2ab1a9fefa99d08a661c5dfd4222 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 20 Mar 2024 21:50:22 -0400 Subject: [PATCH 0048/1141] Improve code comments --- .../table-inspector/column/ColumnMode.svelte | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte index 0685493d86..337206c2dd 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte @@ -22,12 +22,18 @@ $: database = $currentDatabase; $: schema = $currentSchema; $: ({ processedColumns, selection } = $tabularData); - // TODO_3037 verify that table inspector shows selected columns $: selectedColumns = (() => { const ids = $selection.columnIds; const columns = []; for (const id of ids) { - // TODO_3037 add code comments explaining why this is necessary + // This is a little annoying that we need to parse the id as a string to + // a number. The reason is tricky. The cell selection system uses strings + // as column ids because they're more general purpose and can work with + // the string-based ids that the data explorer uses. However the table + // page stores processed columns with numeric ids. We could avoid this + // parsing by either making the selection system generic over the id type + // (which would be a pain, ergonomically), or by using string-based ids + // for columns in the table page too (which would require refactoring). const parsedId = parseInt(id, 10); const column = $processedColumns.get(parsedId); if (column !== undefined) { From 141d27ce3003138a0cf610a3da467f991f347f0e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Thu, 4 Apr 2024 13:10:54 -0400 Subject: [PATCH 0049/1141] Fix positioning of new record message --- .../components/sheet/cells/SheetPositionableCell.svelte | 8 ++++++++ .../src/systems/table-view/row/NewRecordMessage.svelte | 3 +++ 2 files changed, 11 insertions(+) diff --git a/mathesar_ui/src/components/sheet/cells/SheetPositionableCell.svelte b/mathesar_ui/src/components/sheet/cells/SheetPositionableCell.svelte index 27b7806069..67c902a4e3 100644 --- a/mathesar_ui/src/components/sheet/cells/SheetPositionableCell.svelte +++ b/mathesar_ui/src/components/sheet/cells/SheetPositionableCell.svelte @@ -30,3 +30,11 @@
+ + diff --git a/mathesar_ui/src/systems/table-view/row/NewRecordMessage.svelte b/mathesar_ui/src/systems/table-view/row/NewRecordMessage.svelte index cb73fbe93f..3d807219c0 100644 --- a/mathesar_ui/src/systems/table-view/row/NewRecordMessage.svelte +++ b/mathesar_ui/src/systems/table-view/row/NewRecordMessage.svelte @@ -17,9 +17,12 @@ From e4f1ec50f63ee5c8799869069bd195609d919798 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Thu, 4 Apr 2024 13:11:29 -0400 Subject: [PATCH 0050/1141] Only render row control cell for rows with records --- mathesar_ui/src/systems/table-view/row/Row.svelte | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mathesar_ui/src/systems/table-view/row/Row.svelte b/mathesar_ui/src/systems/table-view/row/Row.svelte index cacc964df4..f22f7d333e 100644 --- a/mathesar_ui/src/systems/table-view/row/Row.svelte +++ b/mathesar_ui/src/systems/table-view/row/Row.svelte @@ -71,11 +71,11 @@ style="--cell-height:{rowHeightPx - 1}px;{styleString}" on:mousedown={checkAndCreateEmptyRow} > - - {#if rowHasRecord(row)} + {#if rowHasRecord(row)} + - {/if} - + + {/if} {#if isHelpTextRow(row)} From 218c59bd48a4939468765cee484db725dcec8447 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Thu, 4 Apr 2024 14:44:39 -0400 Subject: [PATCH 0051/1141] Add new record row when clicking placeholder --- .../src/components/sheet/selection/Series.ts | 11 +++++++++-- .../sheet/selection/SheetSelection.ts | 17 +++++++++++++++++ .../src/stores/table-data/tabularData.ts | 5 +++++ .../src/systems/table-view/StatusPane.svelte | 6 +----- .../src/systems/table-view/row/Row.svelte | 13 ++++++------- 5 files changed, 38 insertions(+), 14 deletions(-) diff --git a/mathesar_ui/src/components/sheet/selection/Series.ts b/mathesar_ui/src/components/sheet/selection/Series.ts index b0e897aab3..c7654a8f29 100644 --- a/mathesar_ui/src/components/sheet/selection/Series.ts +++ b/mathesar_ui/src/components/sheet/selection/Series.ts @@ -53,12 +53,19 @@ export default class Series { return this.values.slice(startIndex, endIndex + 1); } + /** + * Get the value at a specific index. + */ + at(index: number): Value | undefined { + return this.values[index]; + } + get first(): Value | undefined { - return this.values[0]; + return this.at(0); } get last(): Value | undefined { - return this.values[this.values.length - 1]; + return this.at(this.values.length - 1); } has(value: Value): boolean { diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts index f177794704..c116f90b1c 100644 --- a/mathesar_ui/src/components/sheet/selection/SheetSelection.ts +++ b/mathesar_ui/src/components/sheet/selection/SheetSelection.ts @@ -232,6 +232,23 @@ export default class SheetSelection { return this.withBasis(basisFromDataCells([firstCellId])); } + /** + * @returns a new selection formed by selecting the one cell that we think + * users are most likely to want selected after choosing to add a new record. + * + * We use the last row because that's where we add new records. If there is + * only one column, then we select the first cell in that column. Otherwise, + * we select the cell in the second column (because we assume the first column + * is probably a PK column which can't accept data entry.) + */ + ofNewRecordDataEntryCell(): SheetSelection { + const rowId = this.plane.rowIds.last; + if (!rowId) return this; + const columnId = this.plane.columnIds.at(1) ?? this.plane.columnIds.first; + if (!columnId) return this; + return this.ofOneCell(makeCellId(rowId, columnId)); + } + /** * @returns a new selection with all rows selected between (and including) the * provided rows. diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index 80700931a6..b54378caaf 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -215,6 +215,11 @@ export class TabularData { return this.refresh(); } + addEmptyRecord() { + void this.recordsData.addEmptyRecord(); + this.selection.update((s) => s.ofNewRecordDataEntryCell()); + } + destroy(): void { this.recordsData.destroy(); this.constraintsDataStore.destroy(); diff --git a/mathesar_ui/src/systems/table-view/StatusPane.svelte b/mathesar_ui/src/systems/table-view/StatusPane.svelte index 4edf76dbaf..b4cf4a8b7e 100644 --- a/mathesar_ui/src/systems/table-view/StatusPane.svelte +++ b/mathesar_ui/src/systems/table-view/StatusPane.svelte @@ -68,11 +68,7 @@ disabled={$isLoading} size="medium" appearance="primary" - on:click={() => { - void recordsData.addEmptyRecord(); - // // TODO_3037 - // selection.selectAndActivateFirstDataEntryCellInLastRow(); - }} + on:click={() => $tabularData.addEmptyRecord()} > {$_('new_record')} diff --git a/mathesar_ui/src/systems/table-view/row/Row.svelte b/mathesar_ui/src/systems/table-view/row/Row.svelte index f22f7d333e..ed31fd4b8d 100644 --- a/mathesar_ui/src/systems/table-view/row/Row.svelte +++ b/mathesar_ui/src/systems/table-view/row/Row.svelte @@ -48,12 +48,11 @@ /** Including whole row errors and individual cell errors */ $: hasAnyErrors = !!status?.errorsFromWholeRowAndCells?.length; - function checkAndCreateEmptyRow() { - // // TODO_3037 - // if (isPlaceholderRow(row)) { - // void recordsData.addEmptyRecord(); - // selection.selectAndActivateFirstDataEntryCellInLastRow(); - // } + function handleMouseDown(e: MouseEvent) { + if (isPlaceholderRow(row)) { + $tabularData.addEmptyRecord(); + e.stopPropagation(); // Prevents cell selection from starting + } } @@ -69,7 +68,7 @@ class:is-add-placeholder={isPlaceholderRow(row)} {...htmlAttributes} style="--cell-height:{rowHeightPx - 1}px;{styleString}" - on:mousedown={checkAndCreateEmptyRow} + on:mousedown={handleMouseDown} > {#if rowHasRecord(row)} Date: Wed, 10 Apr 2024 10:47:44 -0400 Subject: [PATCH 0052/1141] Implement SheetSelectionStore --- .../sheet/selection/SheetSelectionStore.ts | 66 ++++++++++++++++ .../src/stores/PreventableEffectsStore.ts | 76 +++++++++++++++++++ .../src/stores/table-data/tabularData.ts | 17 ++--- .../src/systems/data-explorer/QueryRunner.ts | 15 +--- 4 files changed, 151 insertions(+), 23 deletions(-) create mode 100644 mathesar_ui/src/components/sheet/selection/SheetSelectionStore.ts create mode 100644 mathesar_ui/src/stores/PreventableEffectsStore.ts diff --git a/mathesar_ui/src/components/sheet/selection/SheetSelectionStore.ts b/mathesar_ui/src/components/sheet/selection/SheetSelectionStore.ts new file mode 100644 index 0000000000..780e4ad684 --- /dev/null +++ b/mathesar_ui/src/components/sheet/selection/SheetSelectionStore.ts @@ -0,0 +1,66 @@ +import type { Readable, Writable } from 'svelte/store'; + +import { EventHandler } from '@mathesar/component-library'; +import PreventableEffectsStore from '@mathesar/stores/PreventableEffectsStore'; +import type Plane from './Plane'; +import SheetSelection from './SheetSelection'; + +export default class SheetSelectionStore + extends EventHandler<{ + focus: void; + }> + implements Writable +{ + private selection: PreventableEffectsStore; + + private cleanupFunctions: (() => void)[] = []; + + constructor(plane: Readable) { + super(); + this.selection = new PreventableEffectsStore(new SheetSelection(), { + focus: () => this.focus(), + }); + this.cleanupFunctions.push( + plane.subscribe((p) => this.selection.update((s) => s.forNewPlane(p))), + ); + } + + private focus(): void { + void this.dispatch('focus'); + } + + subscribe(run: (value: SheetSelection) => void): () => void { + return this.selection.subscribe(run); + } + + update(getNewValue: (oldValue: SheetSelection) => SheetSelection): void { + this.selection.update(getNewValue); + } + + /** + * Updates the selection while skipping the side effect of focusing the active + * cell. + */ + updateWithoutFocus( + getNewValue: (oldValue: SheetSelection) => SheetSelection, + ): void { + this.selection.update(getNewValue, { prevent: ['focus'] }); + } + + set(value: SheetSelection): void { + this.selection.set(value); + } + + /** + * Sets the selection while skipping the side effect of focusing the active + * cell. + */ + setWithoutFocus(value: SheetSelection): void { + this.selection.update(() => value, { prevent: ['focus'] }); + } + + destroy(): void { + super.destroy(); + this.cleanupFunctions.forEach((f) => f()); + } +} diff --git a/mathesar_ui/src/stores/PreventableEffectsStore.ts b/mathesar_ui/src/stores/PreventableEffectsStore.ts new file mode 100644 index 0000000000..109b33de66 --- /dev/null +++ b/mathesar_ui/src/stores/PreventableEffectsStore.ts @@ -0,0 +1,76 @@ +import { writable, type Writable } from 'svelte/store'; + +interface Effect { + name: EffectNames; + run: (v: Value) => void; +} + +/** + * This is a writable store that holds effects which run when the value changes. + * Unlike other effect mechanisms in Svelte, this store allows you to prevent + * certain effects from running when the value changes — and control over that + * prevention is delegated to the call site of the update. + * + * The use case for this store is when you have an imperative effect that you + * want to run _almost_ all the time. By default you want the effect to run. But + * for some cases (when you do a little extra work), then you can prevent the + * effect from running while performing an update. + * + * ## Example + * + * ```ts + * const store = new PreventableEffectsStore(0, { + * log: (v) => console.log(v), + * }); + * + * store.update((v) => v + 1); // logs 1 + * store.update((v) => v + 1, { prevent: ['log'] }); // does not log + * ``` + */ +export default class PreventableEffectsStore< + Value, + EffectNames extends string, +> { + private value: Writable; + + private effects: Effect[] = []; + + constructor( + initialValue: Value, + effectMap: Record void>, + ) { + this.value = writable(initialValue); + this.effects = Object.entries(effectMap).map(([name, run]) => ({ + name: name as EffectNames, + run: run as (v: Value) => void, + })); + } + + private runEffects( + value: Value, + options: { prevent?: EffectNames[] } = {}, + ): void { + this.effects + .filter(({ name }) => !options.prevent?.includes(name)) + .forEach(({ run }) => run(value)); + } + + subscribe(run: (value: Value) => void): () => void { + return this.value.subscribe(run); + } + + update( + getNewValue: (oldValue: Value) => Value, + options: { prevent?: EffectNames[] } = {}, + ): void { + this.value.update((oldValue) => { + const newValue = getNewValue(oldValue); + this.runEffects(newValue, options); + return newValue; + }); + } + + set(value: Value): void { + this.value.set(value); + } +} diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index b54378caaf..ebfa7c3156 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -7,7 +7,7 @@ import type { Column } from '@mathesar/api/types/tables/columns'; import { States } from '@mathesar/api/utils/requestUtils'; import Plane from '@mathesar/components/sheet/selection/Plane'; import Series from '@mathesar/components/sheet/selection/Series'; -import SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; +import SheetSelectionStore from '@mathesar/components/sheet/selection/SheetSelectionStore'; import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; import type { ShareConsumer } from '@mathesar/utils/shares'; import { orderProcessedColumns } from '@mathesar/utils/tables'; @@ -57,12 +57,10 @@ export class TabularData { isLoading: Readable; - selection: Writable; + selection: SheetSelectionStore; table: TableEntry; - private cleanupFunctions: (() => void)[] = []; - shareConsumer?: ShareConsumer; constructor(props: TabularDataProps) { @@ -122,16 +120,11 @@ export class TabularData { const rowIds = new Series([...selectableRowsMap.keys()]); const columns = [...processedColumns.values()]; const columnIds = new Series(columns.map((c) => String(c.id))); + // TODO_3037 incorporate placeholder row into plane return new Plane(rowIds, columnIds); }, ); - - // TODO_3037 add id of placeholder row to selection - this.selection = writable(new SheetSelection()); - - this.cleanupFunctions.push( - plane.subscribe((p) => this.selection.update((s) => s.forNewPlane(p))), - ); + this.selection = new SheetSelectionStore(plane); this.isLoading = derived( [ @@ -224,7 +217,7 @@ export class TabularData { this.recordsData.destroy(); this.constraintsDataStore.destroy(); this.columnsDataStore.destroy(); - this.cleanupFunctions.forEach((f) => f()); + this.selection.destroy(); } } diff --git a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts index baee9cb275..51c100a931 100644 --- a/mathesar_ui/src/systems/data-explorer/QueryRunner.ts +++ b/mathesar_ui/src/systems/data-explorer/QueryRunner.ts @@ -12,7 +12,7 @@ import { ApiMultiError } from '@mathesar/api/utils/errors'; import type { RequestStatus } from '@mathesar/api/utils/requestUtils'; import Plane from '@mathesar/components/sheet/selection/Plane'; import Series from '@mathesar/components/sheet/selection/Series'; -import SheetSelection from '@mathesar/components/sheet/selection/SheetSelection'; +import SheetSelectionStore from '@mathesar/components/sheet/selection/SheetSelectionStore'; import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; import { fetchQueryResults, runQuery } from '@mathesar/stores/queries'; import Pagination from '@mathesar/utils/Pagination'; @@ -67,7 +67,7 @@ export default class QueryRunner { /** Keys are row selection ids */ selectableRowsMap: Readable>; - selection: Writable; + selection: SheetSelectionStore; inspector: QueryInspector; @@ -81,8 +81,6 @@ export default class QueryRunner { private shareConsumer?: ShareConsumer; - private cleanupFunctions: (() => void)[] = []; - constructor({ query, abstractTypeMap, @@ -120,12 +118,7 @@ export default class QueryRunner { return new Plane(rowIds, columnIds); }, ); - - this.selection = writable(new SheetSelection()); - - this.cleanupFunctions.push( - plane.subscribe((p) => this.selection.update((s) => s.forNewPlane(p))), - ); + this.selection = new SheetSelectionStore(plane); this.inspector = new QueryInspector(this.query); } @@ -279,6 +272,6 @@ export default class QueryRunner { destroy(): void { this.runPromise?.cancel(); - this.cleanupFunctions.forEach((fn) => fn()); + this.selection.destroy(); } } From 26a8c51d32f2afc3ca774e0453d5ca53318f9831 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 10:53:37 -0400 Subject: [PATCH 0053/1141] Fix some linting errors --- .../components/inspector/cell/CellInspector.svelte | 2 +- .../src/components/sheet/cells/SheetDataCell.svelte | 12 ++++++------ .../table-inspector/record/RowActions.svelte | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mathesar_ui/src/components/inspector/cell/CellInspector.svelte b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte index 0e6c3e8ad0..7f205da4cd 100644 --- a/mathesar_ui/src/components/inspector/cell/CellInspector.svelte +++ b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte @@ -1,8 +1,8 @@ diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record/RowActions.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record/RowActions.svelte index 2a3acc3b46..4c66583175 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record/RowActions.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record/RowActions.svelte @@ -31,7 +31,7 @@ const id = takeFirstAndOnly(selectedRowIds); if (!id) return undefined; const row = $selectableRowsMap.get(id); - if (!row) return; + if (!row) return undefined; try { const recordId = getPkValueInRecord(row.record, $columns); return $storeToGetRecordPageUrl({ recordId }); From 10f0aa283cadec7b18d1e6d955aee6f0b6740134 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 10:39:23 -0400 Subject: [PATCH 0054/1141] Handle CellWrapper focus in pure CSS --- .../data-types/components/CellWrapper.svelte | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte index e97c8f8646..e51a0011d1 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte @@ -23,8 +23,6 @@ */ export let horizontalAlignment: HorizontalAlignment = 'left'; - let isFocused = false; - function shouldAutoFocus( _isActive: boolean, _mode: 'edit' | 'default', @@ -60,23 +58,6 @@ } $: void handleStateChange(isActive, mode); - function handleFocus() { - isFocused = true; - // Note: you might think we ought to automatically activate the cell at this - // point to ensure that we don't have any cells which are focused but not - // active. I tried this and it caused bugs with selecting columns and rows - // via header cells. I didn't want to spend time tracking them down because - // we are planning to refactor the cell selection logic soon anyway. It - // doesn't _seem_ like we have any code which focuses the cell without - // activating it, but it would be nice to eventually build a better - // guarantee into the codebase which prevents cells from being focused - // without being activated. - } - - function handleBlur() { - isFocused = false; - } - function handleCopy(e: ClipboardEvent) { if (e.target !== element) { // When the user copies text _within_ a cell (e.g. from within an input @@ -93,7 +74,6 @@
@@ -146,7 +124,7 @@ box-shadow: 0 0 0 2px var(--slate-300); border-radius: 2px; - &.is-focused { + &:focus { box-shadow: 0 0 0 2px var(--sky-700); } } From bbd1e64362c0ff9d4082f6e770f720a6261c6c6e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 12:28:14 -0400 Subject: [PATCH 0055/1141] Improve cell focus behavior --- .../data-types/components/CellWrapper.svelte | 48 +++++++------------ mathesar_ui/src/components/sheet/Sheet.svelte | 22 +++++++-- .../ExtractColumnsModal.svelte | 4 +- 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte index e51a0011d1..fe9f1e0ebe 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte @@ -1,7 +1,6 @@
String(c.id)); - selection.update((s) => s.ofRowColumnIntersection(s.rowIds, columnIds)); + selection.updateWithoutFocus((s) => + s.ofRowColumnIntersection(s.rowIds, columnIds), + ); } $: handleColumnsChange($columns); From 894065a2cf68e1494c92466f189cc817fc7b7c4a Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 12:52:20 -0400 Subject: [PATCH 0056/1141] Fix mousedown event on cell input element --- .../data-types/components/CellWrapper.svelte | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte index fe9f1e0ebe..29594fe1d4 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/CellWrapper.svelte @@ -1,8 +1,12 @@
Date: Wed, 10 Apr 2024 13:08:16 -0400 Subject: [PATCH 0057/1141] Prevent column resize from altering selection --- mathesar_ui/src/component-library/common/actions/slider.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mathesar_ui/src/component-library/common/actions/slider.ts b/mathesar_ui/src/component-library/common/actions/slider.ts index 695a591db0..8a929d3b01 100644 --- a/mathesar_ui/src/component-library/common/actions/slider.ts +++ b/mathesar_ui/src/component-library/common/actions/slider.ts @@ -128,6 +128,8 @@ export default function slider( } function start(e: MouseEvent | TouchEvent) { + e.stopPropagation(); + e.preventDefault(); opts.onStart(); startingValue = opts.getStartingValue(); startingPosition = getPosition(e, opts.axis); From 2dff73f8ca7f47b7b6f5ce54b729b5d1e362a7b7 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 15:07:59 -0400 Subject: [PATCH 0058/1141] Improve parsing/serialization of cellId values --- mathesar_ui/src/components/sheet/cellIds.ts | 24 ++++++++------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/mathesar_ui/src/components/sheet/cellIds.ts b/mathesar_ui/src/components/sheet/cellIds.ts index fd6a263637..c39c1077d9 100644 --- a/mathesar_ui/src/components/sheet/cellIds.ts +++ b/mathesar_ui/src/components/sheet/cellIds.ts @@ -2,29 +2,23 @@ import { map } from 'iter-tools'; import { cartesianProduct } from '@mathesar/utils/iterUtils'; -const CELL_ID_DELIMITER = '-'; - -/** - * We can serialize a cell id this way only because we're confident that the - * rowId will never contain the delimiter. Some columnIds _do_ contain - * delimiters (e.g. in the Data Explorer), but that's okay because we can still - * separate the values based on the first delimiter. - */ export function makeCellId(rowId: string, columnId: string): string { - return `${rowId}${CELL_ID_DELIMITER}${columnId}`; + return JSON.stringify([rowId, columnId]); } export function parseCellId(cellId: string): { rowId: string; columnId: string; } { - const delimiterIndex = cellId.indexOf(CELL_ID_DELIMITER); - if (delimiterIndex === -1) { - throw new Error(`Unable to parse cell id without a delimiter: ${cellId}.`); + try { + const [rowId, columnId] = JSON.parse(cellId) as unknown[]; + if (typeof rowId !== 'string' || typeof columnId !== 'string') { + throw new Error(); + } + return { rowId, columnId }; + } catch { + throw new Error(`Unable to parse cell id: ${cellId}.`); } - const rowId = cellId.slice(0, delimiterIndex); - const columnId = cellId.slice(delimiterIndex + 1); - return { rowId, columnId }; } export function makeCells( From 0938b6582bf5a5650b91a5f99987d3bf40c46478 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 16:16:14 -0400 Subject: [PATCH 0059/1141] Add placeholder row to Plane --- mathesar_ui/src/stores/table-data/display.ts | 31 +++++++++++++++---- .../src/stores/table-data/tabularData.ts | 11 ++++--- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/mathesar_ui/src/stores/table-data/display.ts b/mathesar_ui/src/stores/table-data/display.ts index 0a969e23db..d4f98202ce 100644 --- a/mathesar_ui/src/stores/table-data/display.ts +++ b/mathesar_ui/src/stores/table-data/display.ts @@ -1,9 +1,10 @@ -import { writable, derived } from 'svelte/store'; -import type { Writable, Readable } from 'svelte/store'; +import type { Readable, Writable } from 'svelte/store'; +import { derived, writable } from 'svelte/store'; + import { WritableMap } from '@mathesar-component-library'; -import type { Meta } from './meta'; import type { ColumnsDataStore } from './columns'; -import { type Row, type RecordsData, filterRecordRows } from './records'; +import type { Meta } from './meta'; +import { filterRecordRows, type RecordsData, type Row } from './records'; // @deprecated export const DEFAULT_COLUMN_WIDTH = 160; @@ -63,6 +64,8 @@ export class Display { displayableRecords: Readable; + placeholderRowId: Readable; + constructor( meta: Meta, columnsDataStore: ColumnsDataStore, @@ -98,6 +101,9 @@ export class Display { ), ); + const placeholderRowId = writable(''); + this.placeholderRowId = placeholderRowId; + const { savedRecordRowsWithGroupHeaders, newRecords } = this.recordsData; this.displayableRecords = derived( [savedRecordRowsWithGroupHeaders, newRecords], @@ -120,11 +126,24 @@ export class Display { }) .concat($newRecords); } - allRecords = allRecords.concat({ + const placeholderRow = { ...this.recordsData.getNewEmptyRecord(), rowIndex: savedRecords.length + $newRecords.length, isAddPlaceholder: true, - }); + }; + + // This is really hacky! We have a side effect (mutating state) within a + // derived store, which I don't like. I put this here during a large + // refactor of the cell selection code because the Plane needs to know + // the id of the placeholder row since cell selection behaves + // differently in the placeholder row. I think we have some major + // refactoring to do across all the code that handles "rows" and + // "records" and things like that. There is a ton of mess there and I + // didn't want to lump any of that refactoring into an already-large + // refactor. + placeholderRowId.set(placeholderRow.identifier); + + allRecords = allRecords.concat(placeholderRow); return allRecords; }, ); diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index ebfa7c3156..22fa91bd52 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -115,13 +115,16 @@ export class TabularData { ); const plane = derived( - [this.recordsData.selectableRowsMap, this.processedColumns], - ([selectableRowsMap, processedColumns]) => { + [ + this.recordsData.selectableRowsMap, + this.processedColumns, + this.display.placeholderRowId, + ], + ([selectableRowsMap, processedColumns, placeholderRowId]) => { const rowIds = new Series([...selectableRowsMap.keys()]); const columns = [...processedColumns.values()]; const columnIds = new Series(columns.map((c) => String(c.id))); - // TODO_3037 incorporate placeholder row into plane - return new Plane(rowIds, columnIds); + return new Plane(rowIds, columnIds, placeholderRowId); }, ); this.selection = new SheetSelectionStore(plane); From 2f9cc960854f48ffc2efe60fa2fb0664334b41a8 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 16:31:37 -0400 Subject: [PATCH 0060/1141] Allow typing into placeholder cells --- mathesar_ui/src/systems/table-view/row/Row.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mathesar_ui/src/systems/table-view/row/Row.svelte b/mathesar_ui/src/systems/table-view/row/Row.svelte index ed31fd4b8d..4cbbd0820b 100644 --- a/mathesar_ui/src/systems/table-view/row/Row.svelte +++ b/mathesar_ui/src/systems/table-view/row/Row.svelte @@ -135,9 +135,9 @@ cursor: pointer; :global( - [data-sheet-element='data-cell']:not(.is-active) + [data-sheet-element='data-cell'] .cell-fabric - .cell-wrapper + .cell-wrapper:not(.is-edit-mode) > * ) { visibility: hidden; From 659c9e210cfd601398d67b434be9024f50481d17 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 21:38:58 -0400 Subject: [PATCH 0061/1141] Remove some dead code --- .../src/systems/table-view/TableView.svelte | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/mathesar_ui/src/systems/table-view/TableView.svelte b/mathesar_ui/src/systems/table-view/TableView.svelte index cb5052b530..64bc2cdb59 100644 --- a/mathesar_ui/src/systems/table-view/TableView.svelte +++ b/mathesar_ui/src/systems/table-view/TableView.svelte @@ -87,42 +87,22 @@ ]); $: showTableInspector = $isTableInspectorVisible && supportsTableInspector; - function checkAndReinstateFocusOnActiveCell(e: Event) { - // // TODO_3037 Figure out what is actually broken without this code. - // //Better document why we need id. - // - // const target = e.target as HTMLElement; - // if (!target.closest('[data-sheet-element="data-cell"')) { - // if ($activeCell) { - // selection.focusCell( - // // TODO make sure to use getRowSelectionId instead of rowIndex - // { rowIndex: $activeCell.rowIndex }, - // { id: Number($activeCell.columnId) }, - // ); - // } - // } - } - - // function selectAndActivateFirstCellOnTableLoad( - // _isLoading: boolean, - // _selection: TabularDataSelection, - // _context: Context, - // ) { + // // TODO_3037 + // function selectFirstCellOnTableLoad(_isLoading: boolean, _context: Context) { // // We only activate the first cell on the page, not in the widget. Doing so // // on the widget causes the cell to focus and the page to scroll down to // // bring that element into view. // if (_context !== 'widget' && !_isLoading) { - // _selection.selectAndActivateFirstCellIfExists(); + // selection.update((s) => s.ofFirstDataCell()); // } // } - // // TODO_3037 Figure out what is actually broken without this code. - // $: void selectAndActivateFirstCellOnTableLoad($isLoading, selection, context); + // $: void selectFirstCellOnTableLoad($isLoading, context);
-
+
{#if $processedColumns.size} Date: Wed, 10 Apr 2024 21:52:14 -0400 Subject: [PATCH 0062/1141] Hide column resize handles during cell selection --- mathesar_ui/src/components/sheet/Sheet.svelte | 9 ++++++++- mathesar_ui/src/components/sheet/SheetCellResizer.svelte | 8 +++++++- .../src/components/sheet/selection/selectionUtils.ts | 4 ++++ mathesar_ui/src/components/sheet/utils.ts | 1 + 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/mathesar_ui/src/components/sheet/Sheet.svelte b/mathesar_ui/src/components/sheet/Sheet.svelte index dde0f20681..d55728ca30 100644 --- a/mathesar_ui/src/components/sheet/Sheet.svelte +++ b/mathesar_ui/src/components/sheet/Sheet.svelte @@ -90,6 +90,7 @@ horizontalScrollOffset: writable(horizontalScrollOffset), scrollOffset: writable(scrollOffset), paddingRight: writable(paddingRight), + selectionInProgress: writable(false), }; // Setting these values in stores for reactivity in context @@ -154,7 +155,13 @@ // cells in the column. e.preventDefault(); - beginSelection({ selection, sheetElement, startingCell, targetCell }); + beginSelection({ + selection, + sheetElement, + startingCell, + targetCell, + selectionInProgress: stores.selectionInProgress, + }); } async function focusActiveCell() { diff --git a/mathesar_ui/src/components/sheet/SheetCellResizer.svelte b/mathesar_ui/src/components/sheet/SheetCellResizer.svelte index db09ee920f..e4c94fa140 100644 --- a/mathesar_ui/src/components/sheet/SheetCellResizer.svelte +++ b/mathesar_ui/src/components/sheet/SheetCellResizer.svelte @@ -4,16 +4,19 @@ type SheetColumnIdentifierKey = $$Generic; - const { api } = getSheetContext(); + const { api, stores } = getSheetContext(); export let minColumnWidth = 50; export let columnIdentifierKey: SheetColumnIdentifierKey; let isResizing = false; + + $: ({ selectionInProgress } = stores);
api.getColumnWidth(columnIdentifierKey), @@ -53,4 +56,7 @@ .column-resizer:not(:hover):not(.is-resizing) .indicator { display: none; } + .column-resizer.selection-in-progress { + display: none; + } diff --git a/mathesar_ui/src/components/sheet/selection/selectionUtils.ts b/mathesar_ui/src/components/sheet/selection/selectionUtils.ts index f79f894597..b8a54af1ab 100644 --- a/mathesar_ui/src/components/sheet/selection/selectionUtils.ts +++ b/mathesar_ui/src/components/sheet/selection/selectionUtils.ts @@ -42,11 +42,13 @@ export function beginSelection({ sheetElement, startingCell, targetCell, + selectionInProgress, }: { selection: Writable; sheetElement: HTMLElement; startingCell: SheetCellDetails; targetCell: SheetCellDetails; + selectionInProgress: Writable; }) { let previousTarget: HTMLElement | undefined; @@ -65,8 +67,10 @@ export function beginSelection({ function finish() { sheetElement.removeEventListener('mousemove', drawToPoint); window.removeEventListener('mouseup', finish); + selectionInProgress.set(false); } + selectionInProgress.set(true); drawToCell(targetCell); sheetElement.addEventListener('mousemove', drawToPoint); window.addEventListener('mouseup', finish); diff --git a/mathesar_ui/src/components/sheet/utils.ts b/mathesar_ui/src/components/sheet/utils.ts index 4f3915d8d0..489f0cf1bc 100644 --- a/mathesar_ui/src/components/sheet/utils.ts +++ b/mathesar_ui/src/components/sheet/utils.ts @@ -15,6 +15,7 @@ export interface SheetContextStores { horizontalScrollOffset: Readable; scrollOffset: Readable; paddingRight: Readable; + selectionInProgress: Readable; } export interface SheetContext { From d7f91d89a567df92a9d6a391af7a209ebdb9da38 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 22:03:37 -0400 Subject: [PATCH 0063/1141] Use default cursor everywhere in sheet during selection --- mathesar_ui/src/components/sheet/Sheet.svelte | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mathesar_ui/src/components/sheet/Sheet.svelte b/mathesar_ui/src/components/sheet/Sheet.svelte index d55728ca30..1899d5aaaf 100644 --- a/mathesar_ui/src/components/sheet/Sheet.svelte +++ b/mathesar_ui/src/components/sheet/Sheet.svelte @@ -84,13 +84,14 @@ }, }; + const selectionInProgress = writable(false); const stores = { columnStyleMap: writable(columnStyleMap), rowWidth: writable(rowWidth), horizontalScrollOffset: writable(horizontalScrollOffset), scrollOffset: writable(scrollOffset), paddingRight: writable(paddingRight), - selectionInProgress: writable(false), + selectionInProgress, }; // Setting these values in stores for reactivity in context @@ -160,7 +161,7 @@ sheetElement, startingCell, targetCell, - selectionInProgress: stores.selectionInProgress, + selectionInProgress, }); } @@ -177,6 +178,7 @@ class:has-border={hasBorder} class:uses-virtual-list={usesVirtualList} class:set-to-row-width={restrictWidthToRowWidth} + class:selection-in-progress={$selectionInProgress} {style} on:mousedown={handleMouseDown} on:focusin={enableClipboard} @@ -229,5 +231,9 @@ :global([data-sheet-element='data-row']) { transition: all 0.2s cubic-bezier(0, 0, 0.2, 1); } + + &.selection-in-progress :global(*) { + cursor: default; + } } From 1557b559ab39924d1eac99a0c6daa5aa969ef30f Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Apr 2024 22:16:13 -0400 Subject: [PATCH 0064/1141] Fix drag to re-order columns --- .../systems/table-view/header/Header.svelte | 39 +++++++++---------- .../header/drag-and-drop/Draggable.svelte | 13 ++++--- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/mathesar_ui/src/systems/table-view/header/Header.svelte b/mathesar_ui/src/systems/table-view/header/Header.svelte index ad28034c9f..11a7f77b41 100644 --- a/mathesar_ui/src/systems/table-view/header/Header.svelte +++ b/mathesar_ui/src/systems/table-view/header/Header.svelte @@ -108,28 +108,25 @@ {#each [...$processedColumns] as [columnId, processedColumn] (columnId)} {@const isSelected = $selection.columnIds.has(String(columnId))} - -
- dragColumn()} - column={processedColumn} - {selection} + dragColumn()} + column={processedColumn} + {selection} + > + dropColumn(processedColumn)} + on:dragover={(e) => e.preventDefault()} + {locationOfFirstDraggedColumn} + columnLocation={columnOrderString.indexOf(columnId.toString())} + {isSelected} > - dropColumn(processedColumn)} - on:dragover={(e) => e.preventDefault()} - {locationOfFirstDraggedColumn} - columnLocation={columnOrderString.indexOf(columnId.toString())} - {isSelected} - > - - - - - - - -
+ + + + + + +
{/each} diff --git a/mathesar_ui/src/systems/table-view/header/drag-and-drop/Draggable.svelte b/mathesar_ui/src/systems/table-view/header/drag-and-drop/Draggable.svelte index 1347bd2f5a..0c46402caa 100644 --- a/mathesar_ui/src/systems/table-view/header/drag-and-drop/Draggable.svelte +++ b/mathesar_ui/src/systems/table-view/header/drag-and-drop/Draggable.svelte @@ -7,17 +7,18 @@ export let selection: Writable; export let column: ProcessedColumn; - // // TODO_3037: Verify that we're not losing functionality here by removing - // `selectionInProgress` logic - // - // $: draggable = !selectionInProgress && selection && - // selection.isCompleteColumnSelected(column); - $: draggable = $selection.fullySelectedColumnIds.has(String(column.id)); + + function handleMouseDown(event: MouseEvent) { + if (draggable) { + event.stopPropagation(); + } + }
Date: Wed, 10 Apr 2024 23:28:04 -0400 Subject: [PATCH 0065/1141] Auto-activate inspector tabs based on selection --- mathesar_ui/src/components/sheet/Sheet.svelte | 7 +++ .../pages/exploration/ExplorationPage.svelte | 13 +++- .../systems/data-explorer/DataExplorer.svelte | 8 ++- .../systems/data-explorer/QueryInspector.ts | 10 --- .../src/systems/data-explorer/QueryRunner.ts | 10 --- .../ExplorationInspector.svelte | 13 +++- .../WithExplorationInspector.svelte | 5 ++ .../result-pane/ResultHeaderCell.svelte | 3 - .../result-pane/ResultPane.svelte | 11 +++- .../data-explorer/result-pane/Results.svelte | 5 ++ .../src/systems/table-view/TableView.svelte | 6 +- .../table-inspector/TableInspector.svelte | 63 +++++++------------ .../table-inspector/WithTableInspector.svelte | 6 +- mathesar_ui/src/utils/MessageBus.ts | 39 ++++++++++++ 14 files changed, 124 insertions(+), 75 deletions(-) create mode 100644 mathesar_ui/src/utils/MessageBus.ts diff --git a/mathesar_ui/src/components/sheet/Sheet.svelte b/mathesar_ui/src/components/sheet/Sheet.svelte index 1899d5aaaf..4a221044d1 100644 --- a/mathesar_ui/src/components/sheet/Sheet.svelte +++ b/mathesar_ui/src/components/sheet/Sheet.svelte @@ -5,6 +5,7 @@ import { ImmutableMap } from '@mathesar-component-library/types'; import type { ClipboardHandler } from '@mathesar/stores/clipboard'; import { getClipboardHandlerStoreFromContext } from '@mathesar/stores/clipboard'; + import type MessageBus from '@mathesar/utils/MessageBus'; import { getModifierKeyCombo } from '@mathesar/utils/pointerUtils'; import { beginSelection, @@ -30,6 +31,8 @@ export let hasPaddingRight = false; export let clipboardHandler: ClipboardHandler | undefined = undefined; export let selection: SheetSelectionStore | undefined = undefined; + export let cellSelectionStarted: MessageBus | undefined = + undefined; export let getColumnIdentifier: ( c: SheetColumnType, @@ -163,6 +166,10 @@ targetCell, selectionInProgress, }); + + if (startingCell === targetCell) { + cellSelectionStarted?.send(targetCell); + } } async function focusActiveCell() { diff --git a/mathesar_ui/src/pages/exploration/ExplorationPage.svelte b/mathesar_ui/src/pages/exploration/ExplorationPage.svelte index b71566c299..17c6a4ac3e 100644 --- a/mathesar_ui/src/pages/exploration/ExplorationPage.svelte +++ b/mathesar_ui/src/pages/exploration/ExplorationPage.svelte @@ -1,8 +1,10 @@
- +
From a2af3ad549d7608efff81fe9635a15a94f4694f6 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Sun, 19 May 2024 13:07:24 +0800 Subject: [PATCH 0198/1141] add column altering smoke test --- db/columns/operations/alter.py | 2 +- db/tests/columns/operations/test_alter.py | 53 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/db/columns/operations/alter.py b/db/columns/operations/alter.py index ea804fb6d4..bb0740c767 100644 --- a/db/columns/operations/alter.py +++ b/db/columns/operations/alter.py @@ -193,7 +193,7 @@ def alter_columns_in_table(table_oid, column_data_list, conn): transformed_column_data = [ _transform_column_alter_dict(column) for column in column_data_list ] - db_conn.execute_msar_func_with_engine( + db_conn.exec_msar_func( conn, 'alter_columns', table_oid, json.dumps(transformed_column_data) ) return len(column_data_list) diff --git a/db/tests/columns/operations/test_alter.py b/db/tests/columns/operations/test_alter.py index 7a783a6bf9..3fa449ed1c 100644 --- a/db/tests/columns/operations/test_alter.py +++ b/db/tests/columns/operations/test_alter.py @@ -1,6 +1,9 @@ +import json +from unittest.mock import patch from sqlalchemy import Column, select, Table, MetaData, VARCHAR, INTEGER from db import constants +from db.columns.operations import alter as col_alt from db.columns.operations.alter import batch_update_columns, rename_column from db.columns.operations.select import ( get_column_attnum_from_name, get_column_name_from_attnum, @@ -18,6 +21,56 @@ from db.schemas.utils import get_schema_oid_from_name +def test_alter_columns_in_table_smoke(): + with patch.object(col_alt.db_conn, 'exec_msar_func') as mock_exec: + col_alt.alter_columns_in_table( + 123, + [ + { + "id": 3, "name": "colname3", "type": "numeric", + "type_options": {"precision": 8}, "nullable": True, + "default": {"value": 8, "is_dynamic": False}, + "description": "third column" + }, + { + "id": 6, "name": "colname6", "type": "character varying", + "type_options": {"length": 32}, "nullable": True, + "default": {"value": "blahblah", "is_dynamic": False}, + "description": "textual column" + } + ], + 'conn' + ) + mock_exec.assert_called_once_with( + 'conn', 'alter_columns', 123, + json.dumps( + [ + { + "attnum": 3, + "name": "colname3", + "type": { + "name": "numeric", "options": {"precision": 8}, + }, + "not_null": False, + "name": "colname3", + "default": 8, + "description": "third column", + }, + { + "attnum": 6, + "type": { + "name": "character varying", "options": {"length": 32}, + }, + "not_null": False, + "name": "colname6", + "default": "blahblah", + "description": "textual column" + } + ], + ) + ) + + def _rename_column_and_assert(table, old_col_name, new_col_name, engine): """ Renames the colum of a table and assert the change went through From d781d7f8df73d1c4ac474cfdd68a00d0db33c250 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Sun, 19 May 2024 14:12:18 +0800 Subject: [PATCH 0199/1141] update basic column alterer test with accurate name and assertion --- db/tests/columns/operations/test_alter.py | 50 +++++++++-------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/db/tests/columns/operations/test_alter.py b/db/tests/columns/operations/test_alter.py index 3fa449ed1c..78dbeb1abe 100644 --- a/db/tests/columns/operations/test_alter.py +++ b/db/tests/columns/operations/test_alter.py @@ -21,7 +21,7 @@ from db.schemas.utils import get_schema_oid_from_name -def test_alter_columns_in_table_smoke(): +def test_alter_columns_in_table_basic(): with patch.object(col_alt.db_conn, 'exec_msar_func') as mock_exec: col_alt.alter_columns_in_table( 123, @@ -31,8 +31,7 @@ def test_alter_columns_in_table_smoke(): "type_options": {"precision": 8}, "nullable": True, "default": {"value": 8, "is_dynamic": False}, "description": "third column" - }, - { + }, { "id": 6, "name": "colname6", "type": "character varying", "type_options": {"length": 32}, "nullable": True, "default": {"value": "blahblah", "is_dynamic": False}, @@ -41,34 +40,23 @@ def test_alter_columns_in_table_smoke(): ], 'conn' ) - mock_exec.assert_called_once_with( - 'conn', 'alter_columns', 123, - json.dumps( - [ - { - "attnum": 3, - "name": "colname3", - "type": { - "name": "numeric", "options": {"precision": 8}, - }, - "not_null": False, - "name": "colname3", - "default": 8, - "description": "third column", - }, - { - "attnum": 6, - "type": { - "name": "character varying", "options": {"length": 32}, - }, - "not_null": False, - "name": "colname6", - "default": "blahblah", - "description": "textual column" - } - ], - ) - ) + expect_json_arg = [ + { + "attnum": 3, "name": "colname3", + "type": {"name": "numeric", "options": {"precision": 8}}, + "not_null": False, "default": 8, "description": "third column", + }, { + "attnum": 6, "name": "colname6", + "type": { + "name": "character varying", "options": {"length": 32}, + }, + "not_null": False, "default": "blahblah", + "description": "textual column" + } + ] + assert mock_exec.call_args.args[:3] == ('conn', 'alter_columns', 123) + # Necessary since `json.dumps` mangles dict ordering, but we don't care. + assert json.loads(mock_exec.call_args.args[3]) == expect_json_arg def _rename_column_and_assert(table, old_col_name, new_col_name, engine): From d85df60c3af1c8d36a73ffa961dd266bb3745a39 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 20 May 2024 09:14:25 -0400 Subject: [PATCH 0200/1141] Remove misleading usage of `min` SQL agg fn --- db/sql/0_msar.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/db/sql/0_msar.sql b/db/sql/0_msar.sql index a7f86d8363..9502d10451 100644 --- a/db/sql/0_msar.sql +++ b/db/sql/0_msar.sql @@ -683,8 +683,8 @@ SELECT jsonb_agg(schema_data) FROM ( SELECT s.oid AS oid, - min(s.nspname) AS name, - min(d.description) AS description, + s.nspname AS name, + d.description AS description, COALESCE(count(c.oid), 0) AS table_count FROM pg_catalog.pg_namespace s LEFT JOIN pg_catalog.pg_description d ON @@ -696,7 +696,10 @@ FROM ( WHERE s.nspname <> 'information_schema' AND s.nspname NOT LIKE 'pg_%' - GROUP BY s.oid + GROUP BY + s.oid, + s.nspname, + d.description ) AS schema_data; $$ LANGUAGE sql; From 38d8ceeb60f8cbdc686c9570e118364d64e6873d Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 20 May 2024 09:22:53 -0400 Subject: [PATCH 0201/1141] Use obj_description SQL fn --- db/sql/0_msar.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/db/sql/0_msar.sql b/db/sql/0_msar.sql index 9502d10451..3b1ca98b49 100644 --- a/db/sql/0_msar.sql +++ b/db/sql/0_msar.sql @@ -684,12 +684,9 @@ FROM ( SELECT s.oid AS oid, s.nspname AS name, - d.description AS description, + obj_description(s.oid) AS description, COALESCE(count(c.oid), 0) AS table_count FROM pg_catalog.pg_namespace s - LEFT JOIN pg_catalog.pg_description d ON - d.objoid = s.oid AND - d.objsubid = 0 LEFT JOIN pg_catalog.pg_class c ON c.relnamespace = s.oid AND c.relkind = 'r' @@ -698,8 +695,7 @@ FROM ( s.nspname NOT LIKE 'pg_%' GROUP BY s.oid, - s.nspname, - d.description + s.nspname ) AS schema_data; $$ LANGUAGE sql; From e79f31c2e1f4a36b4cb68a67bcea92d49bb917fb Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 20 May 2024 09:39:44 -0400 Subject: [PATCH 0202/1141] Add comment explaining SQL filter condition --- db/sql/0_msar.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/sql/0_msar.sql b/db/sql/0_msar.sql index 3b1ca98b49..7b33a20b65 100644 --- a/db/sql/0_msar.sql +++ b/db/sql/0_msar.sql @@ -689,6 +689,8 @@ FROM ( FROM pg_catalog.pg_namespace s LEFT JOIN pg_catalog.pg_class c ON c.relnamespace = s.oid AND + -- Filter on relkind so that we only count tables. This must be done in the ON clause so that + -- we still get a row for schemas with no tables. c.relkind = 'r' WHERE s.nspname <> 'information_schema' AND From f3289641fcf8f33ebe2c64aaf7d6c98dbf9e7bd4 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 20 May 2024 11:10:32 -0400 Subject: [PATCH 0203/1141] Namespace obj_description function --- db/sql/0_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/0_msar.sql b/db/sql/0_msar.sql index 7b33a20b65..77d91d1aea 100644 --- a/db/sql/0_msar.sql +++ b/db/sql/0_msar.sql @@ -684,7 +684,7 @@ FROM ( SELECT s.oid AS oid, s.nspname AS name, - obj_description(s.oid) AS description, + pg_catalog.obj_description(s.oid) AS description, COALESCE(count(c.oid), 0) AS table_count FROM pg_catalog.pg_namespace s LEFT JOIN pg_catalog.pg_class c ON From be9407bbf3e47f5b93b60fd54f461259a37dedb7 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 21 May 2024 00:39:07 +0800 Subject: [PATCH 0204/1141] fix test name typo, namespace catalog tables --- db/sql/00_msar.sql | 4 ++-- db/tests/columns/operations/test_drop.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 6653689c03..bfaee7189c 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -222,7 +222,7 @@ Args: rel_id: The OID of the relation. */ SELECT CASE - WHEN EXISTS (SELECT oid FROM pg_class WHERE oid=rel_id) THEN rel_id::regclass::text + WHEN EXISTS (SELECT oid FROM pg_catalog.pg_class WHERE oid=rel_id) THEN rel_id::regclass::text END $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -1136,7 +1136,7 @@ Args: DECLARE col_names text[]; BEGIN SELECT array_agg(quote_ident(attname)) - FROM pg_attribute + FROM pg_catalog.pg_attribute WHERE attrelid=tab_id AND NOT attisdropped AND ARRAY[attnum::integer] <@ col_ids INTO col_names; PERFORM __msar.drop_columns(msar.get_relation_name_or_null(tab_id), variadic col_names); diff --git a/db/tests/columns/operations/test_drop.py b/db/tests/columns/operations/test_drop.py index 35805cd8bc..0bedaa2cf3 100644 --- a/db/tests/columns/operations/test_drop.py +++ b/db/tests/columns/operations/test_drop.py @@ -10,7 +10,7 @@ def test_drop_columns(): assert result == 3 -def test_get_column_info_for_table(): +def test_drop_columns_single(): with patch.object(col_drop.db_conn, 'exec_msar_func') as mock_exec: mock_exec.return_value.fetchone = lambda: (1,) result = col_drop.drop_columns_from_table(123, [1], 'conn') From 7f41d5ea416324c69b3d7ef2fa1004c27a1c739b Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 20 May 2024 22:45:08 +0530 Subject: [PATCH 0205/1141] fix namespacing --- db/sql/0_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/0_msar.sql b/db/sql/0_msar.sql index 0d3e2f9d3f..f1fb7503bc 100644 --- a/db/sql/0_msar.sql +++ b/db/sql/0_msar.sql @@ -696,7 +696,7 @@ SELECT jsonb_agg( ) ) FROM pg_catalog.pg_class AS pgc - LEFT JOIN pg_namespace AS pgn ON pgc.relnamespace = pgn.oid + LEFT JOIN pg_catalog.pg_namespace AS pgn ON pgc.relnamespace = pgn.oid WHERE pgc.relnamespace = sch_id AND pgc.relkind = 'r'; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; From 2ba206c96ddf2f593631c41df22e0d6666cf8af6 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 20 May 2024 23:08:34 +0530 Subject: [PATCH 0206/1141] id --> oid --- db/sql/00_msar.sql | 4 ++-- db/tables/operations/select.py | 2 +- mathesar/rpc/tables.py | 25 ++++++------------------- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 03727eef9b..05ef995580 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -698,7 +698,7 @@ Given a schema identifier, return an array of objects describing the tables of t Each returned JSON object in the array will have the form: { - "id": , + "oid": , "name": , "schema": , "description": @@ -709,7 +709,7 @@ Args: */ SELECT jsonb_agg( jsonb_build_object( - 'id', pgc.oid, + 'oid', pgc.oid, 'name', pgc.relname, 'schema', pgc.relnamespace, 'description', msar.obj_description(pgc.oid, 'pg_class') diff --git a/db/tables/operations/select.py b/db/tables/operations/select.py index f07c27006d..ec0871d3eb 100644 --- a/db/tables/operations/select.py +++ b/db/tables/operations/select.py @@ -25,7 +25,7 @@ def get_table_info(schema, conn): The returned list contains dictionaries of the following form: { - "id": , + "oid": , "name": , "schema": , "description": diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index b126cae909..948ee35248 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -1,4 +1,4 @@ -from typing import TypedDict +from typing import Optional, TypedDict from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required @@ -13,31 +13,21 @@ class TableInfo(TypedDict): Information about a table. Attributes: - id: The `oid` of the table in the schema. + oid: The `oid` of the table in the schema. name: The name of the table. schema: The `oid` of the schema where the table lives. description: The description of the table. """ - id: int + oid: int name: str schema: int - description: str - - -class TableListReturn(TypedDict): - """ - Information about the tables of a schema. - - Attributes: - table_info: Column information from the user's database. - """ - table_info: list[TableInfo] + description: Optional[str] @rpc_method(name="tables.list") @http_basic_auth_login_required @handle_rpc_exceptions -def list_(*, schema_oid: int, database_id: int, **kwargs) -> TableListReturn: +def list_(*, schema_oid: int, database_id: int, **kwargs) -> list[TableInfo]: """ List information about tables for a schema. Exposed as `list`. @@ -51,9 +41,6 @@ def list_(*, schema_oid: int, database_id: int, **kwargs) -> TableListReturn: user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: raw_table_info = get_table_info(schema_oid, conn) - table_info = [ + return [ TableInfo(tab) for tab in raw_table_info ] - return TableListReturn( - table_info=table_info - ) From 30d7500820aeaaf2384c57c2781b6f9c84857c54 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 21 May 2024 01:19:30 +0530 Subject: [PATCH 0207/1141] add python tests --- config/settings/common_settings.py | 3 +- db/tests/tables/operations/test_select.py | 9 ++++ mathesar/tests/rpc/test_endpoints.py | 6 +++ mathesar/tests/rpc/test_tables.py | 64 +++++++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 mathesar/tests/rpc/test_tables.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index abd0e94204..ce2ce24282 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -67,7 +67,8 @@ def pipe_delim(pipe_string): MODERNRPC_METHODS_MODULES = [ 'mathesar.rpc.connections', 'mathesar.rpc.columns', - 'mathesar.rpc.schemas' + 'mathesar.rpc.schemas', + 'mathesar.rpc.tables' ] TEMPLATES = [ diff --git a/db/tests/tables/operations/test_select.py b/db/tests/tables/operations/test_select.py index 4f442c1f00..36c0cc8ec9 100644 --- a/db/tests/tables/operations/test_select.py +++ b/db/tests/tables/operations/test_select.py @@ -1,4 +1,5 @@ import sys +from unittest.mock import patch from sqlalchemy import text from db.columns.operations.select import get_column_name_from_attnum from db.tables.operations import select as ma_sel @@ -34,6 +35,14 @@ MULTIPLE_RESULTS = ma_sel.MULTIPLE_RESULTS +def test_get_table_info(): + with patch.object(ma_sel, 'exec_msar_func') as mock_exec: + mock_exec.return_value.fetchone = lambda: ('a', 'b') + result = ma_sel.get_table_info('schema', 'conn') + mock_exec.assert_called_once_with('conn', 'get_table_info', 'schema') + assert result == 'a' + + def _transform_row_to_names(row, engine): metadata = get_empty_metadata() output_dict = { diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index e80eb7073f..8240e3fc3a 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -11,6 +11,7 @@ from mathesar.rpc import columns from mathesar.rpc import connections from mathesar.rpc import schemas +from mathesar.rpc import tables METHODS = [ ( @@ -38,6 +39,11 @@ "schemas.list", [user_is_authenticated] ), + ( + tables.list_, + "tables.list", + [user_is_authenticated] + ), ] diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py new file mode 100644 index 0000000000..ab9ad260a8 --- /dev/null +++ b/mathesar/tests/rpc/test_tables.py @@ -0,0 +1,64 @@ +""" +This file tests the table RPC functions. + +Fixtures: + rf(pytest-django): Provides mocked `Request` objects. + monkeypatch(pytest): Lets you monkeypatch an object for testing. +""" +from contextlib import contextmanager + +from mathesar.rpc import tables +from mathesar.models.users import User + + +def test_tables_list(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + schema_oid = 2200 + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_table_info(_schema_oid, conn): + if _schema_oid != schema_oid: + raise AssertionError('incorrect parameters passed') + return [ + { + 'oid': 17408, + 'name': 'Authors', + 'schema': schema_oid, + 'description': 'a description on the authors table.' + }, + { + 'oid': 17809, + 'name': 'Books', + 'schema': schema_oid, + 'description': None + } + ] + monkeypatch.setattr(tables, 'connect', mock_connect) + monkeypatch.setattr(tables, 'get_table_info', mock_table_info) + expect_table_list = [ + { + 'oid': 17408, + 'name': 'Authors', + 'schema': schema_oid, + 'description': 'a description on the authors table.' + }, + { + 'oid': 17809, + 'name': 'Books', + 'schema': schema_oid, + 'description': None + } + ] + actual_table_list = tables.list_(schema_oid=2200, database_id=11, request=request) + assert actual_table_list == expect_table_list From 07663a0d196296d67d8a315ae87efa197222175e Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 21 May 2024 03:57:11 +0530 Subject: [PATCH 0208/1141] add api docs --- docs/docs/api/rpc.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index b7a594b3c8..bce7d6e441 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -56,6 +56,14 @@ To use an RPC function: --- +::: mathesar.rpc.tables + options: + members: + - list_ + - TableInfo + +--- + ::: mathesar.rpc.columns options: members: From b5f7cadd68ee08595fc6369b0a0de0daa26a6794 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 05:36:50 +0000 Subject: [PATCH 0209/1141] --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6c686c71a9..5ae8c5bd9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ psycopg==3.1.18 psycopg-binary==3.1.18 psycopg2-binary==2.9.7 python-decouple==3.4 -requests==2.31.0 +requests==2.32.0 SQLAlchemy==1.4.26 responses==0.22.0 SQLAlchemy-Utils==0.38.2 From b35148f563c0824e043b1539a9c58554f1afc788 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 21 May 2024 17:34:42 +0530 Subject: [PATCH 0210/1141] add sql tests --- db/sql/test_00_msar.sql | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 929cb416bc..947136670d 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2570,6 +2570,61 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION __setup_get_table_info() RETURNS SETOF TEXT AS $$ +BEGIN + CREATE SCHEMA pi; + -- Two tables with one having description + CREATE TABLE pi.three(id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY); + CREATE TABLE pi.one(id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY); + COMMENT ON TABLE pi.one IS 'first decimal digit of pi'; + + CREATE SCHEMA alice; + -- No tables in the schema +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_get_table_info() RETURNS SETOF TEXT AS $$ +DECLARE + pi_table_info jsonb; + alice_table_info jsonb; +BEGIN + PERFORM __setup_get_table_info(); + SELECT msar.get_table_info('pi') INTO pi_table_info; + SELECT msar.get_table_info('alice') INTO alice_table_info; + + -- Test table info for schema 'pi' + -- Check if all the required keys exist in the json blob + -- Check whether the correct name is returned + -- Check whether the correct description is returned + RETURN NEXT is( + pi_table_info->0 ?& array['oid', 'name', 'schema', 'description'], true + ); + RETURN NEXT is( + pi_table_info->0->>'name', 'three' + ); + RETURN NEXT is( + pi_table_info->0->>'description', null + ); + + RETURN NEXT is( + pi_table_info->1 ?& array['oid', 'name', 'schema', 'description'], true + ); + RETURN NEXT is( + pi_table_info->1->>'name', 'one' + ); + RETURN NEXT is( + pi_table_info->1->>'description', 'first decimal digit of pi' + ); + + -- Test table info for schema 'alice' that contains no tables + RETURN NEXT is( + alice_table_info, null + ); +END; +$$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION test_get_schemas() RETURNS SETOF TEXT AS $$ DECLARE initial_schema_count int; From 90e3ff3addd94814a439079e666b0dcba864e684 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 27 May 2024 19:48:58 +0530 Subject: [PATCH 0211/1141] add rpc endpoint for tables.delete with tests --- db/sql/00_msar.sql | 13 +++++++++---- db/sql/test_00_msar.sql | 18 +++--------------- db/tables/operations/drop.py | 20 +++++++++++++++++++- mathesar/rpc/tables.py | 23 +++++++++++++++++++++++ mathesar/tests/rpc/test_tables.py | 27 +++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 20 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 05ef995580..9556b296b4 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2135,16 +2135,21 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.drop_table(tab_id oid, cascade_ boolean, if_exists boolean) RETURNS text AS $$/* -Drop a table, returning the command executed. +msar.drop_table(tab_id oid, cascade_ boolean) RETURNS text AS $$/* +Drop a table, returning the fully qualified name of the dropped table. Args: tab_id: The OID of the table to drop cascade_: Whether to drop dependent objects. - if_exists_: Whether to ignore an error if the table doesn't exist */ +DECLARE relation_name text; BEGIN - RETURN __msar.drop_table(__msar.get_relation_name(tab_id), cascade_, if_exists); + relation_name := __msar.get_relation_name(tab_id); + -- if_exists doesn't work while working with oids because + -- the SQL query gets parameterized with tab_id instead of relation_name + -- since we're unable to find the relation_name for a non existing table. + PERFORM __msar.drop_table(relation_name, cascade_, if_exists => false); + RETURN relation_name; END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 947136670d..ab004a8e84 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -72,24 +72,12 @@ DECLARE BEGIN PERFORM __setup_drop_tables(); rel_id := 'dropme'::regclass::oid; - PERFORM msar.drop_table(tab_id => rel_id, cascade_ => false, if_exists => false); + PERFORM msar.drop_table(tab_id => rel_id, cascade_ => false); RETURN NEXT hasnt_table('dropme', 'Drops table'); END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION test_drop_table_oid_if_exists() RETURNS SETOF TEXT AS $$ -DECLARE - rel_id oid; -BEGIN - PERFORM __setup_drop_tables(); - rel_id := 'dropme'::regclass::oid; - PERFORM msar.drop_table(tab_id => rel_id, cascade_ => false, if_exists => true); - RETURN NEXT hasnt_table('dropme', 'Drops table with IF EXISTS'); -END; -$$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION test_drop_table_oid_restricted_fkey() RETURNS SETOF TEXT AS $$ DECLARE rel_id oid; @@ -99,7 +87,7 @@ BEGIN CREATE TABLE dependent (id SERIAL PRIMARY KEY, col1 integer REFERENCES dropme); RETURN NEXT throws_ok( - format('SELECT msar.drop_table(tab_id => %s, cascade_ => false, if_exists => true);', rel_id), + format('SELECT msar.drop_table(tab_id => %s, cascade_ => false);', rel_id), '2BP01', 'cannot drop table dropme because other objects depend on it', 'Table dropper throws for dependent objects' @@ -116,7 +104,7 @@ BEGIN rel_id := 'dropme'::regclass::oid; CREATE TABLE dependent (id SERIAL PRIMARY KEY, col1 integer REFERENCES dropme); - PERFORM msar.drop_table(tab_id => rel_id, cascade_ => true, if_exists => false); + PERFORM msar.drop_table(tab_id => rel_id, cascade_ => true); RETURN NEXT hasnt_table('dropme', 'Drops table with dependent using CASCADE'); END; $$ LANGUAGE plpgsql; diff --git a/db/tables/operations/drop.py b/db/tables/operations/drop.py index f0f40b3c88..9435dd2c70 100644 --- a/db/tables/operations/drop.py +++ b/db/tables/operations/drop.py @@ -1,5 +1,23 @@ -from db.connection import execute_msar_func_with_engine +from db.connection import execute_msar_func_with_engine, exec_msar_func def drop_table(name, schema, engine, cascade=False, if_exists=False): execute_msar_func_with_engine(engine, 'drop_table', schema, name, cascade, if_exists) + + +def drop_table_from_schema(table_oid, conn, cascade=False): + """ + Drop a table. + + Args: + table_oid: OID of the table to drop. + cascade: Whether to drop the dependent objects. + if_exists: Whether to ignore an error if the schema doesn't + exist. + + Returns: + Returns the fully qualified name of the dropped table. + """ + return exec_msar_func( + conn, 'drop_table', table_oid, cascade + ).fetchone()[0] diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 948ee35248..683e60f307 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -4,6 +4,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from db.tables.operations.select import get_table_info +from db.tables.operations.drop import drop_table_from_schema from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -44,3 +45,25 @@ def list_(*, schema_oid: int, database_id: int, **kwargs) -> list[TableInfo]: return [ TableInfo(tab) for tab in raw_table_info ] + + +@rpc_method(name="tables.delete") +@http_basic_auth_login_required +@handle_rpc_exceptions +def delete( + *, table_oid: int, cascade: bool = False, database_id: int, **kwargs +) -> str: + """ + Delete a table from a schema. + + Args: + table_oid: Identity of the table in the user's database. + cascade: Whether to drop the dependent objects. + database_id: The Django id of the database containing the table. + + Returns: + The name of the dropped table. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + return drop_table_from_schema(table_oid, conn, cascade) diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index ab9ad260a8..b4b7d42887 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -62,3 +62,30 @@ def mock_table_info(_schema_oid, conn): ] actual_table_list = tables.list_(schema_oid=2200, database_id=11, request=request) assert actual_table_list == expect_table_list + + +def test_tables_delete(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + table_oid = 1964474 + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_drop_table(_table_oid, conn, cascade): + if _table_oid != table_oid: + raise AssertionError('incorrect parameters passed') + return 'public."Table 0"' + + monkeypatch.setattr(tables, 'connect', mock_connect) + monkeypatch.setattr(tables, 'drop_table_from_schema', mock_drop_table) + deleted_table = tables.delete(table_oid=1964474, database_id=11, request=request) + assert deleted_table == 'public."Table 0"' From b51dee1b778a3f39a999d3d3d5aabdc9ccd17df2 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 27 May 2024 19:52:55 +0530 Subject: [PATCH 0212/1141] add endpoint test and update docs --- docs/docs/api/rpc.md | 1 + mathesar/tests/rpc/test_endpoints.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index bce7d6e441..06d862f457 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -60,6 +60,7 @@ To use an RPC function: options: members: - list_ + - delete - TableInfo --- diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 8240e3fc3a..b46af51cc0 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -44,6 +44,11 @@ "tables.list", [user_is_authenticated] ), + ( + tables.delete, + "tables.delete", + [user_is_authenticated] + ) ] From 2fbc79d4605cb6f6f1a50839ae7bf233e056145f Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 29 May 2024 14:55:55 -0400 Subject: [PATCH 0213/1141] Implement schemas.delete RPC method --- conftest.py | 4 +- db/schemas/operations/drop.py | 39 ++++++++----- db/sql/00_msar.sql | 59 +++++++------------ db/sql/test_00_msar.sql | 74 ++++++++++-------------- db/tests/schemas/operations/test_drop.py | 9 +-- docs/docs/api/rpc.md | 1 + mathesar/models/base.py | 5 +- mathesar/rpc/schemas.py | 16 +++++ mathesar/tests/rpc/test_endpoints.py | 5 ++ 9 files changed, 104 insertions(+), 108 deletions(-) diff --git a/conftest.py b/conftest.py index 41589a1e07..6089145050 100644 --- a/conftest.py +++ b/conftest.py @@ -13,7 +13,7 @@ from db.engine import add_custom_types_to_ischema_names, create_engine as sa_create_engine from db.types import install from db.sql import install as sql_install -from db.schemas.operations.drop import drop_schema as drop_sa_schema +from db.schemas.operations.drop import drop_schema_via_name as drop_sa_schema from db.schemas.operations.create import create_schema as create_sa_schema from db.schemas.utils import get_schema_oid_from_name, get_schema_name_from_oid @@ -225,7 +225,7 @@ def _create_schema(schema_name, engine, schema_mustnt_exist=True): # Handle schemas being renamed during test schema_name = get_schema_name_from_oid(schema_oid, engine) if schema_name: - drop_sa_schema(schema_name, engine, cascade=True, if_exists=True) + drop_sa_schema(engine, schema_name, cascade=True) logger.debug(f'dropping {schema_name}') except OperationalError as e: logger.debug(f'ignoring operational error: {e}') diff --git a/db/schemas/operations/drop.py b/db/schemas/operations/drop.py index 2b78e60a49..919ce352bb 100644 --- a/db/schemas/operations/drop.py +++ b/db/schemas/operations/drop.py @@ -1,20 +1,33 @@ -from db.connection import execute_msar_func_with_engine +from db.connection import execute_msar_func_with_engine, exec_msar_func -def drop_schema(schema_name, engine, cascade=False, if_exists=False): +def drop_schema_via_name(engine, name, cascade=False): """ - Drop a schema. + Drop a schema by its name. + + If no schema exists with the given name, an exception will be raised. + + Deprecated: + Use drop_schema_via_oid instead. This function is deprecated because we + are phasing out name-based operations in favor of OID-based operations + and we are phasing out SQLAlchemy in favor of psycopg. Args: - schema_name: Name of the schema to drop. - engine: SQLAlchemy engine object for connecting. - cascade: Whether to drop the dependent objects. - if_exists: Whether to ignore an error if the schema doesn't - exist. + engine: SQLAlchemy engine object for connecting. name: Name of the + schema to drop. cascade: Whether to drop the dependent objects. + """ + execute_msar_func_with_engine(engine, 'drop_schema', name, cascade).fetchone() + - Returns: - Returns a string giving the command that was run. +def drop_schema_via_oid(conn, id, cascade=False): + """ + Drop a schema by its OID. + + If no schema exists with the given oid, an exception will be raised. + + Args: + conn: a psycopg connection + id: the OID of the schema to drop. + cascade: Whether to drop the dependent objects. """ - return execute_msar_func_with_engine( - engine, 'drop_schema', schema_name, cascade, if_exists - ).fetchone()[0] + exec_msar_func(conn, 'drop_schema', id, cascade).fetchone() diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 05ef995580..74706fae44 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -957,61 +957,42 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; ---------------------------------------------------------------------------------------------------- --- Drop schema ------------------------------------------------------------------------------------- - CREATE OR REPLACE FUNCTION -__msar.drop_schema(sch_name text, cascade_ boolean, if_exists boolean) RETURNS TEXT AS $$/* -Drop a schema, returning the command executed. +msar.drop_schema(sch_name text, cascade_ boolean) RETURNS void AS $$/* +Drop a schema + +If no schema exists with the given name, an exception will be raised. Args: - sch_name: A properly quoted name of the schema to be dropped - cascade_: Whether to drop dependent objects. - if_exists: Whether to ignore an error if the schema doesn't exist + sch_name: An unqoted name of the schema to be dropped + cascade_: When true, dependent objects will be dropped automatically */ DECLARE - cmd_template text; + cascade_sql text = CASE cascade_ WHEN TRUE THEN ' CASCADE' ELSE '' END; BEGIN - IF if_exists - THEN - cmd_template := 'DROP SCHEMA IF EXISTS %s'; - ELSE - cmd_template := 'DROP SCHEMA %s'; - END IF; - IF cascade_ - THEN - cmd_template = cmd_template || ' CASCADE'; - END IF; - RETURN __msar.exec_ddl(cmd_template, sch_name); + EXECUTE 'DROP SCHEMA ' || quote_ident(sch_name) || cascade_sql; END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.drop_schema(sch_id oid, cascade_ boolean, if_exists boolean) RETURNS TEXT AS $$/* -Drop a schema, returning the command executed. - -Args: - sch_id: The OID of the schema to drop - cascade_: Whether to drop dependent objects. - if_exists: Whether to ignore an error if the schema doesn't exist -*/ -BEGIN - RETURN __msar.drop_schema(__msar.get_schema_name(sch_id), cascade_, if_exists); -END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +msar.drop_schema(sch_id oid, cascade_ boolean) RETURNS void AS $$/* +Drop a schema - -CREATE OR REPLACE FUNCTION -msar.drop_schema(sch_name text, cascade_ boolean, if_exists boolean) RETURNS TEXT AS $$/* -Drop a schema, returning the command executed. +If no schema exists with the given oid, an exception will be raised. Args: - sch_name: An unqoted name of the schema to be dropped - cascade_: Whether to drop dependent objects. - if_exists: Whether to ignore an error if the schema doesn't exist + sch_id: The OID of the schema to drop + cascade_: When true, dependent objects will be dropped automatically */ +DECLARE + sch_name text; BEGIN - RETURN __msar.drop_schema(quote_ident(sch_name), cascade_, if_exists); + SELECT nspname INTO sch_name FROM pg_namespace WHERE oid = sch_id; + IF sch_name IS NULL THEN + RAISE EXCEPTION 'No schema with OID % exists.', sch_id; + END IF; + PERFORM msar.drop_schema(sch_name, cascade_); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 947136670d..6b20d34c78 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -1213,63 +1213,51 @@ END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION test_drop_schema_if_exists_false() RETURNS SETOF TEXT AS $$ +CREATE OR REPLACE FUNCTION test_drop_schema_using_name() RETURNS SETOF TEXT AS $$ BEGIN PERFORM __setup_drop_schema(); PERFORM msar.drop_schema( sch_name => 'drop_test_schema', - cascade_ => false, - if_exists => false + cascade_ => false ); RETURN NEXT hasnt_schema('drop_test_schema'); RETURN NEXT throws_ok( - format( - 'SELECT msar.drop_schema( - sch_name => ''%s'', - cascade_ => false, - if_exists => false - );', - 'drop_non_existing_schema' - ), - '3F000', - 'schema "drop_non_existing_schema" does not exist' + $d$ + SELECT msar.drop_schema( + sch_name => 'drop_non_existing_schema', + cascade_ => false + ) + $d$, + '3F000' ); END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION test_drop_schema_if_exists_true() RETURNS SETOF TEXT AS $$ +CREATE OR REPLACE FUNCTION test_drop_schema_using_oid() RETURNS SETOF TEXT AS $$ BEGIN PERFORM __setup_drop_schema(); PERFORM msar.drop_schema( - sch_name => 'drop_test_schema', - cascade_ => false, - if_exists => true + sch_id => 'drop_test_schema'::regnamespace::oid, + cascade_ => false ); RETURN NEXT hasnt_schema('drop_test_schema'); - RETURN NEXT lives_ok( - format( - 'SELECT msar.drop_schema( - sch_name => ''%s'', - cascade_ => false, - if_exists => true - );', - 'drop_non_existing_schema' - ) - ); END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION test_drop_schema_using_oid() RETURNS SETOF TEXT AS $$ +CREATE OR REPLACE FUNCTION test_drop_schema_using_invalid_oid() RETURNS SETOF TEXT AS $$ BEGIN PERFORM __setup_drop_schema(); - PERFORM msar.drop_schema( - sch_id => 'drop_test_schema'::regnamespace::oid, - cascade_ => false, - if_exists => false + RETURN NEXT throws_ok( + $d$ + SELECT msar.drop_schema( + sch_id => 0, + cascade_ => false + ) + $d$, + 'P0001' ); - RETURN NEXT hasnt_schema('drop_test_schema'); END; $$ LANGUAGE plpgsql; @@ -1290,8 +1278,7 @@ BEGIN PERFORM __setup_schema_with_dependent_obj(); PERFORM msar.drop_schema( sch_name => 'schema1', - cascade_ => true, - if_exists => false + cascade_ => true ); RETURN NEXT hasnt_schema('schema1'); END; @@ -1302,16 +1289,13 @@ CREATE OR REPLACE FUNCTION test_drop_schema_restricted() RETURNS SETOF TEXT AS $ BEGIN PERFORM __setup_schema_with_dependent_obj(); RETURN NEXT throws_ok( - format( - 'SELECT msar.drop_schema( - sch_name => ''%s'', - cascade_ => false, - if_exists => false - );', - 'schema1' - ), - '2BP01', - 'cannot drop schema schema1 because other objects depend on it' + $d$ + SELECT msar.drop_schema( + sch_name => 'schema1', + cascade_ => false + ) + $d$, + '2BP01' ); END; $$ LANGUAGE plpgsql; diff --git a/db/tests/schemas/operations/test_drop.py b/db/tests/schemas/operations/test_drop.py index cf513b11b7..243ec4fbd6 100644 --- a/db/tests/schemas/operations/test_drop.py +++ b/db/tests/schemas/operations/test_drop.py @@ -3,16 +3,13 @@ import db.schemas.operations.drop as sch_drop -@pytest.mark.parametrize( - "cascade, if_exists", [(True, True), (False, True), (True, False), (False, False)] -) -def test_drop_schema(engine_with_schema, cascade, if_exists): +@pytest.mark.parametrize("cascade", [True, False]) +def test_drop_schema(engine_with_schema, cascade): engine = engine_with_schema with patch.object(sch_drop, 'execute_msar_func_with_engine') as mock_exec: - sch_drop.drop_schema('drop_test_schema', engine, cascade, if_exists) + sch_drop.drop_schema_via_name(engine, 'drop_test_schema', cascade) call_args = mock_exec.call_args_list[0][0] assert call_args[0] == engine assert call_args[1] == "drop_schema" assert call_args[2] == "drop_test_schema" assert call_args[3] == cascade - assert call_args[4] == if_exists diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index bce7d6e441..6138958667 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -52,6 +52,7 @@ To use an RPC function: options: members: - list_ + - delete - SchemaInfo --- diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 60b2b5059b..19685b065c 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -28,7 +28,7 @@ from db.records.operations.select import get_column_cast_records, get_count, get_record from db.records.operations.select import get_records from db.records.operations.update import update_record -from db.schemas.operations.drop import drop_schema +from db.schemas.operations.drop import drop_schema_via_name from db.schemas.operations.select import get_schema_description from db.schemas import utils as schema_utils from db.tables import utils as table_utils @@ -241,9 +241,8 @@ def update_sa_schema(self, update_params): return result def delete_sa_schema(self): - result = drop_schema(self.name, self._sa_engine, cascade=True) + drop_schema_via_name(self._sa_engine, self.name, cascade=True) reset_reflection(db_name=self.database.name) - return result def clear_name_cache(self): cache_key = f"{self.database.name}_schema_name_{self.oid}" diff --git a/mathesar/rpc/schemas.py b/mathesar/rpc/schemas.py index 1fdefa5b77..0b166fcd6c 100644 --- a/mathesar/rpc/schemas.py +++ b/mathesar/rpc/schemas.py @@ -8,6 +8,7 @@ from db.constants import INTERNAL_SCHEMAS from db.schemas.operations.select import get_schemas +from db.schemas.operations.drop import drop_schema_via_oid from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -53,3 +54,18 @@ def list_(*, database_id: int, **kwargs) -> list[SchemaInfo]: # refactored the models so that each exploration is associated with a schema # (by oid) in a specific database. return [{**s, "exploration_count": 0} for s in user_defined_schemas] + + +@rpc_method(name="schemas.delete") +@http_basic_auth_login_required +@handle_rpc_exceptions +def delete(*, database_id: int, schema_id: int, **kwargs) -> None: + """ + Delete a schema, given its OID. + + Args: + database_id: The Django id of the database containing the schema. + schema_id: The OID of the schema to delete. + """ + with connect(database_id, kwargs.get(REQUEST_KEY).user) as conn: + drop_schema_via_oid(conn, schema_id) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 8240e3fc3a..b5982367fa 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -39,6 +39,11 @@ "schemas.list", [user_is_authenticated] ), + ( + schemas.delete, + "schemas.delete", + [user_is_authenticated] + ), ( tables.list_, "tables.list", From 529ffef1c1ffbfcc2e3fd9036cb53a479d6ac47e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 29 May 2024 15:23:27 -0400 Subject: [PATCH 0214/1141] Fix SQL test assertions to be order-independent --- db/sql/test_00_msar.sql | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 6b20d34c78..c01c986630 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2403,10 +2403,13 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION test_get_valid_target_type_strings() RETURNS SETOF TEXT AS $$ BEGIN PERFORM __setup_cast_functions(); - RETURN NEXT is(msar.get_valid_target_type_strings('text'), '["numeric", "text"]'::jsonb); - RETURN NEXT is( - msar.get_valid_target_type_strings('text'::regtype::oid), '["numeric", "text"]'::jsonb - ); + + RETURN NEXT ok(msar.get_valid_target_type_strings('text') @> '["numeric", "text"]'); + RETURN NEXT is(jsonb_array_length(msar.get_valid_target_type_strings('text')), 2); + + RETURN NEXT ok(msar.get_valid_target_type_strings('text'::regtype::oid) @> '["numeric", "text"]'); + RETURN NEXT is(jsonb_array_length(msar.get_valid_target_type_strings('text'::regtype::oid)), 2); + RETURN NEXT is(msar.get_valid_target_type_strings('interval'), NULL); END; $$ LANGUAGE plpgsql; From 9592f5c8aa884589e3b6ba5127dedbf4869dfc64 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 31 May 2024 15:47:35 +0530 Subject: [PATCH 0215/1141] implement tables.get rpc endpoint and sql function --- db/sql/00_msar.sql | 23 +++++++++++++++++++++++ db/tables/operations/select.py | 22 ++++++++++++++++++++++ mathesar/rpc/tables.py | 22 +++++++++++++++++++++- 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 9556b296b4..83a0c1e557 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -693,6 +693,29 @@ SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid=tab_id AND attname=col_ $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION msar.get_table(tab_id regclass) RETURNS jsonb AS $$/* +Given a table identifier, return a JSON object describing the table. + +Each returned JSON object will have the form: + { + "oid": , + "name": , + "schema": , + "description": + } + +Args: + tab_id: The OID or name of the table. +*/ +SELECT jsonb_build_object( + 'oid', oid, + 'name', relname, + 'schema', relnamespace, + 'description', msar.obj_description(oid, 'pg_class') +) FROM pg_catalog.pg_class WHERE oid = tab_id; +$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.get_table_info(sch_id regnamespace) RETURNS jsonb AS $$/* Given a schema identifier, return an array of objects describing the tables of the schema. diff --git a/db/tables/operations/select.py b/db/tables/operations/select.py index ec0871d3eb..3299d9b733 100644 --- a/db/tables/operations/select.py +++ b/db/tables/operations/select.py @@ -15,6 +15,28 @@ MULTIPLE_RESULTS = 'multiple_results' +def get_table(table, conn): + """ + Return a list of dictionaries describing the tables of a schema. + + The `table` can be given as either a "qualified name", or an OID. + The OID is the preferred identifier, since it's much more robust. + + The returned dictionary is of the following form: + + { + "oid": , + "name": , + "schema": , + "description": + } + + Args: + table: The table for which we want table info. + """ + return exec_msar_func(conn, 'get_table', table).fetchone()[0] + + def get_table_info(schema, conn): """ Return a list of dictionaries describing the tables of a schema. diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 683e60f307..5e6a54ee09 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -3,7 +3,7 @@ from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required -from db.tables.operations.select import get_table_info +from db.tables.operations.select import get_table_info, get_table from db.tables.operations.drop import drop_table_from_schema from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -47,6 +47,26 @@ def list_(*, schema_oid: int, database_id: int, **kwargs) -> list[TableInfo]: ] +@rpc_method(name="tables.get") +@http_basic_auth_login_required +@handle_rpc_exceptions +def get_(*, table_oid: int, database_id: int, **kwargs) -> TableInfo: + """ + List information about a table for a schema. Exposed as `get_`. + + Args: + schema_oid: Identity of the table in the user's database. + database_id: The Django id of the database containing the table. + + Returns: + Table details for a given table oid. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + raw_table_info = get_table(table_oid, conn) + return TableInfo(raw_table_info) + + @rpc_method(name="tables.delete") @http_basic_auth_login_required @handle_rpc_exceptions From 10f865cec04acfa2a27912d18ce149ea92d353da Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 31 May 2024 16:34:37 +0530 Subject: [PATCH 0216/1141] get_relation_name -> get_relation_name_or_null --- db/sql/00_msar.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 9556b296b4..eb8eff7ead 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2144,8 +2144,8 @@ Args: */ DECLARE relation_name text; BEGIN - relation_name := __msar.get_relation_name(tab_id); - -- if_exists doesn't work while working with oids because + relation_name := msar.get_relation_name_or_null(tab_id); + -- if_exists doesn't work while working with oids because -- the SQL query gets parameterized with tab_id instead of relation_name -- since we're unable to find the relation_name for a non existing table. PERFORM __msar.drop_table(relation_name, cascade_, if_exists => false); From 553aceee20347193479c7c87028cae4848ab3fb1 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 31 May 2024 17:21:33 +0530 Subject: [PATCH 0217/1141] refactor funtion name, and remove if_exists from docstring --- db/tables/operations/drop.py | 4 +--- mathesar/rpc/tables.py | 8 ++++---- mathesar/tests/rpc/test_tables.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/db/tables/operations/drop.py b/db/tables/operations/drop.py index 9435dd2c70..9ffeb170d2 100644 --- a/db/tables/operations/drop.py +++ b/db/tables/operations/drop.py @@ -5,15 +5,13 @@ def drop_table(name, schema, engine, cascade=False, if_exists=False): execute_msar_func_with_engine(engine, 'drop_table', schema, name, cascade, if_exists) -def drop_table_from_schema(table_oid, conn, cascade=False): +def drop_table_from_database(table_oid, conn, cascade=False): """ Drop a table. Args: table_oid: OID of the table to drop. cascade: Whether to drop the dependent objects. - if_exists: Whether to ignore an error if the schema doesn't - exist. Returns: Returns the fully qualified name of the dropped table. diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 683e60f307..6290ea9322 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -4,7 +4,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from db.tables.operations.select import get_table_info -from db.tables.operations.drop import drop_table_from_schema +from db.tables.operations.drop import drop_table_from_database from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -51,19 +51,19 @@ def list_(*, schema_oid: int, database_id: int, **kwargs) -> list[TableInfo]: @http_basic_auth_login_required @handle_rpc_exceptions def delete( - *, table_oid: int, cascade: bool = False, database_id: int, **kwargs + *, table_oid: int, database_id: int, cascade: bool = False, **kwargs ) -> str: """ Delete a table from a schema. Args: table_oid: Identity of the table in the user's database. - cascade: Whether to drop the dependent objects. database_id: The Django id of the database containing the table. + cascade: Whether to drop the dependent objects. Returns: The name of the dropped table. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: - return drop_table_from_schema(table_oid, conn, cascade) + return drop_table_from_database(table_oid, conn, cascade) diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index b4b7d42887..be8fb93275 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -86,6 +86,6 @@ def mock_drop_table(_table_oid, conn, cascade): return 'public."Table 0"' monkeypatch.setattr(tables, 'connect', mock_connect) - monkeypatch.setattr(tables, 'drop_table_from_schema', mock_drop_table) + monkeypatch.setattr(tables, 'drop_table_from_database', mock_drop_table) deleted_table = tables.delete(table_oid=1964474, database_id=11, request=request) assert deleted_table == 'public."Table 0"' From d2c7805aa3a6b1396975b9ebe973be1feeae5911 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 3 Jun 2024 10:50:53 -0400 Subject: [PATCH 0218/1141] Raise custom error code when schema OID not found --- db/sql/00_msar.sql | 3 ++- db/sql/test_00_msar.sql | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 74706fae44..22c02b8dce 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -990,7 +990,8 @@ DECLARE BEGIN SELECT nspname INTO sch_name FROM pg_namespace WHERE oid = sch_id; IF sch_name IS NULL THEN - RAISE EXCEPTION 'No schema with OID % exists.', sch_id; + RAISE EXCEPTION 'No schema with OID % exists.', sch_id + USING ERRCODE = '3F000'; -- invalid_schema_name END IF; PERFORM msar.drop_schema(sch_name, cascade_); END; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index c01c986630..566b68af24 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -1256,7 +1256,7 @@ BEGIN cascade_ => false ) $d$, - 'P0001' + '3F000' ); END; $$ LANGUAGE plpgsql; From ea51de0e422efe90918a27ef2c2b477540b4b251 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 4 Jun 2024 11:28:09 -0400 Subject: [PATCH 0219/1141] Adjust argument order --- mathesar/rpc/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mathesar/rpc/schemas.py b/mathesar/rpc/schemas.py index 0b166fcd6c..caf7f3fc43 100644 --- a/mathesar/rpc/schemas.py +++ b/mathesar/rpc/schemas.py @@ -59,13 +59,13 @@ def list_(*, database_id: int, **kwargs) -> list[SchemaInfo]: @rpc_method(name="schemas.delete") @http_basic_auth_login_required @handle_rpc_exceptions -def delete(*, database_id: int, schema_id: int, **kwargs) -> None: +def delete(*, schema_id: int, database_id: int, **kwargs) -> None: """ Delete a schema, given its OID. Args: - database_id: The Django id of the database containing the schema. schema_id: The OID of the schema to delete. + database_id: The Django id of the database containing the schema. """ with connect(database_id, kwargs.get(REQUEST_KEY).user) as conn: drop_schema_via_oid(conn, schema_id) From e601a5ecb19fd954e6e8e27c3027dd7c62155e29 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 4 Jun 2024 22:42:18 +0530 Subject: [PATCH 0220/1141] add tests and docs --- docs/docs/api/rpc.md | 1 + mathesar/tests/rpc/test_endpoints.py | 5 ++++ mathesar/tests/rpc/test_tables.py | 37 ++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 06d862f457..4d0f327e88 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -60,6 +60,7 @@ To use an RPC function: options: members: - list_ + - get_ - delete - TableInfo diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index b46af51cc0..97d1aafd25 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -44,6 +44,11 @@ "tables.list", [user_is_authenticated] ), + ( + tables.get_, + "tables.get", + [user_is_authenticated] + ), ( tables.delete, "tables.delete", diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index b4b7d42887..1b59460644 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -64,6 +64,43 @@ def mock_table_info(_schema_oid, conn): assert actual_table_list == expect_table_list +def test_tables_get(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + schema_oid = 2200 + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_table_get(_schema_oid, conn): + if _schema_oid != schema_oid: + raise AssertionError('incorrect parameters passed') + return { + 'oid': 17408, + 'name': 'Authors', + 'schema': schema_oid, + 'description': 'a description on the authors table.' + } + monkeypatch.setattr(tables, 'connect', mock_connect) + monkeypatch.setattr(tables, 'get_table', mock_table_get) + expect_table_list = { + 'oid': 17408, + 'name': 'Authors', + 'schema': schema_oid, + 'description': 'a description on the authors table.' + } + actual_table_list = tables.list_(schema_oid=2200, database_id=11, request=request) + assert actual_table_list == expect_table_list + + def test_tables_delete(rf, monkeypatch): request = rf.post('/api/rpc/v0', data={}) request.user = User(username='alice', password='pass1234') From 483c257897b9ab42602bad59b0acdaee5773bb08 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 4 Jun 2024 22:46:53 +0530 Subject: [PATCH 0221/1141] fix var name in docstring --- mathesar/rpc/tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 69ecb23e0e..a55ed6638a 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -55,7 +55,7 @@ def get_(*, table_oid: int, database_id: int, **kwargs) -> TableInfo: List information about a table for a schema. Exposed as `get_`. Args: - schema_oid: Identity of the table in the user's database. + table_oid: Identity of the table in the user's database. database_id: The Django id of the database containing the table. Returns: From 3adbb7363aaf0ba70e16386b8ef2732f833f1668 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 4 Jun 2024 23:18:30 +0530 Subject: [PATCH 0222/1141] fix test --- mathesar/tests/rpc/test_tables.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index 4c97986823..8a2e103795 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -67,7 +67,7 @@ def mock_table_info(_schema_oid, conn): def test_tables_get(rf, monkeypatch): request = rf.post('/api/rpc/v0', data={}) request.user = User(username='alice', password='pass1234') - schema_oid = 2200 + table_oid = 1964474 database_id = 11 @contextmanager @@ -80,24 +80,24 @@ def mock_connect(_database_id, user): else: raise AssertionError('incorrect parameters passed') - def mock_table_get(_schema_oid, conn): - if _schema_oid != schema_oid: + def mock_table_get(_table_oid, conn): + if _table_oid != table_oid: raise AssertionError('incorrect parameters passed') return { - 'oid': 17408, + 'oid': table_oid, 'name': 'Authors', - 'schema': schema_oid, + 'schema': 2200, 'description': 'a description on the authors table.' } monkeypatch.setattr(tables, 'connect', mock_connect) monkeypatch.setattr(tables, 'get_table', mock_table_get) expect_table_list = { - 'oid': 17408, + 'oid': table_oid, 'name': 'Authors', - 'schema': schema_oid, + 'schema': 2200, 'description': 'a description on the authors table.' } - actual_table_list = tables.list_(schema_oid=2200, database_id=11, request=request) + actual_table_list = tables.get_(table_oid=1964474, database_id=11, request=request) assert actual_table_list == expect_table_list From b028ac89fe3275d877863501203d0f2f50555ae0 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 5 Jun 2024 00:53:11 +0530 Subject: [PATCH 0223/1141] fix docsting --- db/tables/operations/select.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/tables/operations/select.py b/db/tables/operations/select.py index 3299d9b733..e25ad3c976 100644 --- a/db/tables/operations/select.py +++ b/db/tables/operations/select.py @@ -17,7 +17,7 @@ def get_table(table, conn): """ - Return a list of dictionaries describing the tables of a schema. + Return a dictionary describing a table of a schema. The `table` can be given as either a "qualified name", or an OID. The OID is the preferred identifier, since it's much more robust. From 2be3675416569f181e0c1aecd9d6eea4066d2a9b Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 5 Jun 2024 14:15:44 +0530 Subject: [PATCH 0224/1141] get_ -> get --- mathesar/rpc/tables.py | 4 ++-- mathesar/tests/rpc/test_tables.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index a55ed6638a..6a3b2f5573 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -50,9 +50,9 @@ def list_(*, schema_oid: int, database_id: int, **kwargs) -> list[TableInfo]: @rpc_method(name="tables.get") @http_basic_auth_login_required @handle_rpc_exceptions -def get_(*, table_oid: int, database_id: int, **kwargs) -> TableInfo: +def get(*, table_oid: int, database_id: int, **kwargs) -> TableInfo: """ - List information about a table for a schema. Exposed as `get_`. + List information about a table for a schema. Args: table_oid: Identity of the table in the user's database. diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index 8a2e103795..59ff818897 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -97,7 +97,7 @@ def mock_table_get(_table_oid, conn): 'schema': 2200, 'description': 'a description on the authors table.' } - actual_table_list = tables.get_(table_oid=1964474, database_id=11, request=request) + actual_table_list = tables.get(table_oid=1964474, database_id=11, request=request) assert actual_table_list == expect_table_list From 97ae1a4d48a1fede85907989861cf2aa691d0050 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 5 Jun 2024 14:23:04 +0530 Subject: [PATCH 0225/1141] fix method name in test_endpoints --- mathesar/tests/rpc/test_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 84aa897408..6fc82334c3 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -50,7 +50,7 @@ [user_is_authenticated] ), ( - tables.get_, + tables.get, "tables.get", [user_is_authenticated] ), From d922f495f33acf8d7fd665c9236c8ad7b18c186a Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 5 Jun 2024 18:17:18 +0530 Subject: [PATCH 0226/1141] add rpc method for tables.add --- db/tables/operations/create.py | 27 ++++++++++++++++++++++++++- mathesar/rpc/tables.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/db/tables/operations/create.py b/db/tables/operations/create.py index 61fc6b6729..c563585749 100644 --- a/db/tables/operations/create.py +++ b/db/tables/operations/create.py @@ -1,7 +1,7 @@ from sqlalchemy.ext import compiler from sqlalchemy.schema import DDLElement import json -from db.connection import execute_msar_func_with_engine +from db.connection import execute_msar_func_with_engine, exec_msar_func from db.types.base import PostgresType from db.tables.operations.select import reflect_table_from_oid from db.metadata import get_empty_metadata @@ -33,6 +33,31 @@ def create_mathesar_table(engine, table_name, schema_oid, columns=[], constraint ).fetchone()[0] +def create_table_on_database(table_name, schema_oid, conn, columns=[], constraints=[], comment=None): + """ + Creates a table with a default id column. + + Args: + table_name: Name of the table to be created. + schema_oid: The OID of the schema where the table will be created. + columns: The columns dict for the new table, in order. (optional) + constraints: The constraints dict for the new table. (optional) + comment: The comment for the new table. (optional) + + Returns: + Returns the OID of the created table. + """ + return exec_msar_func( + conn, + 'add_mathesar_table', + schema_oid, + table_name, + json.dumps(columns), + json.dumps(constraints), + comment + ).fetchone()[0] + + # TODO stop relying on reflections, instead return oid of the created table. def create_string_column_table(name, schema_oid, column_names, engine, comment=None): """ diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 6a3b2f5573..4e0e23d968 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -5,6 +5,7 @@ from db.tables.operations.select import get_table_info, get_table from db.tables.operations.drop import drop_table_from_database +from db.tables.operations.create import create_table_on_database from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -67,6 +68,35 @@ def get(*, table_oid: int, database_id: int, **kwargs) -> TableInfo: return TableInfo(raw_table_info) +@rpc_method(name="tables.add") +@http_basic_auth_login_required +@handle_rpc_exceptions +def add( + *, table_name: str, schema_oid: int, database_id: int, + columns: list[dict] = [], constraints: list[dict] = [], comment: str = None, **kwargs +) -> int: + """ + Add a table with a default id column. + + Args: + table_name: Name of the table to be created. + schema_oid: The oid of the schema where the table will be created. + database_id: The Django id of the database containing the table. + columns: A list of columns dict for the new table, in order. + constraints: A list of constraints dict for the new table. + comment: The comment for the new table. + + Returns: + Table oid of the created table. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + created_table_oid = create_table_on_database( + table_name, schema_oid, conn, columns, constraints, comment + ) + return created_table_oid + + @rpc_method(name="tables.delete") @http_basic_auth_login_required @handle_rpc_exceptions From 88b8e20f2e634bba2d8241f6723e55b7dd484871 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 5 Jun 2024 18:37:55 +0530 Subject: [PATCH 0227/1141] add endpoint test & docs --- docs/docs/api/rpc.md | 3 ++- mathesar/rpc/tables.py | 4 ++-- mathesar/tests/rpc/test_endpoints.py | 5 +++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 4c6f622548..ab1e996d61 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -61,7 +61,8 @@ To use an RPC function: options: members: - list_ - - get_ + - get + - add - delete - TableInfo diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 4e0e23d968..bf3af75513 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -80,14 +80,14 @@ def add( Args: table_name: Name of the table to be created. - schema_oid: The oid of the schema where the table will be created. + schema_oid: Identity of the schema in the user's database. database_id: The Django id of the database containing the table. columns: A list of columns dict for the new table, in order. constraints: A list of constraints dict for the new table. comment: The comment for the new table. Returns: - Table oid of the created table. + The `oid` of the created table. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 6fc82334c3..b9e4b01d13 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -54,6 +54,11 @@ "tables.get", [user_is_authenticated] ), + ( + tables.add, + "tables.add", + [user_is_authenticated] + ), ( tables.delete, "tables.delete", From 9a50533eb92c11193731fee6db7dbeca635cc3c7 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 6 Jun 2024 13:58:48 +0800 Subject: [PATCH 0228/1141] add columns.patch rpc function --- docs/docs/api/rpc.md | 2 ++ mathesar/rpc/columns.py | 70 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index b7a594b3c8..9c86863248 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -60,9 +60,11 @@ To use an RPC function: options: members: - list_ + - patch - delete - ColumnListReturn - ColumnInfo + - SettableColumnInfo - TypeOptions - ColumnDefault diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns.py index 017a780204..cf30b8faa1 100644 --- a/mathesar/rpc/columns.py +++ b/mathesar/rpc/columns.py @@ -1,13 +1,14 @@ """ Classes and functions exposed to the RPC endpoint for managing table columns. """ -from typing import TypedDict +from typing import Optional, TypedDict from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required -from db.columns.operations.select import get_column_info_for_table +from db.columns.operations.alter import alter_columns_in_table from db.columns.operations.drop import drop_columns_from_table +from db.columns.operations.select import get_column_info_for_table from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect from mathesar.utils.columns import get_raw_display_options @@ -73,9 +74,43 @@ def from_dict(cls, col_default): ) +class SettableColumnInfo(TypedDict): + """ + Information about a column, restricted to settable fields. + + When possible, Passing `null` for a key will clear the underlying + setting. E.g., + + - `default = null` clears the column default setting. + - `type_options = null` clears the type options for the column. + - `description = null` clears the column description. + + Setting any of `name`, `type`, or `nullable` is a noop. + + + Only the `id` key is required. + + Attributes: + id: The `attnum` of the column in the table. + name: The name of the column. + type: The type of the column on the database. + type_options: The options applied to the column type. + nullable: Whether or not the column is nullable. + default: The default value. + description: The description of the column. + """ + id: int + name: Optional[str] + type: Optional[str] + type_options: Optional[TypeOptions] + nullable: Optional[bool] + default: Optional[ColumnDefault] + description: Optional[str] + + class ColumnInfo(TypedDict): """ - Information about a column. + Information about a column. Extends the settable fields. Attributes: id: The `attnum` of the column in the table. @@ -158,6 +193,35 @@ def list_(*, table_oid: int, database_id: int, **kwargs) -> ColumnListReturn: ) +@rpc_method(name="columns.patch") +@http_basic_auth_login_required +@handle_rpc_exceptions +def patch( + *, + column_data_list: list[SettableColumnInfo], + table_oid: int, + database_id: int, + **kwargs +) -> int: + """ + Alter details of preexisting columns in a table. + + Does not support altering the type or type options of array columns. + + Args: + column_data_list: A list describing desired column alterations. + table_oid: Identity of the table whose columns we'll modify. + database_id: The Django id of the database containing the table. + + Returns: + The number of columns altered. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + return alter_columns_in_table(table_oid, column_data_list, conn) + + + @rpc_method(name="columns.delete") @http_basic_auth_login_required @handle_rpc_exceptions From 7d88dbb3d49791ba766e14e19633b89c2307a3bb Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 6 Jun 2024 15:20:14 +0800 Subject: [PATCH 0229/1141] add tests for wiring --- mathesar/tests/rpc/test_columns.py | 33 ++++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 5 +++++ 2 files changed, 38 insertions(+) diff --git a/mathesar/tests/rpc/test_columns.py b/mathesar/tests/rpc/test_columns.py index 8f20811ecf..ea9928aca0 100644 --- a/mathesar/tests/rpc/test_columns.py +++ b/mathesar/tests/rpc/test_columns.py @@ -138,6 +138,39 @@ def mock_display_options(_database_id, _table_oid, attnums, user): assert actual_col_list == expect_col_list +def test_columns_patch(rf, monkeypatch): + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username='alice', password='pass1234') + table_oid = 23457 + database_id = 2 + column_data_list = [{"id": 3, "name": "newname"}] + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == 2 and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_column_alter(_table_oid, _column_data_list, conn): + if _table_oid != table_oid or _column_data_list != column_data_list: + raise AssertionError('incorrect parameters passed') + return 1 + + monkeypatch.setattr(columns, 'connect', mock_connect) + monkeypatch.setattr(columns, 'alter_columns_in_table', mock_column_alter) + actual_result = columns.patch( + column_data_list=column_data_list, + table_oid=table_oid, + database_id=database_id, + request=request + ) + assert actual_result == 1 + + def test_columns_delete(rf, monkeypatch): request = rf.post('/api/rpc/v0/', data={}) request.user = User(username='alice', password='pass1234') diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 6fc82334c3..8f8d2bd29f 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -24,6 +24,11 @@ "columns.list", [user_is_authenticated] ), + ( + columns.patch, + "columns.patch", + [user_is_authenticated] + ), ( connections.add_from_known_connection, "connections.add_from_known_connection", From ecc3df6b6aa23c84ea98116ac8374d3eba410059 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 6 Jun 2024 15:22:33 +0800 Subject: [PATCH 0230/1141] fix linter errors --- mathesar/rpc/columns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns.py index cf30b8faa1..860d7aaa6d 100644 --- a/mathesar/rpc/columns.py +++ b/mathesar/rpc/columns.py @@ -221,7 +221,6 @@ def patch( return alter_columns_in_table(table_oid, column_data_list, conn) - @rpc_method(name="columns.delete") @http_basic_auth_login_required @handle_rpc_exceptions From 5489684eb0fd75535de3c66137324a22b0baccad Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 6 Jun 2024 14:17:09 +0530 Subject: [PATCH 0231/1141] Fix regression: Table inspector tabs do not get highlighted based on column/row selection --- .../systems/table-view/table-inspector/TableInspector.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar_ui/src/systems/table-view/table-inspector/TableInspector.svelte b/mathesar_ui/src/systems/table-view/table-inspector/TableInspector.svelte index d044260b97..870fab24d7 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/TableInspector.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/TableInspector.svelte @@ -21,7 +21,7 @@ export let activeTabId: TableInspectorTabId | undefined; $: tabs = Object.entries(tabMap).map(([id, tab]) => ({ id, ...tab })); - $: activeTab = defined(activeTabId, (id) => tabMap[id]); + $: activeTab = defined(activeTabId, (id) => ({ id, ...tabMap[id] })); function handleTabSelected(e: CustomEvent<{ tab: Tab }>) { activeTabId = e.detail.tab.id as TableInspectorTabId; From 5945d6518dae02166af2f77b471660354e76acde Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 11 Jun 2024 01:17:39 +0530 Subject: [PATCH 0232/1141] improve function signature --- db/tables/operations/create.py | 13 ++++++-- mathesar/rpc/columns.py | 22 +++++++++++++ mathesar/rpc/constraints.py | 8 +++++ mathesar/rpc/tables.py | 18 ++++++++--- mathesar/tests/rpc/test_tables.py | 53 +++++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 mathesar/rpc/constraints.py diff --git a/db/tables/operations/create.py b/db/tables/operations/create.py index c563585749..e25a05c12c 100644 --- a/db/tables/operations/create.py +++ b/db/tables/operations/create.py @@ -33,7 +33,14 @@ def create_mathesar_table(engine, table_name, schema_oid, columns=[], constraint ).fetchone()[0] -def create_table_on_database(table_name, schema_oid, conn, columns=[], constraints=[], comment=None): +def create_table_on_database( + conn, + table_name, + schema_oid, + column_data_list=[], + constraint_data_list=[], + comment=None +): """ Creates a table with a default id column. @@ -52,8 +59,8 @@ def create_table_on_database(table_name, schema_oid, conn, columns=[], constrain 'add_mathesar_table', schema_oid, table_name, - json.dumps(columns), - json.dumps(constraints), + json.dumps(column_data_list), + json.dumps(constraint_data_list), comment ).fetchone()[0] diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns.py index 860d7aaa6d..54ad70896a 100644 --- a/mathesar/rpc/columns.py +++ b/mathesar/rpc/columns.py @@ -74,6 +74,28 @@ def from_dict(cls, col_default): ) +class CreateableColumnInfo(TypedDict): + """ + Information about adding a new column. + + Only the `name` & `type` keys are required. + + Attributes: + name: The name of the column. + type: The type of the column on the database. + type_options: The options applied to the column type. + nullable: Whether or not the column is nullable. + default: The default value. + description: The description of the column. + """ + name: str + type: str + type_options: Optional[TypeOptions] + nullable: Optional[bool] + default: Optional[ColumnDefault] + description: Optional[str] + + class SettableColumnInfo(TypedDict): """ Information about a column, restricted to settable fields. diff --git a/mathesar/rpc/constraints.py b/mathesar/rpc/constraints.py new file mode 100644 index 0000000000..b8ebbdabc4 --- /dev/null +++ b/mathesar/rpc/constraints.py @@ -0,0 +1,8 @@ +""" +Classes and functions exposed to the RPC endpoint for managing table constraints. +""" +from typing import TypedDict + + +class CreateableConstraintInfo(TypedDict): + pass diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index bf3af75513..72fd87c519 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -6,6 +6,8 @@ from db.tables.operations.select import get_table_info, get_table from db.tables.operations.drop import drop_table_from_database from db.tables.operations.create import create_table_on_database +from mathesar.rpc.columns import CreateableColumnInfo +from mathesar.rpc.constraints import CreateableConstraintInfo from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -72,8 +74,14 @@ def get(*, table_oid: int, database_id: int, **kwargs) -> TableInfo: @http_basic_auth_login_required @handle_rpc_exceptions def add( - *, table_name: str, schema_oid: int, database_id: int, - columns: list[dict] = [], constraints: list[dict] = [], comment: str = None, **kwargs + *, + table_name: str, + schema_oid: int, + database_id: int, + column_data_list: list[CreateableColumnInfo] = [], + constraint_data_list: list[CreateableConstraintInfo] = [], + comment: str = None, + **kwargs ) -> int: """ Add a table with a default id column. @@ -82,8 +90,8 @@ def add( table_name: Name of the table to be created. schema_oid: Identity of the schema in the user's database. database_id: The Django id of the database containing the table. - columns: A list of columns dict for the new table, in order. - constraints: A list of constraints dict for the new table. + column_data_list: A list describing columns to be created for the new table, in order. + constraint_data_list: A list describing constraints to be created for the new table. comment: The comment for the new table. Returns: @@ -92,7 +100,7 @@ def add( user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: created_table_oid = create_table_on_database( - table_name, schema_oid, conn, columns, constraints, comment + conn, table_name, schema_oid, column_data_list, constraint_data_list, comment ) return created_table_oid diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index 59ff818897..0771077d45 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -126,3 +126,56 @@ def mock_drop_table(_table_oid, conn, cascade): monkeypatch.setattr(tables, 'drop_table_from_database', mock_drop_table) deleted_table = tables.delete(table_oid=1964474, database_id=11, request=request) assert deleted_table == 'public."Table 0"' + + +def test_tables_add(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + schema_oid = 2200 + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_table_add(_schema_oid, conn): + if _schema_oid != schema_oid: + raise AssertionError('incorrect parameters passed') + return [ + { + 'oid': 17408, + 'name': 'Authors', + 'schema': schema_oid, + 'description': 'a description on the authors table.' + }, + { + 'oid': 17809, + 'name': 'Books', + 'schema': schema_oid, + 'description': None + } + ] + monkeypatch.setattr(tables, 'connect', mock_connect) + monkeypatch.setattr(tables, 'create_table_on_database', mock_table_add) + expect_table_list = [ + { + 'oid': 17408, + 'name': 'Authors', + 'schema': schema_oid, + 'description': 'a description on the authors table.' + }, + { + 'oid': 17809, + 'name': 'Books', + 'schema': schema_oid, + 'description': None + } + ] + actual_table_list = tables.add(schema_oid=2200, database_id=11, request=request) + assert actual_table_list == expect_table_list \ No newline at end of file From 4234b01d9f1e7bc4ee19973ded706bae5ef1fd0f Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 11 Jun 2024 01:44:16 +0530 Subject: [PATCH 0233/1141] refactor test --- mathesar/tests/rpc/test_tables.py | 33 +++---------------------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index 0771077d45..673fe5711b 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -147,35 +147,8 @@ def mock_connect(_database_id, user): def mock_table_add(_schema_oid, conn): if _schema_oid != schema_oid: raise AssertionError('incorrect parameters passed') - return [ - { - 'oid': 17408, - 'name': 'Authors', - 'schema': schema_oid, - 'description': 'a description on the authors table.' - }, - { - 'oid': 17809, - 'name': 'Books', - 'schema': schema_oid, - 'description': None - } - ] + return 1964474 monkeypatch.setattr(tables, 'connect', mock_connect) monkeypatch.setattr(tables, 'create_table_on_database', mock_table_add) - expect_table_list = [ - { - 'oid': 17408, - 'name': 'Authors', - 'schema': schema_oid, - 'description': 'a description on the authors table.' - }, - { - 'oid': 17809, - 'name': 'Books', - 'schema': schema_oid, - 'description': None - } - ] - actual_table_list = tables.add(schema_oid=2200, database_id=11, request=request) - assert actual_table_list == expect_table_list \ No newline at end of file + actual_table_oid = tables.add(table_name='newtable', schema_oid=2200, database_id=11, request=request) + assert actual_table_oid == 1964474 From fe0835bd27cd6814d90c172f214227eb555e9fa0 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 11 Jun 2024 02:04:24 +0530 Subject: [PATCH 0234/1141] fix tests --- db/tables/operations/create.py | 2 +- mathesar/rpc/tables.py | 2 +- mathesar/tests/rpc/test_tables.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/tables/operations/create.py b/db/tables/operations/create.py index e25a05c12c..afbee8b32f 100644 --- a/db/tables/operations/create.py +++ b/db/tables/operations/create.py @@ -34,9 +34,9 @@ def create_mathesar_table(engine, table_name, schema_oid, columns=[], constraint def create_table_on_database( - conn, table_name, schema_oid, + conn, column_data_list=[], constraint_data_list=[], comment=None diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 72fd87c519..ec586a4129 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -100,7 +100,7 @@ def add( user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: created_table_oid = create_table_on_database( - conn, table_name, schema_oid, column_data_list, constraint_data_list, comment + table_name, schema_oid, conn, column_data_list, constraint_data_list, comment ) return created_table_oid diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index 673fe5711b..27b678cc54 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -144,7 +144,7 @@ def mock_connect(_database_id, user): else: raise AssertionError('incorrect parameters passed') - def mock_table_add(_schema_oid, conn): + def mock_table_add(table_name, _schema_oid, conn, column_data_list, constraint_data_list, comment): if _schema_oid != schema_oid: raise AssertionError('incorrect parameters passed') return 1964474 From d97f6a80a645eba4784390c48fd320c95bc71a15 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 11 Jun 2024 15:31:23 +0800 Subject: [PATCH 0235/1141] fix class name spelling --- mathesar/rpc/columns.py | 4 ++-- mathesar/rpc/constraints.py | 2 +- mathesar/rpc/tables.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns.py index 54ad70896a..537e48dd38 100644 --- a/mathesar/rpc/columns.py +++ b/mathesar/rpc/columns.py @@ -74,9 +74,9 @@ def from_dict(cls, col_default): ) -class CreateableColumnInfo(TypedDict): +class CreatableColumnInfo(TypedDict): """ - Information about adding a new column. + Information needed to add a new column. Only the `name` & `type` keys are required. diff --git a/mathesar/rpc/constraints.py b/mathesar/rpc/constraints.py index b8ebbdabc4..fe726464d0 100644 --- a/mathesar/rpc/constraints.py +++ b/mathesar/rpc/constraints.py @@ -4,5 +4,5 @@ from typing import TypedDict -class CreateableConstraintInfo(TypedDict): +class CreatableConstraintInfo(TypedDict): pass diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index ec586a4129..8c6d4575a3 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -6,8 +6,8 @@ from db.tables.operations.select import get_table_info, get_table from db.tables.operations.drop import drop_table_from_database from db.tables.operations.create import create_table_on_database -from mathesar.rpc.columns import CreateableColumnInfo -from mathesar.rpc.constraints import CreateableConstraintInfo +from mathesar.rpc.columns import CreatableColumnInfo +from mathesar.rpc.constraints import CreatableConstraintInfo from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -78,8 +78,8 @@ def add( table_name: str, schema_oid: int, database_id: int, - column_data_list: list[CreateableColumnInfo] = [], - constraint_data_list: list[CreateableConstraintInfo] = [], + column_data_list: list[CreatableColumnInfo] = [], + constraint_data_list: list[CreatableConstraintInfo] = [], comment: str = None, **kwargs ) -> int: From 9510b8c4cc2631772215b4e192b9a59b7e15e7ee Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 11 Jun 2024 16:40:08 +0800 Subject: [PATCH 0236/1141] add python wrapper for new column creation --- db/columns/operations/create.py | 78 ++++++++++++++-------- db/tests/columns/operations/test_create.py | 8 +-- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/db/columns/operations/create.py b/db/columns/operations/create.py index 7361276015..da18be26a8 100644 --- a/db/columns/operations/create.py +++ b/db/columns/operations/create.py @@ -5,39 +5,18 @@ from alembic.operations import Operations from psycopg.errors import InvalidTextRepresentation, InvalidParameterValue -from db.columns.defaults import DEFAULT, NAME, NULLABLE, TYPE, DESCRIPTION +from db import connection as db_conn +from db.columns.defaults import DEFAULT, NAME, NULLABLE, DESCRIPTION from db.columns.exceptions import InvalidDefaultError, InvalidTypeOptionError -from db.connection import execute_msar_func_with_engine from db.tables.operations.select import reflect_table_from_oid from db.types.base import PostgresType from db.metadata import get_empty_metadata def create_column(engine, table_oid, column_data): - column_name = (column_data.get(NAME) or '').strip() or None - column_type_id = ( - column_data.get( - # TYPE = 'sa_type'. This is coming straight from the API. - # TODO Determine whether we actually need 'sa_type' and 'type' - TYPE, column_data.get("type") - ) - or PostgresType.CHARACTER_VARYING.id - ) - column_type_options = column_data.get("type_options", {}) - column_nullable = column_data.get(NULLABLE, True) - default_value = column_data.get(DEFAULT, {}).get('value') - column_description = column_data.get(DESCRIPTION) - col_create_def = [ - { - "name": column_name, - "type": {"name": column_type_id, "options": column_type_options}, - "not_null": not column_nullable, - "default": default_value, - "description": column_description, - } - ] + col_create_def = [_transform_column_create_dict(column_data)] try: - curr = execute_msar_func_with_engine( + curr = db_conn.execute_msar_func_with_engine( engine, 'add_columns', table_oid, json.dumps(col_create_def) @@ -49,6 +28,53 @@ def create_column(engine, table_oid, column_data): return curr.fetchone()[0] +def add_columns_to_table(table_oid, column_data_list, conn): + transformed_column_data = [ + _transform_column_create_dict(col) for col in column_data_list + ] + result = db_conn.exec_msar_func( + conn, 'add_columns', table_oid, json.dumps(transformed_column_data) + ).fetchone()[0] + return result + + +# TODO This function wouldn't be needed if we had the same form in the DB +# as the RPC API function. +def _transform_column_create_dict(data): + """ + Transform the data dict into the form needed for the DB functions. + + Input data form: + { + "name": , + "type": , + "type_options": , + "nullable": , + "default": {"value": } + "description": + } + + Output form: + { + "type": {"name": , "options": }, + "name": , + "not_null": , + "default": , + "description": + } + """ + return { + "name": (data.get(NAME) or '').strip() or None, + "type": { + "name": data.get("type") or PostgresType.CHARACTER_VARYING.id, + "options": data.get("type_options", {}) + }, + "not_null": not data.get(NULLABLE, True), + "default": data.get(DEFAULT, {}).get('value'), + "description": data.get(DESCRIPTION), + } + + def bulk_create_mathesar_column(engine, table_oid, columns, schema): # TODO reuse metadata table = reflect_table_from_oid(table_oid, engine, metadata=get_empty_metadata()) @@ -67,7 +93,7 @@ def duplicate_column( copy_data=True, copy_constraints=True ): - curr = execute_msar_func_with_engine( + curr = db_conn.execute_msar_func_with_engine( engine, 'copy_column', table_oid, diff --git a/db/tests/columns/operations/test_create.py b/db/tests/columns/operations/test_create.py index b668e71bf6..cc736a97ff 100644 --- a/db/tests/columns/operations/test_create.py +++ b/db/tests/columns/operations/test_create.py @@ -29,7 +29,7 @@ def test_create_column_name(engine_with_schema, in_name, out_name): given a (maybe empty) name param """ engine, schema = engine_with_schema - with patch.object(col_create, "execute_msar_func_with_engine") as mock_exec: + with patch.object(col_create.db_conn, "execute_msar_func_with_engine") as mock_exec: col_create.create_column(engine, 12345, {"name": in_name}) call_args = mock_exec.call_args_list[0][0] assert call_args[0] == engine @@ -47,7 +47,7 @@ def test_create_column_type(engine_with_schema, in_type, out_type): given a (maybe empty) type """ engine, schema = engine_with_schema - with patch.object(col_create, "execute_msar_func_with_engine") as mock_exec: + with patch.object(col_create.db_conn, "execute_msar_func_with_engine") as mock_exec: col_create.create_column(engine, 12345, {"type": in_type}) call_args = mock_exec.call_args_list[0][0] assert call_args[0] == engine @@ -68,7 +68,7 @@ def test_create_column_type_options(engine_with_schema, in_options, out_options) given a (maybe empty) type options dict. """ engine, schema = engine_with_schema - with patch.object(col_create, "execute_msar_func_with_engine") as mock_exec: + with patch.object(col_create.db_conn, "execute_msar_func_with_engine") as mock_exec: col_create.create_column(engine, 12345, {"type_options": in_options}) call_args = mock_exec.call_args_list[0][0] assert call_args[0] == engine @@ -81,7 +81,7 @@ def test_create_column_type_options(engine_with_schema, in_options, out_options) def test_duplicate_column_smoke(engine_with_schema): """This is just a smoke test, since the underlying function is trivial.""" engine, schema = engine_with_schema - with patch.object(col_create, "execute_msar_func_with_engine") as mock_exec: + with patch.object(col_create.db_conn, "execute_msar_func_with_engine") as mock_exec: col_create.duplicate_column( 12345, 4, From 787a07db450a7a0962245242342dacc85b28c3a6 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 11 Jun 2024 16:59:43 +0800 Subject: [PATCH 0237/1141] reuse column creation tests for new function --- db/tests/columns/operations/test_create.py | 35 ++++++++++------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/db/tests/columns/operations/test_create.py b/db/tests/columns/operations/test_create.py index cc736a97ff..a248fd5d49 100644 --- a/db/tests/columns/operations/test_create.py +++ b/db/tests/columns/operations/test_create.py @@ -21,38 +21,36 @@ def test_type_list_completeness(engine): @pytest.mark.parametrize( - "in_name,out_name", [('test1', 'test1'), ('', None), (None, None)] + "in_name,out_name", [("test1", "test1"), ("", None), (None, None)] ) -def test_create_column_name(engine_with_schema, in_name, out_name): +def test_add_columns_name(in_name, out_name): """ Here, we just check that the PostgreSQL function is called properly, when given a (maybe empty) name param """ - engine, schema = engine_with_schema - with patch.object(col_create.db_conn, "execute_msar_func_with_engine") as mock_exec: - col_create.create_column(engine, 12345, {"name": in_name}) + with patch.object(col_create.db_conn, "exec_msar_func") as mock_exec: + col_create.add_columns_to_table(123, [{"name": in_name}], "conn") call_args = mock_exec.call_args_list[0][0] - assert call_args[0] == engine + assert call_args[0] == "conn" assert call_args[1] == "add_columns" - assert call_args[2] == 12345 + assert call_args[2] == 123 assert json.loads(call_args[3])[0]["name"] == out_name @pytest.mark.parametrize( "in_type,out_type", [("numeric", "numeric"), (None, "character varying")] ) -def test_create_column_type(engine_with_schema, in_type, out_type): +def test_add_columns_type(in_type, out_type): """ Here, we just check that the PostgreSQL function is called properly when given a (maybe empty) type """ - engine, schema = engine_with_schema - with patch.object(col_create.db_conn, "execute_msar_func_with_engine") as mock_exec: - col_create.create_column(engine, 12345, {"type": in_type}) + with patch.object(col_create.db_conn, "exec_msar_func") as mock_exec: + col_create.add_columns_to_table(123, [{"type": in_type}], "conn") call_args = mock_exec.call_args_list[0][0] - assert call_args[0] == engine + assert call_args[0] == "conn" assert call_args[1] == "add_columns" - assert call_args[2] == 12345 + assert call_args[2] == 123 actual_col_data = json.loads(call_args[3])[0] assert actual_col_data["name"] is None assert actual_col_data["type"]["name"] == out_type @@ -62,18 +60,17 @@ def test_create_column_type(engine_with_schema, in_type, out_type): @pytest.mark.parametrize( "in_options,out_options", [({"foo": "bar"}, {"foo": "bar"}), (None, None), ({}, {})] ) -def test_create_column_type_options(engine_with_schema, in_options, out_options): +def test_add_columns_type_options(in_options, out_options): """ Here, we just check that the PostgreSQL function is called properly when given a (maybe empty) type options dict. """ - engine, schema = engine_with_schema - with patch.object(col_create.db_conn, "execute_msar_func_with_engine") as mock_exec: - col_create.create_column(engine, 12345, {"type_options": in_options}) + with patch.object(col_create.db_conn, "exec_msar_func") as mock_exec: + col_create.add_columns_to_table(123, [{"type_options": in_options}], "conn") call_args = mock_exec.call_args_list[0][0] - assert call_args[0] == engine + assert call_args[0] == "conn" assert call_args[1] == "add_columns" - assert call_args[2] == 12345 + assert call_args[2] == 123 assert json.loads(call_args[3])[0]["type"]["name"] == "character varying" assert json.loads(call_args[3])[0]["type"]["options"] == out_options From 9907d0417a8acbc050bfa90e30d9b7ade1249056 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 11 Jun 2024 17:28:02 +0800 Subject: [PATCH 0238/1141] wire column adding up to RPC endpoint --- mathesar/rpc/columns.py | 35 +++++++++++++++++++++++++--- mathesar/tests/rpc/test_columns.py | 33 ++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 5 ++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns.py index 537e48dd38..a4f7d00ca4 100644 --- a/mathesar/rpc/columns.py +++ b/mathesar/rpc/columns.py @@ -7,6 +7,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from db.columns.operations.alter import alter_columns_in_table +from db.columns.operations.create import add_columns_to_table from db.columns.operations.drop import drop_columns_from_table from db.columns.operations.select import get_column_info_for_table from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -78,7 +79,7 @@ class CreatableColumnInfo(TypedDict): """ Information needed to add a new column. - Only the `name` & `type` keys are required. + No keys are required. Attributes: name: The name of the column. @@ -88,8 +89,8 @@ class CreatableColumnInfo(TypedDict): default: The default value. description: The description of the column. """ - name: str - type: str + name: Optional[str] + type: Optional[str] type_options: Optional[TypeOptions] nullable: Optional[bool] default: Optional[ColumnDefault] @@ -215,6 +216,34 @@ def list_(*, table_oid: int, database_id: int, **kwargs) -> ColumnListReturn: ) +@rpc_method(name="columns.add") +@http_basic_auth_login_required +@handle_rpc_exceptions +def add( + *, + column_data_list: list[CreatableColumnInfo], + table_oid: int, + database_id: int, + **kwargs +) -> list[int]: + """ + Alter details of preexisting columns in a table. + + Does not support altering the type or type options of array columns. + + Args: + column_data_list: A list describing desired column alterations. + table_oid: Identity of the table whose columns we'll modify. + database_id: The Django id of the database containing the table. + + Returns: + The number of columns altered. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + return add_columns_to_table(table_oid, column_data_list, conn) + + @rpc_method(name="columns.patch") @http_basic_auth_login_required @handle_rpc_exceptions diff --git a/mathesar/tests/rpc/test_columns.py b/mathesar/tests/rpc/test_columns.py index ea9928aca0..1afd813a33 100644 --- a/mathesar/tests/rpc/test_columns.py +++ b/mathesar/tests/rpc/test_columns.py @@ -171,6 +171,39 @@ def mock_column_alter(_table_oid, _column_data_list, conn): assert actual_result == 1 +def test_columns_add(rf, monkeypatch): + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username='alice', password='pass1234') + table_oid = 23457 + database_id = 2 + column_data_list = [{"id": 3, "name": "newname"}] + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == 2 and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_column_create(_table_oid, _column_data_list, conn): + if _table_oid != table_oid or _column_data_list != column_data_list: + raise AssertionError('incorrect parameters passed') + return [3, 4] + + monkeypatch.setattr(columns, 'connect', mock_connect) + monkeypatch.setattr(columns, 'add_columns_to_table', mock_column_create) + actual_result = columns.add( + column_data_list=column_data_list, + table_oid=table_oid, + database_id=database_id, + request=request + ) + assert actual_result == [3, 4] + + def test_columns_delete(rf, monkeypatch): request = rf.post('/api/rpc/v0/', data={}) request.user = User(username='alice', password='pass1234') diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 6542295a72..6db4a98248 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -29,6 +29,11 @@ "columns.patch", [user_is_authenticated] ), + ( + columns.add, + "columns.add", + [user_is_authenticated] + ), ( connections.add_from_known_connection, "connections.add_from_known_connection", From fb1501eed46f2f8c10ff03262182f50aeb7a00aa Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 11 Jun 2024 17:33:13 +0800 Subject: [PATCH 0239/1141] update docs for new column adder RPC function --- docs/docs/api/rpc.md | 1 + mathesar/rpc/columns.py | 10 ++++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index d749847879..9366a78156 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -72,6 +72,7 @@ To use an RPC function: options: members: - list_ + - add - patch - delete - ColumnListReturn diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns.py index a4f7d00ca4..6a74a6f329 100644 --- a/mathesar/rpc/columns.py +++ b/mathesar/rpc/columns.py @@ -227,17 +227,15 @@ def add( **kwargs ) -> list[int]: """ - Alter details of preexisting columns in a table. - - Does not support altering the type or type options of array columns. + Add columns to a table. Args: - column_data_list: A list describing desired column alterations. - table_oid: Identity of the table whose columns we'll modify. + column_data_list: A list describing desired columns to add. + table_oid: Identity of the table to which we'll add columns. database_id: The Django id of the database containing the table. Returns: - The number of columns altered. + An array of the attnums of the new columns. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: From 899fb921153e90db0f2356f86fff262abc98acb5 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 11 Jun 2024 17:36:11 +0800 Subject: [PATCH 0240/1141] document db library python function --- db/columns/operations/create.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/db/columns/operations/create.py b/db/columns/operations/create.py index da18be26a8..ddaf29d4b3 100644 --- a/db/columns/operations/create.py +++ b/db/columns/operations/create.py @@ -29,6 +29,17 @@ def create_column(engine, table_oid, column_data): def add_columns_to_table(table_oid, column_data_list, conn): + """ + Add columns to the given table. + + For a description of the members of column_data_list, see + _transform_column_create_dict + + Args: + table_oid: The OID of the table whose columns we'll alter. + column_data_list: A list of dicts describing columns to add. + conn: A psycopg connection. + """ transformed_column_data = [ _transform_column_create_dict(col) for col in column_data_list ] From 03247651fd25319c9ccce139f719a0aca4d46905 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 12 Jun 2024 00:25:01 +0530 Subject: [PATCH 0241/1141] implement table.patch with docs and tests --- db/sql/00_msar.sql | 19 ++++++++++++- db/tables/operations/alter.py | 20 ++++++++++++++ docs/docs/api/rpc.md | 2 ++ mathesar/rpc/tables.py | 29 +++++++++++++++++++- mathesar/tests/rpc/test_endpoints.py | 5 ++++ mathesar/tests/rpc/test_tables.py | 40 ++++++++++++++++++++++++++++ 6 files changed, 113 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index a18dabb1c4..fbc8dcaf1a 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1121,7 +1121,24 @@ SELECT __msar.comment_on_table( $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; --- Alter Table: LEFT IN PYTHON (for now) ----------------------------------------------------------- +-- Alter table ------------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION +msar.alter_table(tab_id oid, tab_alters jsonb) RETURNS text AS $$ +DECLARE + new_tab_name text; + comment text; + col_alters jsonb; +BEGIN + new_tab_name := tab_alters->>'name'; + comment := tab_alters->>'description'; + col_alters := tab_alters->>'columns'; + PERFORM msar.rename_table(tab_id, new_tab_name); + PERFORM msar.comment_on_table(tab_id, comment); + PERFORM msar.alter_columns(tab_id, col_alters); + RETURN tab_id::regclass::text; +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- diff --git a/db/tables/operations/alter.py b/db/tables/operations/alter.py index 7ecb160163..d1d554bc29 100644 --- a/db/tables/operations/alter.py +++ b/db/tables/operations/alter.py @@ -50,6 +50,26 @@ def alter_table(table_name, table_oid, schema, engine, update_data): batch_update_columns(table_oid, engine, update_data['columns']) +def alter_table_on_database(table_oid, table_data_dict, conn): + """ + Alter the name, description, or columns of a table, returning name of the altered table. + + Args: + table_oid: The OID of the table to be altered. + table_data_dict: A dict describing the alterations to make. + + table_data_dict should have the form: + { + "name": , + "description": , + "columns": of column_data describing columns to alter. + } + """ + return db_conn.exec_msar_func( + conn, 'alter_table', table_oid, table_data_dict + ).fetchone()[0] + + def update_pk_sequence_to_latest(engine, table, connection=None): """ Update the primary key sequence to the current maximum. diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index d749847879..c93d6e7db0 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -64,7 +64,9 @@ To use an RPC function: - get - add - delete + - patch - TableInfo + - SettableTableInfo --- diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index ec586a4129..09a243ecaf 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -6,7 +6,8 @@ from db.tables.operations.select import get_table_info, get_table from db.tables.operations.drop import drop_table_from_database from db.tables.operations.create import create_table_on_database -from mathesar.rpc.columns import CreateableColumnInfo +from db.tables.operations.alter import alter_table_on_database +from mathesar.rpc.columns import CreateableColumnInfo, SettableColumnInfo from mathesar.rpc.constraints import CreateableConstraintInfo from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -28,6 +29,12 @@ class TableInfo(TypedDict): description: Optional[str] +class SettableTableInfo(TypedDict): + name: Optional[str] + description: Optional[str] + columns: Optional[list[SettableColumnInfo]] + + @rpc_method(name="tables.list") @http_basic_auth_login_required @handle_rpc_exceptions @@ -125,3 +132,23 @@ def delete( user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: return drop_table_from_database(table_oid, conn, cascade) + + +@rpc_method(name="tables.patch") +@http_basic_auth_login_required +@handle_rpc_exceptions +def patch(*, table_oid: str, table_data_dict: SettableTableInfo, database_id: int, **kwargs): + """ + Alter details of preexisting tables in a database. + + Args: + table_oid: Identity of the table whose name, description or columns we'll modify. + table_data_dict: A list describing desired table alterations. + database_id: The Django id of the database containing the table. + + Returns: + The name of the altered table. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + return alter_table_on_database(table_oid, table_data_dict, conn) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 6542295a72..3644b808b5 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -68,6 +68,11 @@ tables.delete, "tables.delete", [user_is_authenticated] + ), + ( + tables.patch, + "tables.patch", + [user_is_authenticated] ) ] diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index 27b678cc54..76da0bbb1d 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -152,3 +152,43 @@ def mock_table_add(table_name, _schema_oid, conn, column_data_list, constraint_d monkeypatch.setattr(tables, 'create_table_on_database', mock_table_add) actual_table_oid = tables.add(table_name='newtable', schema_oid=2200, database_id=11, request=request) assert actual_table_oid == 1964474 + + +def test_tables_patch(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + table_oid = 1964474 + database_id = 11 + table_data_dict = { + "name": "newtabname", + "description": "this is a description", + "columns": {} + } + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_table_patch(_table_oid, _table_data_dict, conn): + if _table_oid != table_oid and _table_data_dict != table_data_dict: + raise AssertionError('incorrect parameters passed') + return 'newtabname' + monkeypatch.setattr(tables, 'connect', mock_connect) + monkeypatch.setattr(tables, 'alter_table_on_database', mock_table_patch) + altered_table_name = tables.patch( + table_oid=1964474, + table_data_dict={ + "name": "newtabname", + "description": "this is a description", + "columns": {} + }, + database_id=11, + request=request + ) + assert altered_table_name == 'newtabname' From 52c45f0a6fedc4955421e208458397b137248950 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 12 Jun 2024 00:31:14 +0530 Subject: [PATCH 0242/1141] add return value to function signature --- mathesar/rpc/tables.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 09a243ecaf..166acdf680 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -137,7 +137,9 @@ def delete( @rpc_method(name="tables.patch") @http_basic_auth_login_required @handle_rpc_exceptions -def patch(*, table_oid: str, table_data_dict: SettableTableInfo, database_id: int, **kwargs): +def patch( + *, table_oid: str, table_data_dict: SettableTableInfo, database_id: int, **kwargs +) -> int: """ Alter details of preexisting tables in a database. From 905e5f901e23cf51b498ff0a2221024cd677cf9b Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 12 Jun 2024 00:32:56 +0530 Subject: [PATCH 0243/1141] alter return from int->str --- mathesar/rpc/tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 166acdf680..3e3a5e3cf6 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -139,7 +139,7 @@ def delete( @handle_rpc_exceptions def patch( *, table_oid: str, table_data_dict: SettableTableInfo, database_id: int, **kwargs -) -> int: +) -> str: """ Alter details of preexisting tables in a database. From 5b835fb58700eabdaace2925c27eb3c91459e10a Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 12 Jun 2024 01:13:25 +0530 Subject: [PATCH 0244/1141] fix comment_on_table sql funtions for null input and add docstring for funtions --- db/sql/00_msar.sql | 27 ++++++++++++++++++++++----- mathesar/rpc/tables.py | 17 ++++++++++++++++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index fbc8dcaf1a..7cd4fbc0bc 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1089,8 +1089,12 @@ Args: tab_name: The qualified, quoted name of the table whose comment we will change. comment_: The new comment. Any quotes or special characters must be escaped. */ -SELECT __msar.exec_ddl('COMMENT ON TABLE %s IS %s', tab_name, comment_); -$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +DECLARE + comment_or_null text := COALESCE(comment_, 'NULL'); +BEGIN +RETURN __msar.exec_ddl('COMMENT ON TABLE %s IS %s', tab_name, comment_or_null); +END; +$$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION @@ -1102,7 +1106,7 @@ Args: comment_: The new comment. */ SELECT __msar.comment_on_table(__msar.get_relation_name(tab_id), quote_literal(comment_)); -$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +$$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION @@ -1118,12 +1122,25 @@ SELECT __msar.comment_on_table( msar.get_fully_qualified_object_name(sch_name, tab_name), quote_literal(comment_) ); -$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +$$ LANGUAGE SQL; -- Alter table ------------------------------------------------------------------------------------- CREATE OR REPLACE FUNCTION -msar.alter_table(tab_id oid, tab_alters jsonb) RETURNS text AS $$ +msar.alter_table(tab_id oid, tab_alters jsonb) RETURNS text AS $$/* +Alter columns of the given table in bulk, returning the IDs of the columns so altered. + +Args: + tab_id: The OID of the table whose columns we'll alter. + tab_alters: a JSONB describing the alterations to make. + + The tab_alters should have the form: + { + "name": , + "description": + "columns": , + } +*/ DECLARE new_tab_name text; comment text; diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 3e3a5e3cf6..6a5ceaadac 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -30,6 +30,21 @@ class TableInfo(TypedDict): class SettableTableInfo(TypedDict): + """ + Information about a table, restricted to settable fields. + + When possible, Passing `null` for a key will clear the underlying + setting. E.g., + + - `description = null` clears the table description. + + Setting any of `name`, `columns` to `null` is a noop. + + Attributes: + name: The new name of the table. + description: The description of the table. + columns: A list describing desired column alterations. + """ name: Optional[str] description: Optional[str] columns: Optional[list[SettableColumnInfo]] @@ -141,7 +156,7 @@ def patch( *, table_oid: str, table_data_dict: SettableTableInfo, database_id: int, **kwargs ) -> str: """ - Alter details of preexisting tables in a database. + Alter details of a preexisting table in a database. Args: table_oid: Identity of the table whose name, description or columns we'll modify. From 07a6bdd7d2a8de7a1935f2e0210c89c8475626d6 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 12 Jun 2024 18:18:14 +0530 Subject: [PATCH 0245/1141] use get_relation_name_or_null --- db/sql/00_msar.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 7cd4fbc0bc..bb15f962d3 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1057,7 +1057,7 @@ Args: new_tab_name: unquoted, unqualified table name */ BEGIN - RETURN __msar.rename_table(__msar.get_relation_name(tab_id), quote_ident(new_tab_name)); + RETURN __msar.rename_table(msar.get_relation_name_or_null(tab_id), quote_ident(new_tab_name)); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1105,7 +1105,7 @@ Args: tab_id: The OID of the table whose comment we will change. comment_: The new comment. */ -SELECT __msar.comment_on_table(__msar.get_relation_name(tab_id), quote_literal(comment_)); +SELECT __msar.comment_on_table(msar.get_relation_name_or_null(tab_id), quote_literal(comment_)); $$ LANGUAGE SQL; @@ -1152,7 +1152,7 @@ BEGIN PERFORM msar.rename_table(tab_id, new_tab_name); PERFORM msar.comment_on_table(tab_id, comment); PERFORM msar.alter_columns(tab_id, col_alters); - RETURN tab_id::regclass::text; + RETURN msar.get_relation_name_or_null(tab_id); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; From 2bfaddeda486e54630f58c4ab43e46510a71f629 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 12 Jun 2024 18:32:42 +0530 Subject: [PATCH 0246/1141] add mock test for alter_table --- db/tests/tables/operations/test_alter.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/db/tests/tables/operations/test_alter.py b/db/tests/tables/operations/test_alter.py index 993e307364..754a588956 100644 --- a/db/tests/tables/operations/test_alter.py +++ b/db/tests/tables/operations/test_alter.py @@ -29,3 +29,17 @@ def test_comment_on_table(engine_with_schema): assert call_args[2] == schema_name assert call_args[3] == "comment_on_me" assert call_args[4] == "This is a comment" + + +def test_alter_table(): + with patch.object(tab_alter.db_conn, 'exec_msar_func') as mock_exec: + tab_alter.alter_table_on_database( + 12345, + {"name": "newname", "description": "this is a comment", "columns": {}}, + "conn" + ) + call_args = mock_exec.call_args_list[0][0] + assert call_args[0] == "conn" + assert call_args[1] == "alter_table" + assert call_args[2] == 12345 + assert call_args[3] == {"name": "newname", "description": "this is a comment", "columns": {}} From 43dc1f0e6e7d871e1b21b1b9ec7a3d170dda167e Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 12 Jun 2024 18:36:43 +0530 Subject: [PATCH 0247/1141] use -> instead of ->> for columns --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index bb15f962d3..a6ee233ad2 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1148,7 +1148,7 @@ DECLARE BEGIN new_tab_name := tab_alters->>'name'; comment := tab_alters->>'description'; - col_alters := tab_alters->>'columns'; + col_alters := tab_alters->'columns'; PERFORM msar.rename_table(tab_id, new_tab_name); PERFORM msar.comment_on_table(tab_id, comment); PERFORM msar.alter_columns(tab_id, col_alters); From c6f994cd1adeb58fd514ac0f28ac51553ebdb71d Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 12 Jun 2024 14:13:03 -0400 Subject: [PATCH 0248/1141] Add schemas.add RPC function --- conftest.py | 4 +- db/schemas/operations/alter.py | 6 +- db/schemas/operations/create.py | 55 ++++++++++---- db/sql/00_msar.sql | 86 +++++++++++++--------- db/sql/test_00_msar.sql | 33 +++++++-- db/tables/operations/infer_types.py | 4 +- db/tests/schemas/operations/test_create.py | 16 ++-- db/types/install.py | 8 +- docs/docs/api/rpc.md | 1 + mathesar/rpc/schemas.py | 41 ++++++++--- mathesar/tests/imports/test_csv.py | 4 +- mathesar/tests/imports/test_json.py | 4 +- mathesar/tests/rpc/test_endpoints.py | 5 ++ mathesar/utils/schemas.py | 4 +- 14 files changed, 178 insertions(+), 93 deletions(-) diff --git a/conftest.py b/conftest.py index 6089145050..05cd0978ba 100644 --- a/conftest.py +++ b/conftest.py @@ -14,7 +14,7 @@ from db.types import install from db.sql import install as sql_install from db.schemas.operations.drop import drop_schema_via_name as drop_sa_schema -from db.schemas.operations.create import create_schema as create_sa_schema +from db.schemas.operations.create import create_schema_if_not_exists_via_sql_alchemy from db.schemas.utils import get_schema_oid_from_name, get_schema_name_from_oid from fixtures.utils import create_scoped_fixtures @@ -210,7 +210,7 @@ def _create_schema(schema_name, engine, schema_mustnt_exist=True): if schema_mustnt_exist: assert schema_name not in created_schemas logger.debug(f'creating {schema_name}') - create_sa_schema(schema_name, engine, if_not_exists=True) + create_schema_if_not_exists_via_sql_alchemy(schema_name, engine) schema_oid = get_schema_oid_from_name(schema_name, engine) db_name = engine.url.database created_schemas_in_this_engine = created_schemas.setdefault(db_name, {}) diff --git a/db/schemas/operations/alter.py b/db/schemas/operations/alter.py index e6b345720f..919997eff0 100644 --- a/db/schemas/operations/alter.py +++ b/db/schemas/operations/alter.py @@ -27,10 +27,8 @@ def comment_on_schema(schema_name, engine, comment): Change description of a schema. Args: - schema_name: The name of the schema whose comment we will - change. - comment: The new comment. Any quotes or special characters must - be escaped. + schema_name: The name of the schema whose comment we will change. + comment: The new comment. engine: SQLAlchemy engine object for connecting. Returns: diff --git a/db/schemas/operations/create.py b/db/schemas/operations/create.py index a079ec2cd3..8be771c362 100644 --- a/db/schemas/operations/create.py +++ b/db/schemas/operations/create.py @@ -1,26 +1,53 @@ -from db.schemas.operations.alter import comment_on_schema -from db.connection import execute_msar_func_with_engine +from db.connection import execute_msar_func_with_engine, exec_msar_func -def create_schema(schema_name, engine, comment=None, if_not_exists=False): +def create_schema_via_sql_alchemy(schema_name, engine, description=None): """ - Creates a schema. + Creates a schema using a SQLAlchemy engine. Args: schema_name: Name of the schema to create. engine: SQLAlchemy engine object for connecting. - comment: The new comment. Any quotes or special characters must - be escaped. - if_not_exists: Whether to ignore an error if the schema does - exist. + description: A new description to set on the schema. + + If a schema already exists with the given name, this function will raise an error. Returns: - Returns a string giving the command that was run. + The integer oid of the newly created schema. """ - result = execute_msar_func_with_engine( - engine, 'create_schema', schema_name, if_not_exists + return execute_msar_func_with_engine( + engine, 'create_schema', schema_name, description ).fetchone()[0] - if comment: - comment_on_schema(schema_name, engine, comment) - return result + +def create_schema_if_not_exists_via_sql_alchemy(schema_name, engine): + """ + Ensure that a schema exists using a SQLAlchemy engine. + + Args: + schema_name: Name of the schema to create. + engine: SQLAlchemy engine object for connecting. + + Returns: + The integer oid of the newly created schema. + """ + return execute_msar_func_with_engine( + engine, 'create_schema_if_not_exists', schema_name + ).fetchone()[0] + + +def create_schema(schema_name, conn, description=None): + """ + Create a schema using a psycopg connection. + + Args: + schema_name: Name of the schema to create. + conn: a psycopg connection + description: A new description to set on the schema. + + If a schema already exists with the given name, this function will raise an error. + + Returns: + The integer oid of the newly created schema. + """ + return exec_msar_func(conn, 'create_schema', schema_name, description).fetchone()[0] diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index a18dabb1c4..31689896a7 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -143,24 +143,27 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION msar.schema_exists(schema_name text) RETURNS boolean AS $$/* -Return true if the given schema exists in the current database, false otherwise. +Return true if the schema exists, false otherwise. + +Args : + sch_name: The name of the schema, UNQUOTED. */ SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname=schema_name); $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION __msar.get_schema_oid(sch_name text) RETURNS oid AS $$/* -Return the OID of a schema, if it can be diretly found from a name. +CREATE OR REPLACE FUNCTION msar.get_schema_oid(sch_name text) RETURNS oid AS $$/* +Return the OID of a schema, or NULL if the schema does not exist. Args : - sch_name: The name of the schema. + sch_name: The name of the schema, UNQUOTED. */ -SELECT CASE WHEN msar.schema_exists(sch_name) THEN sch_name::regnamespace::oid END; +SELECT oid FROM pg_namespace WHERE nspname=sch_name; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION __msar.get_schema_name(sch_id oid) RETURNS TEXT AS $$/* -Return the name for a given schema, quoted as appropriate. +Return the QUOTED name for a given schema. The schema *must* be in the pg_namespace table to use this function. @@ -612,7 +615,7 @@ Args: SELECT jsonb_agg(prorettype::regtype::text) FROM pg_proc WHERE - pronamespace=__msar.get_schema_oid('mathesar_types') + pronamespace=msar.get_schema_oid('mathesar_types') AND proargtypes[0]=typ_id AND left(proname, 5) = 'cast_'; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -885,8 +888,8 @@ __msar.comment_on_schema(sch_name text, comment_ text) RETURNS TEXT AS $$/* Change the description of a schema, returning command executed. Args: - sch_name: The quoted name of the schema whose comment we will change. - comment_: The new comment. Any quotes or special characters must be escaped. + sch_name: The QUOTED name of the schema whose comment we will change. + comment_: The new comment, QUOTED */ DECLARE cmd_template text; @@ -902,8 +905,8 @@ msar.comment_on_schema(sch_name text, comment_ text) RETURNS TEXT AS $$/* Change the description of a schema, returning command executed. Args: - sch_name: The quoted name of the schema whose comment we will change. - comment_: The new comment. + sch_name: The UNQUOTED name of the schema whose comment we will change. + comment_: The new comment, UNQUOTED */ BEGIN RETURN __msar.comment_on_schema(quote_ident(sch_name), quote_literal(comment_)); @@ -916,7 +919,7 @@ Change the description of a schema, returning command executed. Args: sch_id: The OID of the schema. - comment_: The new comment. + comment_: The new comment, UNQUOTED */ BEGIN RETURN __msar.comment_on_schema(__msar.get_schema_name(sch_id), quote_literal(comment_)); @@ -932,43 +935,54 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- +-- This gets rid of `msar.create_schema` as defined in Mathesar 0.1.7. We don't want that old +-- function definition hanging around because it will get invoked when passing NULL as the second +-- argument like `msar.create_schema('foo', NULL)`. +DROP FUNCTION IF EXISTS msar.create_schema(text, boolean); --- Create schema ----------------------------------------------------------------------------------- - -CREATE OR REPLACE FUNCTION -__msar.create_schema(sch_name text, if_not_exists boolean) RETURNS TEXT AS $$/* -Create a schema, returning the command executed. +CREATE OR REPLACE FUNCTION msar.create_schema_if_not_exists(sch_name text) RETURNS oid AS $$/* +Ensure that a schema exists in the database. Args: - sch_name: A properly quoted name of the schema to be created - if_not_exists: Whether to ignore an error if the schema does exist + sch_name: the name of the schema to be created, UNQUOTED. + +Returns: + The integer OID of the schema */ -DECLARE - cmd_template text; BEGIN - IF if_not_exists - THEN - cmd_template := 'CREATE SCHEMA IF NOT EXISTS %s'; - ELSE - cmd_template := 'CREATE SCHEMA %s'; - END IF; - RETURN __msar.exec_ddl(cmd_template, sch_name); + EXECUTE 'CREATE SCHEMA IF NOT EXISTS ' || quote_ident(sch_name); + RETURN msar.get_schema_oid(sch_name); END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +$$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION -msar.create_schema(sch_name text, if_not_exists boolean) RETURNS TEXT AS $$/* -Create a schema, returning the command executed. +CREATE OR REPLACE FUNCTION msar.create_schema( + sch_name text, + description text DEFAULT '' +) RETURNS oid AS $$/* +Create a schema, possibly with a description. + +If a schema with the given name already exists, an exception will be raised. Args: - sch_name: An unquoted name of the schema to be created - if_not_exists: Whether to ignore an error if the schema does exist + sch_name: the name of the schema to be created, UNQUOTED. + description: (optional) A description for the schema, UNQUOTED. + +Returns: + The integer OID of the schema + +Note: This function does not support IF NOT EXISTS because it's simpler that way. I originally tried +to support descriptions and if_not_exists in the same function, but as I discovered more edge cases +and inconsistencies, it got too complex, and I didn't think we'd have a good enough use case for it. */ +DECLARE schema_oid oid; BEGIN - RETURN __msar.create_schema(quote_ident(sch_name), if_not_exists); + EXECUTE 'CREATE SCHEMA ' || quote_ident(sch_name); + schema_oid := msar.get_schema_oid(sch_name); + PERFORM msar.comment_on_schema(schema_oid, description); + RETURN schema_oid; END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +$$ LANGUAGE plpgsql; ---------------------------------------------------------------------------------------------------- diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index cb11d87e8e..db10bb73e5 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -1183,17 +1183,38 @@ $$ LANGUAGE plpgsql; -- msar.schema_ddl -------------------------------------------------------------------------------- -CREATE OR REPLACE FUNCTION test_create_schema() RETURNS SETOF TEXT AS $$ +CREATE OR REPLACE FUNCTION test_create_schema_without_description() RETURNS SETOF TEXT AS $$ +DECLARE sch_oid oid; BEGIN - PERFORM msar.create_schema( - sch_name => 'create_schema'::text, - if_not_exists => false - ); - RETURN NEXT has_schema('create_schema'); + SELECT msar.create_schema('foo bar') INTO sch_oid; + RETURN NEXT has_schema('foo bar'); + RETURN NEXT is(sch_oid, msar.get_schema_oid('foo bar')); + RETURN NEXT is(obj_description(sch_oid), NULL); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_create_schema_with_description() RETURNS SETOF TEXT AS $$ +DECLARE sch_oid oid; +BEGIN + SELECT msar.create_schema('foo bar', 'yay') INTO sch_oid; + RETURN NEXT has_schema('foo bar'); + RETURN NEXT is(sch_oid, msar.get_schema_oid('foo bar')); + RETURN NEXT is(obj_description(sch_oid), 'yay'); END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION test_create_schema_that_already_exists() RETURNS SETOF TEXT AS $t$ +DECLARE sch_oid oid; +BEGIN + SELECT msar.create_schema('foo bar') INTO sch_oid; + RETURN NEXT throws_ok($$SELECT msar.create_schema('foo bar')$$, '42P06'); + RETURN NEXT is(msar.create_schema_if_not_exists('foo bar'), sch_oid); +END; +$t$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION __setup_drop_schema() RETURNS SETOF TEXT AS $$ BEGIN CREATE SCHEMA drop_test_schema; diff --git a/db/tables/operations/infer_types.py b/db/tables/operations/infer_types.py index e7d3c522a3..11c571103c 100644 --- a/db/tables/operations/infer_types.py +++ b/db/tables/operations/infer_types.py @@ -5,7 +5,7 @@ from db import constants from db.columns.base import MathesarColumn from db.columns.operations.infer_types import infer_column_type -from db.schemas.operations.create import create_schema +from db.schemas.operations.create import create_schema_if_not_exists_via_sql_alchemy from db.tables.operations.create import CreateTableAs from db.tables.operations.select import reflect_table from db.types.operations.convert import get_db_type_enum_from_class @@ -43,7 +43,7 @@ def infer_table_column_types(schema, table_name, engine, metadata=None, columns_ table = reflect_table(table_name, schema, engine, metadata=metadata) temp_name = TEMP_TABLE % (int(time())) - create_schema(TEMP_SCHEMA, engine, if_not_exists=True) + create_schema_if_not_exists_via_sql_alchemy(TEMP_SCHEMA, engine) with engine.begin() as conn: while engine.dialect.has_table(conn, temp_name, schema=TEMP_SCHEMA): temp_name = TEMP_TABLE.format(int(time())) diff --git a/db/tests/schemas/operations/test_create.py b/db/tests/schemas/operations/test_create.py index 8c6d796f80..5ca649fc21 100644 --- a/db/tests/schemas/operations/test_create.py +++ b/db/tests/schemas/operations/test_create.py @@ -1,22 +1,16 @@ -import pytest from unittest.mock import patch -import db.schemas.operations.create as sch_create +from db.schemas.operations.create import create_schema_via_sql_alchemy -@pytest.mark.parametrize( - "if_not_exists", [(True), (False), (None)] -) -def test_create_schema(engine_with_schema, if_not_exists): +def test_create_schema_via_sql_alchemy(engine_with_schema): engine = engine_with_schema - with patch.object(sch_create, 'execute_msar_func_with_engine') as mock_exec: - sch_create.create_schema( + with patch.object(create_schema_via_sql_alchemy, 'execute_msar_func_with_engine') as mock_exec: + create_schema_via_sql_alchemy( schema_name='new_schema', engine=engine, comment=None, - if_not_exists=if_not_exists ) call_args = mock_exec.call_args_list[0][0] assert call_args[0] == engine - assert call_args[1] == "create_schema" + assert call_args[1] == "create_schema_via_sql_alchemy" assert call_args[2] == "new_schema" - assert call_args[3] == if_not_exists or False diff --git a/db/types/install.py b/db/types/install.py index 5651af1eed..7c1dfd92d6 100644 --- a/db/types/install.py +++ b/db/types/install.py @@ -1,12 +1,12 @@ from db.types.custom import email, money, multicurrency, uri, json_array, json_object from db.constants import TYPES_SCHEMA -from db.schemas.operations.create import create_schema +from db.schemas.operations.create import create_schema_if_not_exists_via_sql_alchemy from db.types.operations.cast import install_all_casts import psycopg -def create_type_schema(engine): - create_schema(TYPES_SCHEMA, engine, if_not_exists=True) +def create_type_schema(engine) -> None: + create_schema_if_not_exists_via_sql_alchemy(TYPES_SCHEMA, engine) def install_mathesar_on_database(engine): @@ -24,4 +24,6 @@ def install_mathesar_on_database(engine): def uninstall_mathesar_from_database(engine): conn_str = str(engine.url) with psycopg.connect(conn_str) as conn: + # TODO: Clean up this code so that it references all the schemas in our + # `INTERNAL_SCHEMAS` constant. conn.execute(f"DROP SCHEMA IF EXISTS __msar, msar, {TYPES_SCHEMA} CASCADE") diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index d749847879..911805ae19 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -52,6 +52,7 @@ To use an RPC function: options: members: - list_ + - add - delete - SchemaInfo diff --git a/mathesar/rpc/schemas.py b/mathesar/rpc/schemas.py index caf7f3fc43..890f5c5747 100644 --- a/mathesar/rpc/schemas.py +++ b/mathesar/rpc/schemas.py @@ -7,6 +7,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from db.constants import INTERNAL_SCHEMAS +from db.schemas.operations.create import create_schema from db.schemas.operations.select import get_schemas from db.schemas.operations.drop import drop_schema_via_oid from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -22,13 +23,40 @@ class SchemaInfo(TypedDict): name: The name of the schema description: A description of the schema table_count: The number of tables in the schema - exploration_count: The number of explorations in the schema """ oid: int name: str description: Optional[str] table_count: int - exploration_count: int + + +@rpc_method(name="schemas.add") +@http_basic_auth_login_required +@handle_rpc_exceptions +def add( + *, + name: str, + database_id: int, + description: Optional[str] = None, + **kwargs, +) -> int: + """ + Add a schema + + Args: + name: The name of the schema to add. + database_id: The Django id of the database containing the schema. + description: A description of the schema + + Returns: + The integer OID of the schema created + """ + with connect(database_id, kwargs.get(REQUEST_KEY).user) as conn: + return create_schema( + schema_name=name, + conn=conn, + description=description + ) @rpc_method(name="schemas.list") @@ -42,18 +70,13 @@ def list_(*, database_id: int, **kwargs) -> list[SchemaInfo]: database_id: The Django id of the database containing the table. Returns: - A list of schema details + A list of SchemaInfo objects """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: schemas = get_schemas(conn) - user_defined_schemas = [s for s in schemas if s['name'] not in INTERNAL_SCHEMAS] - - # TODO_FOR_BETA: join exploration count from internal DB here after we've - # refactored the models so that each exploration is associated with a schema - # (by oid) in a specific database. - return [{**s, "exploration_count": 0} for s in user_defined_schemas] + return [s for s in schemas if s['name'] not in INTERNAL_SCHEMAS] @rpc_method(name="schemas.delete") diff --git a/mathesar/tests/imports/test_csv.py b/mathesar/tests/imports/test_csv.py index a7b860df8d..ea88f53607 100644 --- a/mathesar/tests/imports/test_csv.py +++ b/mathesar/tests/imports/test_csv.py @@ -7,7 +7,7 @@ from mathesar.errors import InvalidTableError from mathesar.imports.base import create_table_from_data_file from mathesar.imports.csv import get_sv_dialect, get_sv_reader -from db.schemas.operations.create import create_schema +from db.schemas.operations.create import create_schema_via_sql_alchemy from db.schemas.utils import get_schema_oid_from_name from db.constants import COLUMN_NAME_TEMPLATE from psycopg.errors import DuplicateTable @@ -45,7 +45,7 @@ def col_headers_empty_data_file(col_headers_empty_csv_filepath): @pytest.fixture() def schema(engine, test_db_model): - create_schema(TEST_SCHEMA, engine) + create_schema_via_sql_alchemy(TEST_SCHEMA, engine) schema_oid = get_schema_oid_from_name(TEST_SCHEMA, engine) yield Schema.current_objects.create(oid=schema_oid, database=test_db_model) with engine.begin() as conn: diff --git a/mathesar/tests/imports/test_json.py b/mathesar/tests/imports/test_json.py index b099e48680..a35b47dc74 100644 --- a/mathesar/tests/imports/test_json.py +++ b/mathesar/tests/imports/test_json.py @@ -5,7 +5,7 @@ from mathesar.models.base import DataFile, Schema from mathesar.imports.base import create_table_from_data_file -from db.schemas.operations.create import create_schema +from db.schemas.operations.create import create_schema_via_sql_alchemy from db.schemas.utils import get_schema_oid_from_name from psycopg.errors import DuplicateTable @@ -21,7 +21,7 @@ def data_file(patents_json_filepath): @pytest.fixture() def schema(engine, test_db_model): - create_schema(TEST_SCHEMA, engine) + create_schema_via_sql_alchemy(TEST_SCHEMA, engine) schema_oid = get_schema_oid_from_name(TEST_SCHEMA, engine) yield Schema.current_objects.create(oid=schema_oid, database=test_db_model) with engine.begin() as conn: diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 6542295a72..c0854ad2fd 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -39,6 +39,11 @@ "connections.add_from_scratch", [user_is_superuser] ), + ( + schemas.add, + "schemas.add", + [user_is_authenticated] + ), ( schemas.list_, "schemas.list", diff --git a/mathesar/utils/schemas.py b/mathesar/utils/schemas.py index 1d12c0060b..0552745704 100644 --- a/mathesar/utils/schemas.py +++ b/mathesar/utils/schemas.py @@ -1,7 +1,7 @@ from django.core.exceptions import ObjectDoesNotExist from rest_framework.exceptions import ValidationError -from db.schemas.operations.create import create_schema +from db.schemas.operations.create import create_schema_via_sql_alchemy from db.schemas.utils import get_schema_oid_from_name, get_mathesar_schemas from mathesar.database.base import create_mathesar_engine from mathesar.models.base import Schema, Database @@ -19,7 +19,7 @@ def create_schema_and_object(name, connection_id, comment=None): all_schemas = get_mathesar_schemas(engine) if name in all_schemas: raise ValidationError({"name": f"Schema name {name} is not unique"}) - create_schema(name, engine, comment=comment) + create_schema_via_sql_alchemy(name, engine, comment) schema_oid = get_schema_oid_from_name(name, engine) schema = Schema.objects.create(oid=schema_oid, database=database_model) From 05eb1769f6a997bd0f060fd11907fb7818891a7d Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 12 Jun 2024 14:30:03 -0400 Subject: [PATCH 0249/1141] Fix failing test --- db/tests/schemas/operations/test_create.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/db/tests/schemas/operations/test_create.py b/db/tests/schemas/operations/test_create.py index 5ca649fc21..9452b95e1a 100644 --- a/db/tests/schemas/operations/test_create.py +++ b/db/tests/schemas/operations/test_create.py @@ -1,16 +1,16 @@ from unittest.mock import patch -from db.schemas.operations.create import create_schema_via_sql_alchemy +import db.schemas.operations.create as sch_create def test_create_schema_via_sql_alchemy(engine_with_schema): engine = engine_with_schema - with patch.object(create_schema_via_sql_alchemy, 'execute_msar_func_with_engine') as mock_exec: - create_schema_via_sql_alchemy( + with patch.object(sch_create, 'execute_msar_func_with_engine') as mock_exec: + sch_create.create_schema_via_sql_alchemy( schema_name='new_schema', engine=engine, - comment=None, + description=None, ) call_args = mock_exec.call_args_list[0][0] assert call_args[0] == engine - assert call_args[1] == "create_schema_via_sql_alchemy" + assert call_args[1] == "create_schema" assert call_args[2] == "new_schema" From e4ee2bede9c6607f5813ed50cbc6b280fb8d0bb7 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 13 Jun 2024 11:30:23 +0800 Subject: [PATCH 0250/1141] add documentation of column creation defaults --- mathesar/rpc/columns.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns.py index 6a74a6f329..6f198d8553 100644 --- a/mathesar/rpc/columns.py +++ b/mathesar/rpc/columns.py @@ -229,6 +229,10 @@ def add( """ Add columns to a table. + There are defaults for both the name and type of a column, and so + passing `[{}]` for `column_data_list` would add a single column of + type `CHARACTER VARYING`, with an auto-generated name. + Args: column_data_list: A list describing desired columns to add. table_oid: Identity of the table to which we'll add columns. From 4c2758cb8eed7b23e272d8697b1714f01533252f Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 13 Jun 2024 18:23:34 +0800 Subject: [PATCH 0251/1141] move Database model to Connection --- mathesar/api/db/viewsets/databases.py | 4 +- mathesar/api/dj_filters.py | 4 +- mathesar/api/serializers/databases.py | 6 +- mathesar/api/serializers/schemas.py | 4 +- mathesar/api/ui/permissions/database_role.py | 4 +- mathesar/api/ui/permissions/schema_role.py | 4 +- mathesar/api/ui/serializers/users.py | 4 +- mathesar/api/ui/viewsets/databases.py | 4 +- mathesar/install.py | 4 +- ...007_rename_database_connection_and_more.py | 34 +++++++++ mathesar/models/base.py | 4 +- mathesar/models/users.py | 4 +- mathesar/rpc/utils.py | 12 ++-- mathesar/state/django.py | 6 +- mathesar/tests/api/test_database_api.py | 8 +-- mathesar/tests/api/test_ui_filters_api.py | 4 +- mathesar/tests/api/test_ui_types_api.py | 6 +- mathesar/tests/api/test_user_api.py | 38 +++++----- mathesar/tests/conftest.py | 6 +- mathesar/tests/database/test_types.py | 4 +- mathesar/tests/test_models.py | 8 +-- mathesar/tests/test_multi_db.py | 6 +- mathesar/utils/connections.py | 72 +++++++++---------- mathesar/utils/schemas.py | 4 +- mathesar/views.py | 8 +-- 25 files changed, 148 insertions(+), 114 deletions(-) create mode 100644 mathesar/migrations/0007_rename_database_connection_and_more.py diff --git a/mathesar/api/db/viewsets/databases.py b/mathesar/api/db/viewsets/databases.py index 95d4eccbef..b09b94b027 100644 --- a/mathesar/api/db/viewsets/databases.py +++ b/mathesar/api/db/viewsets/databases.py @@ -5,7 +5,7 @@ from rest_framework.response import Response from mathesar.api.db.permissions.database import DatabaseAccessPolicy -from mathesar.models.base import Database +from mathesar.models.base import Connection from mathesar.api.dj_filters import DatabaseFilter from mathesar.api.pagination import DefaultLimitOffsetPagination @@ -28,7 +28,7 @@ class ConnectionViewSet(AccessViewSetMixin, viewsets.ModelViewSet): def get_queryset(self): return self.access_policy.scope_queryset( self.request, - Database.objects.all().order_by('-created_at') + Connection.objects.all().order_by('-created_at') ) def destroy(self, request, pk=None): diff --git a/mathesar/api/dj_filters.py b/mathesar/api/dj_filters.py index d72e9f67cd..e22d3ff03d 100644 --- a/mathesar/api/dj_filters.py +++ b/mathesar/api/dj_filters.py @@ -1,7 +1,7 @@ from django_filters import BooleanFilter, DateTimeFromToRangeFilter, OrderingFilter from django_property_filter import PropertyFilterSet, PropertyBaseInFilter, PropertyCharFilter, PropertyOrderingFilter -from mathesar.models.base import Schema, Table, Database, DataFile +from mathesar.models.base import Schema, Table, Connection, DataFile from mathesar.models.query import UIQuery @@ -19,7 +19,7 @@ class DatabaseFilter(PropertyFilterSet): ) class Meta: - model = Database + model = Connection fields = ['deleted'] diff --git a/mathesar/api/serializers/databases.py b/mathesar/api/serializers/databases.py index 7911b909e7..8a8ab575a7 100644 --- a/mathesar/api/serializers/databases.py +++ b/mathesar/api/serializers/databases.py @@ -3,7 +3,7 @@ from mathesar.api.display_options import DISPLAY_OPTIONS_BY_UI_TYPE from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin -from mathesar.models.base import Database +from mathesar.models.base import Connection class ConnectionSerializer(MathesarErrorMessageMixin, serializers.ModelSerializer): @@ -12,7 +12,7 @@ class ConnectionSerializer(MathesarErrorMessageMixin, serializers.ModelSerialize database = serializers.CharField(source='db_name') class Meta: - model = Database + model = Connection fields = ['id', 'nickname', 'database', 'supported_types_url', 'username', 'password', 'host', 'port'] read_only_fields = ['id', 'supported_types_url'] extra_kwargs = { @@ -20,7 +20,7 @@ class Meta: } def get_supported_types_url(self, obj): - if isinstance(obj, Database) and not self.partial: + if isinstance(obj, Connection) and not self.partial: # Only get records if we are serializing an existing table request = self.context['request'] return request.build_absolute_uri(reverse('connection-types', kwargs={'pk': obj.pk})) diff --git a/mathesar/api/serializers/schemas.py b/mathesar/api/serializers/schemas.py index f345362ed5..9dbd752817 100644 --- a/mathesar/api/serializers/schemas.py +++ b/mathesar/api/serializers/schemas.py @@ -6,7 +6,7 @@ from mathesar.api.db.permissions.table import TableAccessPolicy from mathesar.api.db.permissions.database import DatabaseAccessPolicy from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin -from mathesar.models.base import Database, Schema, Table +from mathesar.models.base import Connection, Schema, Table from mathesar.api.exceptions.database_exceptions import ( exceptions as database_api_exceptions ) @@ -19,7 +19,7 @@ class SchemaSerializer(MathesarErrorMessageMixin, serializers.HyperlinkedModelSe connection_id = PermittedPkRelatedField( source='database', access_policy=DatabaseAccessPolicy, - queryset=Database.current_objects.all() + queryset=Connection.current_objects.all() ) description = serializers.CharField( required=False, allow_blank=True, default=None, allow_null=True diff --git a/mathesar/api/ui/permissions/database_role.py b/mathesar/api/ui/permissions/database_role.py index 1f75662001..20d0c2205c 100644 --- a/mathesar/api/ui/permissions/database_role.py +++ b/mathesar/api/ui/permissions/database_role.py @@ -1,7 +1,7 @@ from django.db.models import Q from rest_access_policy import AccessPolicy -from mathesar.models.base import Database +from mathesar.models.base import Connection from mathesar.models.users import DatabaseRole, Role @@ -32,7 +32,7 @@ def scope_queryset(cls, request, qs): if not (request.user.is_superuser or request.user.is_anonymous): # TODO Consider moving to more reusable place allowed_roles = (Role.MANAGER.value, Role.EDITOR.value, Role.VIEWER.value) - databases_with_view_access = Database.objects.filter( + databases_with_view_access = Connection.objects.filter( Q(database_role__role__in=allowed_roles) & Q(database_role__user=request.user) ) qs = qs.filter(database__in=databases_with_view_access) diff --git a/mathesar/api/ui/permissions/schema_role.py b/mathesar/api/ui/permissions/schema_role.py index a6d58521c7..5b6ef84d21 100644 --- a/mathesar/api/ui/permissions/schema_role.py +++ b/mathesar/api/ui/permissions/schema_role.py @@ -1,7 +1,7 @@ from django.db.models import Q from rest_access_policy import AccessPolicy -from mathesar.models.base import Database, Schema +from mathesar.models.base import Connection, Schema from mathesar.models.users import DatabaseRole, Role, SchemaRole @@ -27,7 +27,7 @@ class SchemaRoleAccessPolicy(AccessPolicy): def scope_queryset(cls, request, qs): if not (request.user.is_superuser or request.user.is_anonymous): allowed_roles = (Role.MANAGER.value, Role.EDITOR.value, Role.VIEWER.value) - databases_with_view_access = Database.objects.filter( + databases_with_view_access = Connection.objects.filter( Q(database_role__role__in=allowed_roles) & Q(database_role__user=request.user) ) schema_with_view_access = Schema.objects.filter( diff --git a/mathesar/api/ui/serializers/users.py b/mathesar/api/ui/serializers/users.py index 250cfa9005..d092ba9524 100644 --- a/mathesar/api/ui/serializers/users.py +++ b/mathesar/api/ui/serializers/users.py @@ -8,7 +8,7 @@ from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin from mathesar.api.exceptions.validation_exceptions.exceptions import IncorrectOldPassword from mathesar.api.ui.permissions.users import UserAccessPolicy -from mathesar.models.base import Database, Schema +from mathesar.models.base import Connection, Schema from mathesar.models.users import User, DatabaseRole, SchemaRole @@ -107,7 +107,7 @@ class Meta: # Refer https://rsinger86.github.io/drf-access-policy/policy_reuse/ for the usage of `PermittedPkRelatedField` database = PermittedPkRelatedField( access_policy=DatabaseAccessPolicy, - queryset=Database.current_objects.all() + queryset=Connection.current_objects.all() ) diff --git a/mathesar/api/ui/viewsets/databases.py b/mathesar/api/ui/viewsets/databases.py index 0bdc1770cb..b01f539849 100644 --- a/mathesar/api/ui/viewsets/databases.py +++ b/mathesar/api/ui/viewsets/databases.py @@ -7,7 +7,7 @@ from rest_framework.response import Response from mathesar.api.ui.permissions.ui_database import UIDatabaseAccessPolicy -from mathesar.models.base import Database +from mathesar.models.base import Connection from mathesar.api.dj_filters import DatabaseFilter from mathesar.api.exceptions.validation_exceptions.exceptions import ( DictHasBadKeys, UnsupportedInstallationDatabase @@ -39,7 +39,7 @@ class ConnectionViewSet( def get_queryset(self): return self.access_policy.scope_queryset( self.request, - Database.objects.all().order_by('-created_at') + Connection.objects.all().order_by('-created_at') ) @action(methods=['get'], detail=True) diff --git a/mathesar/install.py b/mathesar/install.py index 3efaa9bd00..5f4d5973a3 100644 --- a/mathesar/install.py +++ b/mathesar/install.py @@ -40,8 +40,8 @@ def main(skip_static_collection=False): def install_on_db_with_key(database_key, skip_confirm): - from mathesar.models.base import Database - db_model = Database.create_from_settings_key(database_key) + from mathesar.models.base import Connection + db_model = Connection.create_from_settings_key(database_key) db_model.save() try: install.install_mathesar( diff --git a/mathesar/migrations/0007_rename_database_connection_and_more.py b/mathesar/migrations/0007_rename_database_connection_and_more.py new file mode 100644 index 0000000000..d47e6768f1 --- /dev/null +++ b/mathesar/migrations/0007_rename_database_connection_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 4.2.11 on 2024-06-13 09:05 + +from django.db import migrations, models +import django.db.models.deletion +import mathesar.models.base + + +class Migration(migrations.Migration): + + dependencies = [ + ('mathesar', '0006_mathesar_databases_to_model'), + ] + + operations = [ + migrations.RenameModel( + old_name='Database', + new_name='Connection', + ), + migrations.AlterField( + model_name='databaserole', + name='database', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mathesar.connection'), + ), + migrations.AlterField( + model_name='schemarole', + name='schema', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mathesar.schema'), + ), + migrations.AlterField( + model_name='tablesettings', + name='column_order', + field=models.JSONField(blank=True, default=None, null=True, validators=[mathesar.models.base.validate_column_order]), + ), + ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 19685b065c..0e54d0fa2e 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -109,7 +109,7 @@ def __repr__(self): _engine_cache = {} -class Database(ReflectionManagerMixin, BaseModel): +class Connection(ReflectionManagerMixin, BaseModel): name = models.CharField(max_length=128, unique=True) db_name = models.CharField(max_length=128) username = EncryptedCharField(max_length=255) @@ -178,7 +178,7 @@ def save(self, **kwargs): class Schema(DatabaseObject): - database = models.ForeignKey('Database', on_delete=models.CASCADE, + database = models.ForeignKey('Connection', on_delete=models.CASCADE, related_name='schemas') class Meta: diff --git a/mathesar/models/users.py b/mathesar/models/users.py index e1b77a8161..4db3748959 100644 --- a/mathesar/models/users.py +++ b/mathesar/models/users.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import AbstractUser from django.db import models -from mathesar.models.base import BaseModel, Database, Schema +from mathesar.models.base import BaseModel, Connection, Schema class User(AbstractUser): @@ -29,7 +29,7 @@ class Role(models.TextChoices): class DatabaseRole(BaseModel): user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='database_roles') - database = models.ForeignKey(Database, on_delete=models.CASCADE) + database = models.ForeignKey(Connection, on_delete=models.CASCADE) role = models.CharField(max_length=10, choices=Role.choices) class Meta: diff --git a/mathesar/rpc/utils.py b/mathesar/rpc/utils.py index 4ae3a0dc4e..7ab6ce44ef 100644 --- a/mathesar/rpc/utils.py +++ b/mathesar/rpc/utils.py @@ -1,14 +1,14 @@ from mathesar.database.base import get_psycopg_connection -from mathesar.models.base import Database +from mathesar.models.base import Connection -def connect(db_id, user): +def connect(conn_id, user): """ - Return a psycopg connection, given a Database model id. + Return a psycopg connection, given a Connection model id. Args: - db_id: The Django id corresponding to the Database. + conn_id: The Django id corresponding to the Connection. """ print("User is: ", user) - db_model = Database.current_objects.get(id=db_id) - return get_psycopg_connection(db_model) + conn_model = Connection.current_objects.get(id=conn_id) + return get_psycopg_connection(conn_model) diff --git a/mathesar/state/django.py b/mathesar/state/django.py index a812a9ca93..e60a671fd5 100644 --- a/mathesar/state/django.py +++ b/mathesar/state/django.py @@ -32,7 +32,7 @@ def clear_dj_cache(): def reflect_db_objects(metadata, db_name=None): - databases = models.Database.current_objects.all() + databases = models.Connection.current_objects.all() if db_name is not None: databases = databases.filter(name=db_name) sync_databases_status(databases) @@ -53,7 +53,7 @@ def reflect_db_objects(metadata, db_name=None): def sync_databases_status(databases): - """Update status and check health for current Database Model instances.""" + """Update status and check health for current Connection Model instances.""" for db in databases: try: db._sa_engine.connect() @@ -65,7 +65,7 @@ def sync_databases_status(databases): def _set_db_is_deleted(db, deleted): """ - Assures that a Django Database model's `deleted` field is equal to the `deleted` + Assures that a Django Connection model's `deleted` field is equal to the `deleted` parameter, updating if necessary. Takes care to `save()` only when an update has been performed, to save on the noteworthy performance cost. """ diff --git a/mathesar/tests/api/test_database_api.py b/mathesar/tests/api/test_database_api.py index e63b384bef..d8333b9708 100644 --- a/mathesar/tests/api/test_database_api.py +++ b/mathesar/tests/api/test_database_api.py @@ -5,7 +5,7 @@ from db.metadata import get_empty_metadata from mathesar.models.users import DatabaseRole from mathesar.state.django import reflect_db_objects -from mathesar.models.base import Table, Schema, Database +from mathesar.models.base import Table, Schema, Connection from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from db.install import install_mathesar @@ -50,7 +50,7 @@ def test_db_name(worker_id): @pytest.fixture def db_dj_model(test_db_name): _recreate_db(test_db_name) - db = Database.objects.get_or_create( + db = Connection.objects.get_or_create( name=test_db_name, defaults={ 'db_name': test_db_name, @@ -70,7 +70,7 @@ def test_database_reflection_delete(db_dj_model): assert db_dj_model.deleted is False # check DB is not marked deleted inappropriately _remove_db(db_dj_model.name) reflect_db_objects(get_empty_metadata()) - fresh_db_model = Database.objects.get(name=db_dj_model.name) + fresh_db_model = Connection.objects.get(name=db_dj_model.name) assert fresh_db_model.deleted is True # check DB is marked deleted appropriately @@ -166,7 +166,7 @@ def test_delete_dbconn_with_msar_schemas(client, db_dj_model): after_deletion = conn.execute(check_schema_exists) with pytest.raises(ObjectDoesNotExist): - Database.objects.get(id=db_dj_model.id) + Connection.objects.get(id=db_dj_model.id) assert response.status_code == 204 assert before_deletion.rowcount == 3 assert after_deletion.rowcount == 0 diff --git a/mathesar/tests/api/test_ui_filters_api.py b/mathesar/tests/api/test_ui_filters_api.py index b0cba3519a..38e73e01ac 100644 --- a/mathesar/tests/api/test_ui_filters_api.py +++ b/mathesar/tests/api/test_ui_filters_api.py @@ -1,10 +1,10 @@ -from mathesar.models.base import Database +from mathesar.models.base import Connection from mathesar.filters.base import get_available_filters from mathesar.models.users import DatabaseRole def test_filter_list(client, test_db_name): - database = Database.objects.get(name=test_db_name) + database = Connection.objects.get(name=test_db_name) response = client.get(f'/api/ui/v0/connections/{database.id}/filters/') response_data = response.json() diff --git a/mathesar/tests/api/test_ui_types_api.py b/mathesar/tests/api/test_ui_types_api.py index 6a33ebf8c1..74450168f5 100644 --- a/mathesar/tests/api/test_ui_types_api.py +++ b/mathesar/tests/api/test_ui_types_api.py @@ -1,12 +1,12 @@ from mathesar.api.display_options import DISPLAY_OPTIONS_BY_UI_TYPE -from mathesar.models.base import Database +from mathesar.models.base import Connection from mathesar.database.types import get_ui_type_from_id, UIType from db.types.base import PostgresType, MathesarCustomType from mathesar.models.users import DatabaseRole def test_type_list(client, test_db_name): - database = Database.objects.get(name=test_db_name) + database = Connection.objects.get(name=test_db_name) response = client.get(f'/api/ui/v0/connections/{database.id}/types/') response_data = response.json() @@ -62,7 +62,7 @@ def test_database_types_installed(client, test_db_name): 'display_options': None }, ] - default_database = Database.objects.get(name=test_db_name) + default_database = Connection.objects.get(name=test_db_name) response = client.get(f'/api/ui/v0/connections/{default_database.id}/types/') assert response.status_code == 200 diff --git a/mathesar/tests/api/test_user_api.py b/mathesar/tests/api/test_user_api.py index d655022188..8687873ff6 100644 --- a/mathesar/tests/api/test_user_api.py +++ b/mathesar/tests/api/test_user_api.py @@ -1,7 +1,7 @@ from django.db import transaction from db.schemas.utils import get_schema_oid_from_name -from mathesar.models.base import Database, Schema +from mathesar.models.base import Connection, Schema from mathesar.models.users import User, DatabaseRole, SchemaRole @@ -253,7 +253,7 @@ def test_user_delete_different_user(client_bob, user_alice): def test_database_role_list_user_without_view_permission(client_bob, user_alice): role = 'manager' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] DatabaseRole.objects.create(user=user_alice, database=database, role=role) response = client_bob.get('/api/ui/v0/database_roles/') @@ -267,7 +267,7 @@ def test_db_role_list_with_roles_on_multiple_database(FUN_create_dj_db, client_b FUN_create_dj_db(get_uid()) FUN_create_dj_db(get_uid()) FUN_create_dj_db(get_uid()) - databases = Database.objects.all() + databases = Connection.objects.all() database_with_viewer_access = databases[0] DatabaseRole.objects.create(user=user_bob, database=database_with_viewer_access, role='viewer') database_with_manager_access = databases[1] @@ -280,7 +280,7 @@ def test_db_role_list_with_roles_on_multiple_database(FUN_create_dj_db, client_b def test_database_role_list_user_with_view_permission(client_bob, user_alice, user_bob): role = 'manager' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] DatabaseRole.objects.create(user=user_alice, database=database, role=role) DatabaseRole.objects.create(user=user_bob, database=database, role=role) @@ -293,7 +293,7 @@ def test_database_role_list_user_with_view_permission(client_bob, user_alice, us def test_database_role_list_superuser(client, user_bob): role = 'manager' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] DatabaseRole.objects.create(user=user_bob, database=database, role=role) response = client.get('/api/ui/v0/database_roles/') @@ -384,7 +384,7 @@ def test_schema_role_list_with_roles_on_multiple_database( def test_database_role_detail(client, user_bob): role = 'editor' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] database_role = DatabaseRole.objects.create(user=user_bob, database=database, role=role) response = client.get(f'/api/ui/v0/database_roles/{database_role.id}/') @@ -414,7 +414,7 @@ def test_schema_role_detail(client, user_bob): def test_database_role_update(client, user_bob): role = 'viewer' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] database_role = DatabaseRole.objects.create(user=user_bob, database=database, role=role) data = {'user': user_bob.id, 'role': role, 'database': database.id} @@ -440,7 +440,7 @@ def test_schema_role_update(client, user_bob): def test_database_role_partial_update(client, user_bob): role = 'viewer' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] database_role = DatabaseRole.objects.create(user=user_bob, database=database, role=role) data = {'role': 'editor'} @@ -466,7 +466,7 @@ def test_schema_role_partial_update(client, user_bob): def test_database_role_create_by_superuser(client, user_bob): role = 'editor' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] data = {'user': user_bob.id, 'role': role, 'database': database.id} response = client.post('/api/ui/v0/database_roles/', data) @@ -480,7 +480,7 @@ def test_database_role_create_by_superuser(client, user_bob): def test_database_role_create_by_manager(client_bob, user_bob, user_alice): - database = Database.objects.all()[0] + database = Connection.objects.all()[0] DatabaseRole.objects.create(user=user_bob, database=database, role='manager') role = 'viewer' @@ -500,7 +500,7 @@ def test_db_role_create_with_roles_on_multiple_database(FUN_create_dj_db, client FUN_create_dj_db(get_uid()) FUN_create_dj_db(get_uid()) FUN_create_dj_db(get_uid()) - databases = Database.objects.all() + databases = Connection.objects.all() database_with_viewer_access = databases[0] DatabaseRole.objects.create(user=user_bob, database=database_with_viewer_access, role='viewer') database_with_manager_access = databases[1] @@ -516,7 +516,7 @@ def test_db_role_create_with_roles_on_multiple_database(FUN_create_dj_db, client def test_database_role_create_non_superuser(client_bob, user_bob): role = 'editor' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] data = {'user': user_bob.id, 'role': role, 'database': database.id} response = client_bob.post('/api/ui/v0/database_roles/', data) @@ -642,7 +642,7 @@ def test_schema_role_create_with_multiple_database( def test_database_role_create_with_incorrect_role(client, user_bob): role = 'nonsense' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] data = {'user': user_bob.id, 'role': role, 'database': database.id} response = client.post('/api/ui/v0/database_roles/', data) @@ -666,7 +666,7 @@ def test_schema_role_create_with_incorrect_role(client, user_bob): def test_database_role_create_with_incorrect_database(client, user_bob): role = 'editor' - database = Database.objects.order_by('-id')[0] + database = Connection.objects.order_by('-id')[0] data = {'user': user_bob.id, 'role': role, 'database': database.id + 1} response = client.post('/api/ui/v0/database_roles/', data) @@ -690,7 +690,7 @@ def test_schema_role_create_with_incorrect_schema(client, user_bob): def test_database_role_destroy(client, user_bob): role = 'viewer' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] database_role = DatabaseRole.objects.create(user=user_bob, database=database, role=role) response = client.delete(f'/api/ui/v0/database_roles/{database_role.id}/') @@ -698,7 +698,7 @@ def test_database_role_destroy(client, user_bob): def test_database_role_destroy_by_manager(client_bob, user_bob, user_alice): - database = Database.objects.all()[0] + database = Connection.objects.all()[0] DatabaseRole.objects.create(user=user_bob, database=database, role='manager') role = 'viewer' @@ -709,7 +709,7 @@ def test_database_role_destroy_by_manager(client_bob, user_bob, user_alice): def test_database_role_destroy_by_non_manager(client_bob, user_bob, user_alice): - database = Database.objects.all()[0] + database = Connection.objects.all()[0] DatabaseRole.objects.create(user=user_bob, database=database, role='viewer') role = 'viewer' @@ -720,7 +720,7 @@ def test_database_role_destroy_by_non_manager(client_bob, user_bob, user_alice): def test_database_role_destroy_by_user_without_role(client_bob, user_alice): - database = Database.objects.all()[0] + database = Connection.objects.all()[0] role = 'viewer' database_role = DatabaseRole.objects.create(user=user_alice, database=database, role=role) @@ -769,7 +769,7 @@ def test_schema_role_destroy_by_db_manager(client_bob, user_bob, user_alice): def test_database_role_create_multiple_roles_on_same_object(client, user_bob): role = 'manager' - database = Database.objects.all()[0] + database = Connection.objects.all()[0] DatabaseRole.objects.create(user=user_bob, database=database, role=role) data = {'user': user_bob.id, 'role': 'editor', 'database': database.id} diff --git a/mathesar/tests/conftest.py b/mathesar/tests/conftest.py index fdea559cae..80b75edfdf 100644 --- a/mathesar/tests/conftest.py +++ b/mathesar/tests/conftest.py @@ -22,7 +22,7 @@ import mathesar.tests.conftest from mathesar.imports.base import create_table_from_data_file -from mathesar.models.base import Schema, Table, Database, DataFile +from mathesar.models.base import Schema, Table, Connection, DataFile from mathesar.models.base import Column as mathesar_model_column from mathesar.models.users import DatabaseRole, SchemaRole, User @@ -102,7 +102,7 @@ def _create_and_add(db_name): create_db(db_name) add_db_to_dj_settings(db_name) credentials = settings.DATABASES.get(db_name) - database_model = Database.current_objects.create( + database_model = Connection.current_objects.create( name=db_name, db_name=db_name, username=credentials['USER'], @@ -132,7 +132,7 @@ def test_db_model(request, test_db_name, django_db_blocker): add_db_to_dj_settings(test_db_name) with django_db_blocker.unblock(): credentials = settings.DATABASES.get(test_db_name) - database_model = Database.current_objects.create( + database_model = Connection.current_objects.create( name=test_db_name, db_name=test_db_name, username=credentials['USER'], diff --git a/mathesar/tests/database/test_types.py b/mathesar/tests/database/test_types.py index be86bdc828..f60e5407f7 100644 --- a/mathesar/tests/database/test_types.py +++ b/mathesar/tests/database/test_types.py @@ -1,5 +1,5 @@ from mathesar.database.types import UIType -from mathesar.models.base import Database +from mathesar.models.base import Connection from db.types.base import known_db_types @@ -33,6 +33,6 @@ def _verify_type_mapping(supported_ui_types): def test_type_mapping(): - databases = Database.objects.all() + databases = Connection.objects.all() for database in databases: _verify_type_mapping(database.supported_ui_types) diff --git a/mathesar/tests/test_models.py b/mathesar/tests/test_models.py index 127c4a0b08..d219c619a8 100644 --- a/mathesar/tests/test_models.py +++ b/mathesar/tests/test_models.py @@ -2,7 +2,7 @@ from unittest.mock import patch from django.core.cache import cache -from mathesar.models.base import Database, Schema, Table, schema_utils +from mathesar.models.base import Connection, Schema, Table, schema_utils from mathesar.utils.models import attempt_dumb_query @@ -50,14 +50,14 @@ def mock_name_getter(*_): assert name_ == 'MISSING' -@pytest.mark.parametrize("model", [Database, Schema, Table]) +@pytest.mark.parametrize("model", [Connection, Schema, Table]) def test_model_queryset_reflects_db_objects(model): with patch('mathesar.state.base.reflect_db_objects') as mock_reflect: model.objects.all() mock_reflect.assert_called() -@pytest.mark.parametrize("model", [Database, Schema, Table]) +@pytest.mark.parametrize("model", [Connection, Schema, Table]) def test_model_current_queryset_does_not_reflects_db_objects(model): with patch('mathesar.state.base.reflect_db_objects') as mock_reflect: model.current_objects.all() @@ -78,5 +78,5 @@ def test_database_engine_cache_stability(FUN_create_dj_db, iteration, uid): del iteration # An unused parameter some_db_name = uid FUN_create_dj_db(some_db_name) - db_model = Database.objects.get(name=some_db_name) + db_model = Connection.objects.get(name=some_db_name) attempt_dumb_query(db_model._sa_engine) diff --git a/mathesar/tests/test_multi_db.py b/mathesar/tests/test_multi_db.py index 79af7489f4..6c951a67a8 100644 --- a/mathesar/tests/test_multi_db.py +++ b/mathesar/tests/test_multi_db.py @@ -1,7 +1,7 @@ import pytest from django.core.exceptions import ValidationError -from mathesar.models.base import Table, Schema, Database +from mathesar.models.base import Table, Schema, Connection @pytest.fixture(autouse=True) @@ -66,7 +66,7 @@ def test_multi_db_oid_unique(): """ schema_oid = 5000 table_oid = 5001 - all_dbs = Database.objects.all() + all_dbs = Connection.objects.all() assert len(all_dbs) > 1 for db in all_dbs: schema = Schema.objects.create(oid=schema_oid, database=db) @@ -75,7 +75,7 @@ def test_multi_db_oid_unique(): def test_single_db_oid_unique_exception(): table_oid = 5001 - dbs = Database.objects.all() + dbs = Connection.objects.all() assert len(dbs) > 0 db = dbs[0] schema_1 = Schema.objects.create(oid=4000, database=db) diff --git a/mathesar/utils/connections.py b/mathesar/utils/connections.py index 507d95afc7..70a12ec541 100644 --- a/mathesar/utils/connections.py +++ b/mathesar/utils/connections.py @@ -1,7 +1,7 @@ """Utilities to help with creating and managing connections in Mathesar.""" from psycopg2.errors import DuplicateSchema from sqlalchemy.exc import OperationalError, ProgrammingError -from mathesar.models.base import Database +from mathesar.models.base import Connection from db import install, connection as dbconn from mathesar.state import reset_reflection from mathesar.examples.library_dataset import load_library_dataset @@ -17,71 +17,71 @@ def copy_connection_from_preexisting( connection, nickname, db_name, create_db, sample_data ): if connection['connection_type'] == 'internal_database': - db_model = Database.create_from_settings_key('default') + conn_model = Connection.create_from_settings_key('default') elif connection['connection_type'] == 'user_database': - db_model = Database.current_objects.get(id=connection['id']) - db_model.id = None + conn_model = Connection.current_objects.get(id=connection['id']) + conn_model.id = None else: raise KeyError("connection_type") - root_db = db_model.db_name + root_db = conn_model.db_name return _save_and_install( - db_model, db_name, root_db, nickname, create_db, sample_data + conn_model, db_name, root_db, nickname, create_db, sample_data ) def create_connection_from_scratch( user, password, host, port, nickname, db_name, sample_data ): - db_model = Database(username=user, password=password, host=host, port=port) + conn_model = Connection(username=user, password=password, host=host, port=port) root_db = db_name return _save_and_install( - db_model, db_name, root_db, nickname, False, sample_data + conn_model, db_name, root_db, nickname, False, sample_data ) def create_connection_with_new_user( connection, user, password, nickname, db_name, create_db, sample_data ): - db_model = copy_connection_from_preexisting( + conn_model = copy_connection_from_preexisting( connection, nickname, db_name, create_db, [] ) - engine = db_model._sa_engine - db_model.username = user - db_model.password = password - db_model.save() + engine = conn_model._sa_engine + conn_model.username = user + conn_model.password = password + conn_model.save() dbconn.execute_msar_func_with_engine( engine, 'create_basic_mathesar_user', - db_model.username, - db_model.password + conn_model.username, + conn_model.password ) - _load_sample_data(db_model._sa_engine, sample_data) - return db_model + _load_sample_data(conn_model._sa_engine, sample_data) + return conn_model def _save_and_install( - db_model, db_name, root_db, nickname, create_db, sample_data + conn_model, db_name, root_db, nickname, create_db, sample_data ): - db_model.name = nickname - db_model.db_name = db_name - _validate_db_model(db_model) - db_model.save() + conn_model.name = nickname + conn_model.db_name = db_name + _validate_conn_model(conn_model) + conn_model.save() try: install.install_mathesar( - database_name=db_model.db_name, - username=db_model.username, - password=db_model.password, - hostname=db_model.host, - port=db_model.port, + database_name=conn_model.db_name, + username=conn_model.username, + password=conn_model.password, + hostname=conn_model.host, + port=conn_model.port, skip_confirm=True, create_db=create_db, root_db=root_db, ) except OperationalError as e: - db_model.delete() + conn_model.delete() raise e - _load_sample_data(db_model._sa_engine, sample_data) - return db_model + _load_sample_data(conn_model._sa_engine, sample_data) + return conn_model def _load_sample_data(engine, sample_data): @@ -100,13 +100,13 @@ def _load_sample_data(engine, sample_data): reset_reflection() -def _validate_db_model(db_model): - internal_db_model = Database.create_from_settings_key('default') +def _validate_conn_model(conn_model): + internal_conn_model = Connection.create_from_settings_key('default') if ( - internal_db_model is not None - and db_model.host == internal_db_model.host - and db_model.port == internal_db_model.port - and db_model.db_name == internal_db_model.db_name + internal_conn_model is not None + and conn_model.host == internal_conn_model.host + and conn_model.port == internal_conn_model.port + and conn_model.db_name == internal_conn_model.db_name ): raise BadInstallationTarget( "Mathesar can't be installed in the internal DB namespace" diff --git a/mathesar/utils/schemas.py b/mathesar/utils/schemas.py index 1d12c0060b..985249b372 100644 --- a/mathesar/utils/schemas.py +++ b/mathesar/utils/schemas.py @@ -4,12 +4,12 @@ from db.schemas.operations.create import create_schema from db.schemas.utils import get_schema_oid_from_name, get_mathesar_schemas from mathesar.database.base import create_mathesar_engine -from mathesar.models.base import Schema, Database +from mathesar.models.base import Schema, Connection def create_schema_and_object(name, connection_id, comment=None): try: - database_model = Database.objects.get(id=connection_id) + database_model = Connection.objects.get(id=connection_id) database_name = database_model.name except ObjectDoesNotExist: raise ValidationError({"database": f"Database '{database_name}' not found"}) diff --git a/mathesar/views.py b/mathesar/views.py index 947c6241b0..26807d39d3 100644 --- a/mathesar/views.py +++ b/mathesar/views.py @@ -18,7 +18,7 @@ from mathesar.api.ui.serializers.users import UserSerializer from mathesar.api.utils import is_valid_uuid_v4 from mathesar.database.types import UIType -from mathesar.models.base import Database, Schema, Table +from mathesar.models.base import Connection, Schema, Table from mathesar.models.query import UIQuery from mathesar.models.shares import SharedTable, SharedQuery from mathesar.state import reset_reflection @@ -45,11 +45,11 @@ def _get_permissible_db_queryset(request): SchemaAccessPolicy. """ for deleted in (True, False): - dbs_qs = Database.objects.filter(deleted=deleted) + dbs_qs = Connection.objects.filter(deleted=deleted) permitted_dbs_qs = DatabaseAccessPolicy.scope_queryset(request, dbs_qs) schemas_qs = Schema.objects.all() permitted_schemas_qs = SchemaAccessPolicy.scope_queryset(request, schemas_qs) - dbs_containing_permitted_schemas_qs = Database.objects.filter(schemas__in=permitted_schemas_qs, deleted=deleted) + dbs_containing_permitted_schemas_qs = Connection.objects.filter(schemas__in=permitted_schemas_qs, deleted=deleted) permitted_dbs_qs = permitted_dbs_qs | dbs_containing_permitted_schemas_qs permitted_dbs_qs = permitted_dbs_qs.distinct() if deleted: @@ -144,7 +144,7 @@ def get_base_data_all_routes(request, database=None, schema=None): def _get_internal_db_meta(): - internal_db = Database.create_from_settings_key('default') + internal_db = Connection.create_from_settings_key('default') if internal_db is not None: return { 'type': 'postgres', From 1dde9ba8094b8f4392c48af86e13988bf6f12549 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 14 Jun 2024 02:53:44 +0530 Subject: [PATCH 0252/1141] setup scaffolding for table.import endpoint --- db/tables/operations/create.py | 22 ++++++++ db/tables/operations/select.py | 8 +++ docs/docs/api/rpc.md | 1 + mathesar/imports/csv.py | 92 ++++++++++++++++++++++++++++++++-- mathesar/rpc/tables.py | 7 +++ 5 files changed, 127 insertions(+), 3 deletions(-) diff --git a/db/tables/operations/create.py b/db/tables/operations/create.py index afbee8b32f..d93d4a7972 100644 --- a/db/tables/operations/create.py +++ b/db/tables/operations/create.py @@ -66,6 +66,7 @@ def create_table_on_database( # TODO stop relying on reflections, instead return oid of the created table. +# TODO remove this function def create_string_column_table(name, schema_oid, column_names, engine, comment=None): """ This method creates a Postgres table in the specified schema, with all @@ -82,6 +83,27 @@ def create_string_column_table(name, schema_oid, column_names, engine, comment=N return table +def prepare_table_for_import(table_name, schema_oid, column_names, conn, comment=None): + """ + This method creates a Postgres table in the specified schema, with all + columns being String type. + """ + columns_ = [ + { + "name": column_name, + "type": {"name": PostgresType.TEXT.id} + } for column_name in column_names + ] + table_oid = create_table_on_database( + table_name=table_name, + schema_oid=schema_oid, + conn=conn, + column_data_list=columns_, + comment=comment + ) + return table_oid + + class CreateTableAs(DDLElement): def __init__(self, name, selectable): self.name = name diff --git a/db/tables/operations/select.py b/db/tables/operations/select.py index e25ad3c976..69149ea469 100644 --- a/db/tables/operations/select.py +++ b/db/tables/operations/select.py @@ -59,6 +59,14 @@ def get_table_info(schema, conn): return exec_msar_func(conn, 'get_table_info', schema).fetchone()[0] +def get_relation_name(table_oid, conn): + """ + Return a fully qualified table name for a given table_oid + when table is not included in the search path. + """ + return exec_msar_func(conn, 'get_relation_name_or_null', table_oid).fetchone()[0] + + def reflect_table(name, schema, engine, metadata, connection_to_use=None, keep_existing=False): extend_existing = not keep_existing autoload_with = engine if connection_to_use is None else connection_to_use diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index c93d6e7db0..6bcbac6123 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -65,6 +65,7 @@ To use an RPC function: - add - delete - patch + - import_ - TableInfo - SettableTableInfo diff --git a/mathesar/imports/csv.py b/mathesar/imports/csv.py index e1b45236c5..428634e92d 100644 --- a/mathesar/imports/csv.py +++ b/mathesar/imports/csv.py @@ -1,17 +1,20 @@ from io import TextIOWrapper - +import tempfile import clevercsv as csv +from psycopg import sql from db.tables.operations.alter import update_pk_sequence_to_latest from mathesar.database.base import create_mathesar_engine from db.records.operations.insert import insert_records_from_csv -from db.tables.operations.create import create_string_column_table +from db.tables.operations.create import create_string_column_table, prepare_table_for_import from db.tables.operations.drop import drop_table +from db.tables.operations.select import get_relation_name from mathesar.errors import InvalidTableError from mathesar.imports.utils import get_alternate_column_names, process_column_names from db.constants import COLUMN_NAME_TEMPLATE from psycopg2.errors import IntegrityError, DataError - +from mathesar.models.base import DataFile +from db.encoding_utils import get_sql_compatible_encoding from mathesar.state import reset_reflection # The user-facing documentation replicates these delimiter characters. If you @@ -161,3 +164,86 @@ def create_db_table_from_csv_data_file(data_file, name, schema, comment=None): table = insert_records_from_csv_data_file(name, schema, column_names_alt, engine, comment, data_file) reset_reflection(db_name=db_model.name) return table + + +def insert_csv(data_file_id, table_name, schema_oid, conn, comment=None): + data_file = DataFile.current_objects.get(id=data_file_id) + file_path = data_file.file.path + header = data_file.header + dialect = csv.dialect.SimpleDialect( + data_file.delimiter, + data_file.quotechar, + data_file.escapechar + ) + encoding = get_file_encoding(data_file.file) + with open(file_path, 'rb', encoding=encoding) as csv_file: + csv_reader = get_sv_reader(csv_file, header, dialect) + column_names = process_column_names(csv_reader.fieldnames) + table_oid = prepare_table_for_import( + table_name, + schema_oid, + column_names, + conn, + comment + ) + insert_csv_records( + table_oid, + conn, + csv_file, + column_names, + header, + dialect.delimiter, + dialect.escapechar, + dialect.quotechar, + encoding + ) + + +def insert_csv_records( + table_oid, + conn, + csv_file, + column_names, + header, + delimiter=None, + escape=None, + quote=None, + encoding=None +): + conversion_encoding, sql_encoding = get_sql_compatible_encoding(encoding) + fq_table_name = sql.SQL(get_relation_name(table_oid, conn)) + formatted_columns = sql.SQL(",").join( + sql.Identifier(column_name) for column_name in column_names + ) + copy_sql = sql.SQL( + f"COPY {fq_table_name} ({formatted_columns}) FROM STDIN CSV {header} {delimiter} {escape} {quote} {encoding}" + ).format( + fq_table_name=fq_table_name, + formatted_columns=formatted_columns, + header=sql.SQL("HEADER" if header else ""), + delimiter=sql.SQL(f"DELIMITER E'{delimiter}'" if delimiter else ""), + escape=sql.SQL(f"ESCAPE '{escape}'" if escape else ""), + quote=sql.SQL( + ("QUOTE ''''" if quote == "'" else f"QUOTE '{quote}'") + if quote + else "" + ), + encoding=sql.SQL(f"ENCODING '{sql_encoding}'" if sql_encoding else ""), + ) + cursor = conn.connection.cursor() + if conversion_encoding == encoding: + with cursor.copy(copy_sql) as copy: + if data := csv_file.read(): + copy.write(data) + else: + # File needs to be converted to compatible database supported encoding + with tempfile.SpooledTemporaryFile(mode='wb+', encoding=conversion_encoding) as temp_file: + while True: + contents = csv_file.read(SAMPLE_SIZE).encode(conversion_encoding, "replace") + if not contents: + break + temp_file.write(contents) + temp_file.seek(0) + with cursor.copy(copy_sql) as copy: + if data := temp_file.read(): + copy.write(data) diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 6a5ceaadac..931a89dac1 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -169,3 +169,10 @@ def patch( user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: return alter_table_on_database(table_oid, table_data_dict, conn) + + +@rpc_method(name="tables.import") +@http_basic_auth_login_required +@handle_rpc_exceptions +def import_(): + pass From c4ee70c9745e758a3655d70dd996d5476fe0ca06 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 14 Jun 2024 02:56:30 +0530 Subject: [PATCH 0253/1141] fix inaccurate docstring --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index a6ee233ad2..c6693b00a6 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1128,7 +1128,7 @@ $$ LANGUAGE SQL; -- Alter table ------------------------------------------------------------------------------------- CREATE OR REPLACE FUNCTION msar.alter_table(tab_id oid, tab_alters jsonb) RETURNS text AS $$/* -Alter columns of the given table in bulk, returning the IDs of the columns so altered. +Alter the name, description, or columns of a table, returning name of the altered table. Args: tab_id: The OID of the table whose columns we'll alter. From 4c41397b34b9a5ab81597f1088c7a6684b3d0145 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 14 Jun 2024 03:24:39 +0530 Subject: [PATCH 0254/1141] some reorganization --- mathesar/imports/csv.py | 91 +---------------------------------------- mathesar/rpc/tables.py | 9 +++- 2 files changed, 9 insertions(+), 91 deletions(-) diff --git a/mathesar/imports/csv.py b/mathesar/imports/csv.py index 428634e92d..93478b8357 100644 --- a/mathesar/imports/csv.py +++ b/mathesar/imports/csv.py @@ -1,20 +1,16 @@ from io import TextIOWrapper -import tempfile + import clevercsv as csv -from psycopg import sql from db.tables.operations.alter import update_pk_sequence_to_latest from mathesar.database.base import create_mathesar_engine from db.records.operations.insert import insert_records_from_csv -from db.tables.operations.create import create_string_column_table, prepare_table_for_import +from db.tables.operations.create import create_string_column_table from db.tables.operations.drop import drop_table -from db.tables.operations.select import get_relation_name from mathesar.errors import InvalidTableError from mathesar.imports.utils import get_alternate_column_names, process_column_names from db.constants import COLUMN_NAME_TEMPLATE from psycopg2.errors import IntegrityError, DataError -from mathesar.models.base import DataFile -from db.encoding_utils import get_sql_compatible_encoding from mathesar.state import reset_reflection # The user-facing documentation replicates these delimiter characters. If you @@ -164,86 +160,3 @@ def create_db_table_from_csv_data_file(data_file, name, schema, comment=None): table = insert_records_from_csv_data_file(name, schema, column_names_alt, engine, comment, data_file) reset_reflection(db_name=db_model.name) return table - - -def insert_csv(data_file_id, table_name, schema_oid, conn, comment=None): - data_file = DataFile.current_objects.get(id=data_file_id) - file_path = data_file.file.path - header = data_file.header - dialect = csv.dialect.SimpleDialect( - data_file.delimiter, - data_file.quotechar, - data_file.escapechar - ) - encoding = get_file_encoding(data_file.file) - with open(file_path, 'rb', encoding=encoding) as csv_file: - csv_reader = get_sv_reader(csv_file, header, dialect) - column_names = process_column_names(csv_reader.fieldnames) - table_oid = prepare_table_for_import( - table_name, - schema_oid, - column_names, - conn, - comment - ) - insert_csv_records( - table_oid, - conn, - csv_file, - column_names, - header, - dialect.delimiter, - dialect.escapechar, - dialect.quotechar, - encoding - ) - - -def insert_csv_records( - table_oid, - conn, - csv_file, - column_names, - header, - delimiter=None, - escape=None, - quote=None, - encoding=None -): - conversion_encoding, sql_encoding = get_sql_compatible_encoding(encoding) - fq_table_name = sql.SQL(get_relation_name(table_oid, conn)) - formatted_columns = sql.SQL(",").join( - sql.Identifier(column_name) for column_name in column_names - ) - copy_sql = sql.SQL( - f"COPY {fq_table_name} ({formatted_columns}) FROM STDIN CSV {header} {delimiter} {escape} {quote} {encoding}" - ).format( - fq_table_name=fq_table_name, - formatted_columns=formatted_columns, - header=sql.SQL("HEADER" if header else ""), - delimiter=sql.SQL(f"DELIMITER E'{delimiter}'" if delimiter else ""), - escape=sql.SQL(f"ESCAPE '{escape}'" if escape else ""), - quote=sql.SQL( - ("QUOTE ''''" if quote == "'" else f"QUOTE '{quote}'") - if quote - else "" - ), - encoding=sql.SQL(f"ENCODING '{sql_encoding}'" if sql_encoding else ""), - ) - cursor = conn.connection.cursor() - if conversion_encoding == encoding: - with cursor.copy(copy_sql) as copy: - if data := csv_file.read(): - copy.write(data) - else: - # File needs to be converted to compatible database supported encoding - with tempfile.SpooledTemporaryFile(mode='wb+', encoding=conversion_encoding) as temp_file: - while True: - contents = csv_file.read(SAMPLE_SIZE).encode(conversion_encoding, "replace") - if not contents: - break - temp_file.write(contents) - temp_file.seek(0) - with cursor.copy(copy_sql) as copy: - if data := temp_file.read(): - copy.write(data) diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 931a89dac1..783e3262e4 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -7,6 +7,7 @@ from db.tables.operations.drop import drop_table_from_database from db.tables.operations.create import create_table_on_database from db.tables.operations.alter import alter_table_on_database +from db.tables.operations.import_ import import_csv from mathesar.rpc.columns import CreateableColumnInfo, SettableColumnInfo from mathesar.rpc.constraints import CreateableConstraintInfo from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -174,5 +175,9 @@ def patch( @rpc_method(name="tables.import") @http_basic_auth_login_required @handle_rpc_exceptions -def import_(): - pass +def import_( + *, data_file_id: int, table_name: str, schema_oid: int, database_id: int, comment=None, **kwargs +) -> int: + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + return import_csv(data_file_id, table_name, schema_oid, conn, comment) From 401aed3ab5441a93a482c94cdb047418628a8748 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 14 Jun 2024 17:29:53 +0800 Subject: [PATCH 0255/1141] move old models to deprecated namespace --- mathesar/admin.py | 2 +- mathesar/api/db/viewsets/columns.py | 2 +- mathesar/api/db/viewsets/constraints.py | 2 +- mathesar/api/db/viewsets/data_files.py | 2 +- mathesar/api/db/viewsets/databases.py | 2 +- mathesar/api/db/viewsets/records.py | 2 +- mathesar/api/db/viewsets/schemas.py | 2 +- mathesar/api/db/viewsets/table_settings.py | 2 +- mathesar/api/db/viewsets/tables.py | 2 +- mathesar/api/dj_filters.py | 2 +- .../database_exceptions/exceptions.py | 2 +- mathesar/api/pagination.py | 2 +- mathesar/api/serializers/columns.py | 2 +- mathesar/api/serializers/constraints.py | 2 +- mathesar/api/serializers/data_files.py | 2 +- mathesar/api/serializers/databases.py | 2 +- mathesar/api/serializers/dependents.py | 2 +- mathesar/api/serializers/links.py | 2 +- mathesar/api/serializers/queries.py | 2 +- mathesar/api/serializers/records.py | 2 +- mathesar/api/serializers/schemas.py | 2 +- mathesar/api/serializers/table_settings.py | 2 +- mathesar/api/serializers/tables.py | 2 +- mathesar/api/ui/permissions/database_role.py | 2 +- mathesar/api/ui/permissions/schema_role.py | 2 +- mathesar/api/ui/serializers/users.py | 2 +- mathesar/api/ui/viewsets/databases.py | 2 +- mathesar/api/ui/viewsets/records.py | 2 +- mathesar/api/utils.py | 2 +- mathesar/imports/base.py | 2 +- mathesar/install.py | 2 +- mathesar/migrations/0005_release_0_1_4.py | 4 +- ...007_rename_database_connection_and_more.py | 4 +- mathesar/models/{base.py => deprecated.py} | 0 mathesar/models/query.py | 2 +- mathesar/models/shares.py | 2 +- mathesar/models/users.py | 2 +- mathesar/rpc/utils.py | 2 +- mathesar/signals.py | 2 +- mathesar/state/django.py | 50 +++++++++---------- mathesar/tests/api/conftest.py | 2 +- mathesar/tests/api/query/test_query_run.py | 2 +- .../api/test_column_api_display_options.py | 2 +- mathesar/tests/api/test_constraint_api.py | 2 +- mathesar/tests/api/test_data_file_api.py | 2 +- mathesar/tests/api/test_database_api.py | 2 +- mathesar/tests/api/test_links.py | 2 +- mathesar/tests/api/test_long_identifiers.py | 2 +- mathesar/tests/api/test_record_api.py | 6 +-- mathesar/tests/api/test_schema_api.py | 2 +- mathesar/tests/api/test_table_api.py | 2 +- mathesar/tests/api/test_table_settings_api.py | 8 +-- mathesar/tests/api/test_ui_filters_api.py | 2 +- mathesar/tests/api/test_ui_types_api.py | 2 +- mathesar/tests/api/test_user_api.py | 2 +- mathesar/tests/conftest.py | 4 +- mathesar/tests/database/test_types.py | 2 +- .../display_options_inference/test_money.py | 2 +- mathesar/tests/imports/test_csv.py | 2 +- mathesar/tests/imports/test_excel.py | 2 +- mathesar/tests/imports/test_json.py | 2 +- mathesar/tests/test_models.py | 2 +- mathesar/tests/test_multi_db.py | 2 +- mathesar/utils/columns.py | 2 +- mathesar/utils/connections.py | 2 +- mathesar/utils/datafiles.py | 2 +- mathesar/utils/joins.py | 2 +- mathesar/utils/preview.py | 2 +- mathesar/utils/schemas.py | 2 +- mathesar/utils/tables.py | 2 +- mathesar/views.py | 2 +- 71 files changed, 102 insertions(+), 102 deletions(-) rename mathesar/models/{base.py => deprecated.py} (100%) diff --git a/mathesar/admin.py b/mathesar/admin.py index 213b0595e2..6eca4ac3d5 100644 --- a/mathesar/admin.py +++ b/mathesar/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin -from mathesar.models.base import Table, Schema, DataFile +from mathesar.models.deprecated import Table, Schema, DataFile from mathesar.models.users import User from mathesar.models.query import UIQuery from mathesar.models.shares import SharedTable, SharedQuery diff --git a/mathesar/api/db/viewsets/columns.py b/mathesar/api/db/viewsets/columns.py index cd535b76e4..6e6b2b76af 100644 --- a/mathesar/api/db/viewsets/columns.py +++ b/mathesar/api/db/viewsets/columns.py @@ -20,7 +20,7 @@ from mathesar.api.pagination import DefaultLimitOffsetPagination from mathesar.api.serializers.columns import ColumnSerializer from mathesar.api.utils import get_table_or_404 -from mathesar.models.base import Column +from mathesar.models.deprecated import Column class ColumnViewSet(AccessViewSetMixin, viewsets.ModelViewSet): diff --git a/mathesar/api/db/viewsets/constraints.py b/mathesar/api/db/viewsets/constraints.py index f276ae8af4..9e4aece735 100644 --- a/mathesar/api/db/viewsets/constraints.py +++ b/mathesar/api/db/viewsets/constraints.py @@ -12,7 +12,7 @@ from mathesar.api.pagination import DefaultLimitOffsetPagination from mathesar.api.serializers.constraints import ConstraintSerializer from mathesar.api.utils import get_table_or_404 -from mathesar.models.base import Constraint +from mathesar.models.deprecated import Constraint class ConstraintViewSet(AccessViewSetMixin, ListModelMixin, RetrieveModelMixin, CreateModelMixin, viewsets.GenericViewSet): diff --git a/mathesar/api/db/viewsets/data_files.py b/mathesar/api/db/viewsets/data_files.py index fc20278de3..4c3eb1a29f 100644 --- a/mathesar/api/db/viewsets/data_files.py +++ b/mathesar/api/db/viewsets/data_files.py @@ -10,7 +10,7 @@ import mathesar.api.exceptions.generic_exceptions.base_exceptions as base_api_exceptions from mathesar.api.exceptions.error_codes import ErrorCodes from mathesar.errors import InvalidTableError -from mathesar.models.base import DataFile +from mathesar.models.deprecated import DataFile from mathesar.api.pagination import DefaultLimitOffsetPagination from mathesar.api.serializers.data_files import DataFileSerializer from mathesar.utils.datafiles import create_datafile diff --git a/mathesar/api/db/viewsets/databases.py b/mathesar/api/db/viewsets/databases.py index b09b94b027..0edac6828e 100644 --- a/mathesar/api/db/viewsets/databases.py +++ b/mathesar/api/db/viewsets/databases.py @@ -5,7 +5,7 @@ from rest_framework.response import Response from mathesar.api.db.permissions.database import DatabaseAccessPolicy -from mathesar.models.base import Connection +from mathesar.models.deprecated import Connection from mathesar.api.dj_filters import DatabaseFilter from mathesar.api.pagination import DefaultLimitOffsetPagination diff --git a/mathesar/api/db/viewsets/records.py b/mathesar/api/db/viewsets/records.py index 4e14a6df83..b28782371f 100644 --- a/mathesar/api/db/viewsets/records.py +++ b/mathesar/api/db/viewsets/records.py @@ -22,7 +22,7 @@ from mathesar.api.serializers.records import RecordListParameterSerializer, RecordSerializer from mathesar.api.utils import get_table_or_404 from mathesar.functions.operations.convert import rewrite_db_function_spec_column_ids_to_names -from mathesar.models.base import Table +from mathesar.models.deprecated import Table from mathesar.utils.json import MathesarJSONRenderer diff --git a/mathesar/api/db/viewsets/schemas.py b/mathesar/api/db/viewsets/schemas.py index fb7dc64f24..153c081198 100644 --- a/mathesar/api/db/viewsets/schemas.py +++ b/mathesar/api/db/viewsets/schemas.py @@ -10,7 +10,7 @@ from mathesar.api.pagination import DefaultLimitOffsetPagination from mathesar.api.serializers.dependents import DependentSerializer, DependentFilterSerializer from mathesar.api.serializers.schemas import SchemaSerializer -from mathesar.models.base import Schema +from mathesar.models.deprecated import Schema from mathesar.utils.schemas import create_schema_and_object from mathesar.api.exceptions.validation_exceptions.exceptions import EditingPublicSchemaIsDisallowed diff --git a/mathesar/api/db/viewsets/table_settings.py b/mathesar/api/db/viewsets/table_settings.py index cb2b525f65..62690e0ad0 100644 --- a/mathesar/api/db/viewsets/table_settings.py +++ b/mathesar/api/db/viewsets/table_settings.py @@ -5,7 +5,7 @@ from mathesar.api.pagination import DefaultLimitOffsetPagination from mathesar.api.serializers.table_settings import TableSettingsSerializer from mathesar.api.utils import get_table_or_404 -from mathesar.models.base import TableSettings +from mathesar.models.deprecated import TableSettings class TableSettingsViewSet(AccessViewSetMixin, ModelViewSet): diff --git a/mathesar/api/db/viewsets/tables.py b/mathesar/api/db/viewsets/tables.py index 635ab234d4..6bb4a533f6 100644 --- a/mathesar/api/db/viewsets/tables.py +++ b/mathesar/api/db/viewsets/tables.py @@ -28,7 +28,7 @@ TableImportSerializer, MoveTableRequestSerializer ) -from mathesar.models.base import Table +from mathesar.models.deprecated import Table from mathesar.utils.tables import get_table_column_types from mathesar.utils.joins import get_processed_joinable_tables diff --git a/mathesar/api/dj_filters.py b/mathesar/api/dj_filters.py index e22d3ff03d..bb850f8ebb 100644 --- a/mathesar/api/dj_filters.py +++ b/mathesar/api/dj_filters.py @@ -1,7 +1,7 @@ from django_filters import BooleanFilter, DateTimeFromToRangeFilter, OrderingFilter from django_property_filter import PropertyFilterSet, PropertyBaseInFilter, PropertyCharFilter, PropertyOrderingFilter -from mathesar.models.base import Schema, Table, Connection, DataFile +from mathesar.models.deprecated import Schema, Table, Connection, DataFile from mathesar.models.query import UIQuery diff --git a/mathesar/api/exceptions/database_exceptions/exceptions.py b/mathesar/api/exceptions/database_exceptions/exceptions.py index a74727b764..f9e7d3c089 100644 --- a/mathesar/api/exceptions/database_exceptions/exceptions.py +++ b/mathesar/api/exceptions/database_exceptions/exceptions.py @@ -13,7 +13,7 @@ MathesarAPIException, get_default_exception_detail, ) -from mathesar.models.base import Column, Constraint +from mathesar.models.deprecated import Column, Constraint from mathesar.state import get_cached_metadata diff --git a/mathesar/api/pagination.py b/mathesar/api/pagination.py index 3a730c1832..ce0f137b57 100644 --- a/mathesar/api/pagination.py +++ b/mathesar/api/pagination.py @@ -5,7 +5,7 @@ from db.records.operations.group import GroupBy from mathesar.api.utils import get_table_or_404, process_annotated_records -from mathesar.models.base import Column, Table +from mathesar.models.deprecated import Column, Table from mathesar.models.query import UIQuery from mathesar.utils.preview import get_preview_info diff --git a/mathesar/api/serializers/columns.py b/mathesar/api/serializers/columns.py index 338d3c166e..3abaa7e1b1 100644 --- a/mathesar/api/serializers/columns.py +++ b/mathesar/api/serializers/columns.py @@ -17,7 +17,7 @@ DisplayOptionsMappingSerializer, DISPLAY_OPTIONS_SERIALIZER_MAPPING_KEY, ) -from mathesar.models.base import Column +from mathesar.models.deprecated import Column class InputValueField(serializers.CharField): diff --git a/mathesar/api/serializers/constraints.py b/mathesar/api/serializers/constraints.py index 0dc9b8cde8..15401f170b 100644 --- a/mathesar/api/serializers/constraints.py +++ b/mathesar/api/serializers/constraints.py @@ -14,7 +14,7 @@ MathesarPolymorphicErrorMixin, ReadWritePolymorphicSerializerMappingMixin, ) -from mathesar.models.base import Column, Constraint, Table +from mathesar.models.deprecated import Column, Constraint, Table class TableFilteredPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): diff --git a/mathesar/api/serializers/data_files.py b/mathesar/api/serializers/data_files.py index 2c684b0776..89cf7f9907 100644 --- a/mathesar/api/serializers/data_files.py +++ b/mathesar/api/serializers/data_files.py @@ -5,7 +5,7 @@ from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin from mathesar.errors import URLNotReachable, URLInvalidContentTypeError -from mathesar.models.base import DataFile +from mathesar.models.deprecated import DataFile SUPPORTED_URL_CONTENT_TYPES = {'text/csv', 'text/plain'} diff --git a/mathesar/api/serializers/databases.py b/mathesar/api/serializers/databases.py index 8a8ab575a7..4ddb3e76e4 100644 --- a/mathesar/api/serializers/databases.py +++ b/mathesar/api/serializers/databases.py @@ -3,7 +3,7 @@ from mathesar.api.display_options import DISPLAY_OPTIONS_BY_UI_TYPE from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin -from mathesar.models.base import Connection +from mathesar.models.deprecated import Connection class ConnectionSerializer(MathesarErrorMessageMixin, serializers.ModelSerializer): diff --git a/mathesar/api/serializers/dependents.py b/mathesar/api/serializers/dependents.py index 2cf5c7b792..1b7751793b 100644 --- a/mathesar/api/serializers/dependents.py +++ b/mathesar/api/serializers/dependents.py @@ -1,7 +1,7 @@ from mathesar.api.serializers.shared_serializers import MathesarPolymorphicErrorMixin, ReadOnlyPolymorphicSerializerMappingMixin from rest_framework import serializers -from mathesar.models.base import Constraint, Schema, Table +from mathesar.models.deprecated import Constraint, Schema, Table DATABASE_OBJECT_TYPES = [ diff --git a/mathesar/api/serializers/links.py b/mathesar/api/serializers/links.py index 32156aa41e..fe36bf7326 100644 --- a/mathesar/api/serializers/links.py +++ b/mathesar/api/serializers/links.py @@ -11,7 +11,7 @@ MathesarPolymorphicErrorMixin, ReadWritePolymorphicSerializerMappingMixin, ) -from mathesar.models.base import Table +from mathesar.models.deprecated import Table from mathesar.state import reset_reflection diff --git a/mathesar/api/serializers/queries.py b/mathesar/api/serializers/queries.py index 1eeb705ffc..2e4187d6b0 100644 --- a/mathesar/api/serializers/queries.py +++ b/mathesar/api/serializers/queries.py @@ -8,7 +8,7 @@ from mathesar.api.db.permissions.query_table import QueryTableAccessPolicy from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin from mathesar.api.exceptions.validation_exceptions.exceptions import DuplicateUIQueryInSchemaAPIException -from mathesar.models.base import Table +from mathesar.models.deprecated import Table from mathesar.models.query import UIQuery diff --git a/mathesar/api/serializers/records.py b/mathesar/api/serializers/records.py index 9367c5dff4..506b277c9c 100644 --- a/mathesar/api/serializers/records.py +++ b/mathesar/api/serializers/records.py @@ -6,7 +6,7 @@ import mathesar.api.exceptions.database_exceptions.exceptions as database_api_exceptions from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin -from mathesar.models.base import Column +from mathesar.models.deprecated import Column from mathesar.api.utils import follows_json_number_spec from mathesar.database.types import UIType diff --git a/mathesar/api/serializers/schemas.py b/mathesar/api/serializers/schemas.py index 9dbd752817..5bdf290285 100644 --- a/mathesar/api/serializers/schemas.py +++ b/mathesar/api/serializers/schemas.py @@ -6,7 +6,7 @@ from mathesar.api.db.permissions.table import TableAccessPolicy from mathesar.api.db.permissions.database import DatabaseAccessPolicy from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin -from mathesar.models.base import Connection, Schema, Table +from mathesar.models.deprecated import Connection, Schema, Table from mathesar.api.exceptions.database_exceptions import ( exceptions as database_api_exceptions ) diff --git a/mathesar/api/serializers/table_settings.py b/mathesar/api/serializers/table_settings.py index 576fb0c569..d0b7ddc6e4 100644 --- a/mathesar/api/serializers/table_settings.py +++ b/mathesar/api/serializers/table_settings.py @@ -2,7 +2,7 @@ from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin from mathesar.api.exceptions.validation_exceptions.exceptions import InvalidColumnOrder -from mathesar.models.base import PreviewColumnSettings, TableSettings, compute_default_preview_template, ValidationError +from mathesar.models.deprecated import PreviewColumnSettings, TableSettings, compute_default_preview_template, ValidationError class PreviewColumnSerializer(MathesarErrorMessageMixin, serializers.ModelSerializer): diff --git a/mathesar/api/serializers/tables.py b/mathesar/api/serializers/tables.py index ea3fecce3a..3f8e1a871a 100644 --- a/mathesar/api/serializers/tables.py +++ b/mathesar/api/serializers/tables.py @@ -26,7 +26,7 @@ from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin from mathesar.api.serializers.columns import SimpleColumnSerializer from mathesar.api.serializers.table_settings import TableSettingsSerializer -from mathesar.models.base import Column, Schema, Table, DataFile +from mathesar.models.deprecated import Column, Schema, Table, DataFile from mathesar.utils.tables import gen_table_name, create_table_from_datafile, create_empty_table diff --git a/mathesar/api/ui/permissions/database_role.py b/mathesar/api/ui/permissions/database_role.py index 20d0c2205c..090a4ded6e 100644 --- a/mathesar/api/ui/permissions/database_role.py +++ b/mathesar/api/ui/permissions/database_role.py @@ -1,7 +1,7 @@ from django.db.models import Q from rest_access_policy import AccessPolicy -from mathesar.models.base import Connection +from mathesar.models.deprecated import Connection from mathesar.models.users import DatabaseRole, Role diff --git a/mathesar/api/ui/permissions/schema_role.py b/mathesar/api/ui/permissions/schema_role.py index 5b6ef84d21..b793fc98da 100644 --- a/mathesar/api/ui/permissions/schema_role.py +++ b/mathesar/api/ui/permissions/schema_role.py @@ -1,7 +1,7 @@ from django.db.models import Q from rest_access_policy import AccessPolicy -from mathesar.models.base import Connection, Schema +from mathesar.models.deprecated import Connection, Schema from mathesar.models.users import DatabaseRole, Role, SchemaRole diff --git a/mathesar/api/ui/serializers/users.py b/mathesar/api/ui/serializers/users.py index d092ba9524..4ff9069b6a 100644 --- a/mathesar/api/ui/serializers/users.py +++ b/mathesar/api/ui/serializers/users.py @@ -8,7 +8,7 @@ from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin from mathesar.api.exceptions.validation_exceptions.exceptions import IncorrectOldPassword from mathesar.api.ui.permissions.users import UserAccessPolicy -from mathesar.models.base import Connection, Schema +from mathesar.models.deprecated import Connection, Schema from mathesar.models.users import User, DatabaseRole, SchemaRole diff --git a/mathesar/api/ui/viewsets/databases.py b/mathesar/api/ui/viewsets/databases.py index b01f539849..01a9ab26ac 100644 --- a/mathesar/api/ui/viewsets/databases.py +++ b/mathesar/api/ui/viewsets/databases.py @@ -7,7 +7,7 @@ from rest_framework.response import Response from mathesar.api.ui.permissions.ui_database import UIDatabaseAccessPolicy -from mathesar.models.base import Connection +from mathesar.models.deprecated import Connection from mathesar.api.dj_filters import DatabaseFilter from mathesar.api.exceptions.validation_exceptions.exceptions import ( DictHasBadKeys, UnsupportedInstallationDatabase diff --git a/mathesar/api/ui/viewsets/records.py b/mathesar/api/ui/viewsets/records.py index e59a1f9ece..981452e5a0 100644 --- a/mathesar/api/ui/viewsets/records.py +++ b/mathesar/api/ui/viewsets/records.py @@ -9,7 +9,7 @@ import mathesar.api.exceptions.database_exceptions.exceptions as database_api_exceptions from mathesar.api.utils import get_table_or_404 -from mathesar.models.base import Table +from mathesar.models.deprecated import Table class RecordViewSet(AccessViewSetMixin, viewsets.GenericViewSet): diff --git a/mathesar/api/utils.py b/mathesar/api/utils.py index 3f3a4382f6..d7a24d8927 100644 --- a/mathesar/api/utils.py +++ b/mathesar/api/utils.py @@ -6,7 +6,7 @@ from db.records.operations import group from mathesar.api.exceptions.error_codes import ErrorCodes -from mathesar.models.base import Table +from mathesar.models.deprecated import Table from mathesar.models.query import UIQuery from mathesar.utils.preview import column_alias_from_preview_template from mathesar.api.exceptions.generic_exceptions.base_exceptions import BadDBCredentials diff --git a/mathesar/imports/base.py b/mathesar/imports/base.py index 4805eb6d04..a43ba100b3 100644 --- a/mathesar/imports/base.py +++ b/mathesar/imports/base.py @@ -1,4 +1,4 @@ -from mathesar.models.base import Table +from mathesar.models.deprecated import Table from mathesar.imports.csv import create_db_table_from_csv_data_file from mathesar.imports.excel import create_db_table_from_excel_data_file from mathesar.imports.json import create_db_table_from_json_data_file diff --git a/mathesar/install.py b/mathesar/install.py index 5f4d5973a3..e026bd9591 100644 --- a/mathesar/install.py +++ b/mathesar/install.py @@ -40,7 +40,7 @@ def main(skip_static_collection=False): def install_on_db_with_key(database_key, skip_confirm): - from mathesar.models.base import Connection + from mathesar.models.deprecated import Connection db_model = Connection.create_from_settings_key(database_key) db_model.save() try: diff --git a/mathesar/migrations/0005_release_0_1_4.py b/mathesar/migrations/0005_release_0_1_4.py index 04df9ca932..0849b2a6e3 100644 --- a/mathesar/migrations/0005_release_0_1_4.py +++ b/mathesar/migrations/0005_release_0_1_4.py @@ -1,7 +1,7 @@ from django.db import migrations, models, connection import django.contrib.postgres.fields import encrypted_fields.fields -import mathesar.models.base +import mathesar.models.deprecated def column_order_to_jsonb_postgres_fwd(apps, schema_editor): @@ -12,7 +12,7 @@ def column_order_to_jsonb_postgres_fwd(apps, schema_editor): migrations.AlterField( model_name='tablesettings', name='column_order', - field=models.JSONField(blank=True, default=None, null=True, validators=[mathesar.models.base.validate_column_order]), + field=models.JSONField(blank=True, default=None, null=True, validators=[mathesar.models.deprecated.validate_column_order]), ), diff --git a/mathesar/migrations/0007_rename_database_connection_and_more.py b/mathesar/migrations/0007_rename_database_connection_and_more.py index d47e6768f1..34a6ad644a 100644 --- a/mathesar/migrations/0007_rename_database_connection_and_more.py +++ b/mathesar/migrations/0007_rename_database_connection_and_more.py @@ -2,7 +2,7 @@ from django.db import migrations, models import django.db.models.deletion -import mathesar.models.base +import mathesar.models.deprecated class Migration(migrations.Migration): @@ -29,6 +29,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='tablesettings', name='column_order', - field=models.JSONField(blank=True, default=None, null=True, validators=[mathesar.models.base.validate_column_order]), + field=models.JSONField(blank=True, default=None, null=True, validators=[mathesar.models.deprecated.validate_column_order]), ), ] diff --git a/mathesar/models/base.py b/mathesar/models/deprecated.py similarity index 100% rename from mathesar/models/base.py rename to mathesar/models/deprecated.py diff --git a/mathesar/models/query.py b/mathesar/models/query.py index 539dcead89..06eed199e6 100644 --- a/mathesar/models/query.py +++ b/mathesar/models/query.py @@ -29,7 +29,7 @@ TransformationsValidator, ) from mathesar.state.cached_property import cached_property -from mathesar.models.base import BaseModel, Column +from mathesar.models.deprecated import BaseModel, Column from mathesar.models.relation import Relation from mathesar.state import get_cached_metadata diff --git a/mathesar/models/shares.py b/mathesar/models/shares.py index 20f076080a..de78b406c3 100644 --- a/mathesar/models/shares.py +++ b/mathesar/models/shares.py @@ -1,7 +1,7 @@ import uuid from django.db import models -from mathesar.models.base import BaseModel +from mathesar.models.deprecated import BaseModel class SharedEntity(BaseModel): diff --git a/mathesar/models/users.py b/mathesar/models/users.py index 4db3748959..693348e558 100644 --- a/mathesar/models/users.py +++ b/mathesar/models/users.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import AbstractUser from django.db import models -from mathesar.models.base import BaseModel, Connection, Schema +from mathesar.models.deprecated import BaseModel, Connection, Schema class User(AbstractUser): diff --git a/mathesar/rpc/utils.py b/mathesar/rpc/utils.py index 7ab6ce44ef..57118f0f9b 100644 --- a/mathesar/rpc/utils.py +++ b/mathesar/rpc/utils.py @@ -1,5 +1,5 @@ from mathesar.database.base import get_psycopg_connection -from mathesar.models.base import Connection +from mathesar.models.deprecated import Connection def connect(conn_id, user): diff --git a/mathesar/signals.py b/mathesar/signals.py index 83b673e93b..bcec182ab1 100644 --- a/mathesar/signals.py +++ b/mathesar/signals.py @@ -1,7 +1,7 @@ from django.db.models.signals import post_save from django.dispatch import receiver -from mathesar.models.base import ( +from mathesar.models.deprecated import ( Column, Table, _set_default_preview_template, _create_table_settings, ) diff --git a/mathesar/state/django.py b/mathesar/state/django.py index e60a671fd5..ebb5cfbcfb 100644 --- a/mathesar/state/django.py +++ b/mathesar/state/django.py @@ -11,8 +11,8 @@ from db.constraints.operations.select import get_constraints_with_oids from db.schemas.operations.select import get_mathesar_schemas_with_oids from db.tables.operations.select import get_table_oids_from_schemas -# We import the entire models.base module to avoid a circular import error -from mathesar.models import base as models +# We import the entire models_deprecated.deprecated module to avoid a circular import error +from mathesar.models import deprecated as models_deprecated from mathesar.api.serializers.shared_serializers import DisplayOptionsMappingSerializer, \ DISPLAY_OPTIONS_SERIALIZER_MAPPING_KEY from mathesar.database.base import create_mathesar_engine @@ -32,24 +32,24 @@ def clear_dj_cache(): def reflect_db_objects(metadata, db_name=None): - databases = models.Connection.current_objects.all() + databases = models_deprecated.Connection.current_objects.all() if db_name is not None: databases = databases.filter(name=db_name) sync_databases_status(databases) for database in databases: if database.deleted is False: reflect_schemas_from_database(database) - schemas = models.Schema.current_objects.filter(database=database).prefetch_related( + schemas = models_deprecated.Schema.current_objects.filter(database=database).prefetch_related( Prefetch('database', queryset=databases) ) reflect_tables_from_schemas(schemas, metadata=metadata) - tables = models.Table.current_objects.filter(schema__in=schemas).prefetch_related( + tables = models_deprecated.Table.current_objects.filter(schema__in=schemas).prefetch_related( Prefetch('schema', queryset=schemas) ) reflect_columns_from_tables(tables, metadata=metadata) reflect_constraints_from_database(database) else: - models.Schema.current_objects.filter(database=database).delete() + models_deprecated.Schema.current_objects.filter(database=database).delete() def sync_databases_status(databases): @@ -83,10 +83,10 @@ def reflect_schemas_from_database(database): schemas = [] for oid in db_schema_oids: - schema = models.Schema(oid=oid, database=database) + schema = models_deprecated.Schema(oid=oid, database=database) schemas.append(schema) - models.Schema.current_objects.bulk_create(schemas, ignore_conflicts=True) - for schema in models.Schema.current_objects.all().select_related('database'): + models_deprecated.Schema.current_objects.bulk_create(schemas, ignore_conflicts=True) + for schema in models_deprecated.Schema.current_objects.all().select_related('database'): if schema.database == database and schema.oid not in db_schema_oids: # Deleting Schemas are a rare occasion, not worth deleting in bulk schema.delete() @@ -105,17 +105,17 @@ def reflect_tables_from_schemas(schemas, metadata): tables = [] for oid, schema_oid in db_table_oids: schema = next(schema for schema in schemas if schema.oid == schema_oid) - table = models.Table(oid=oid, schema=schema) + table = models_deprecated.Table(oid=oid, schema=schema) tables.append(table) - models.Table.current_objects.bulk_create(tables, ignore_conflicts=True) + models_deprecated.Table.current_objects.bulk_create(tables, ignore_conflicts=True) # Calling signals manually because bulk create does not emit any signals - models._create_table_settings(models.Table.current_objects.filter(settings__isnull=True)) + models_deprecated._create_table_settings(models_deprecated.Table.current_objects.filter(settings__isnull=True)) deleted_tables = [] - for table in models.Table.current_objects.filter(schema__in=schemas).select_related('schema'): + for table in models_deprecated.Table.current_objects.filter(schema__in=schemas).select_related('schema'): if (table.oid, table.schema.oid) not in db_table_oids: deleted_tables.append(table.id) - models.Table.current_objects.filter(id__in=deleted_tables).delete() + models_deprecated.Table.current_objects.filter(id__in=deleted_tables).delete() def reflect_columns_from_tables(tables, metadata): @@ -130,14 +130,14 @@ def reflect_columns_from_tables(tables, metadata): _delete_stale_columns(attnum_tuples, tables) # Manually trigger preview templates computation signal for table in tables: - models._set_default_preview_template(table) + models_deprecated._set_default_preview_template(table) _invalidate_columns_with_incorrect_display_options(tables) def _invalidate_columns_with_incorrect_display_options(tables): columns_with_invalid_display_option = [] - columns = models.Column.current_objects.filter(table__in=tables) + columns = models_deprecated.Column.current_objects.filter(table__in=tables) for column in columns: if column.display_options: # If the type of column has changed, existing display options won't be valid anymore. @@ -148,16 +148,16 @@ def _invalidate_columns_with_incorrect_display_options(tables): if not serializer.is_valid(raise_exception=False): columns_with_invalid_display_option.append(column.id) if len(columns_with_invalid_display_option) > 0: - models.Column.current_objects.filter(id__in=columns_with_invalid_display_option).update(display_options=None) + models_deprecated.Column.current_objects.filter(id__in=columns_with_invalid_display_option).update(display_options=None) def _create_reflected_columns(attnum_tuples, tables): columns = [] for attnum, table_oid in attnum_tuples: table = next(table for table in tables if table.oid == table_oid) - column = models.Column(attnum=attnum, table=table, display_options=None) + column = models_deprecated.Column(attnum=attnum, table=table, display_options=None) columns.append(column) - models.Column.current_objects.bulk_create(columns, ignore_conflicts=True) + models_deprecated.Column.current_objects.bulk_create(columns, ignore_conflicts=True) def _delete_stale_columns(attnum_tuples, tables): @@ -176,7 +176,7 @@ def _delete_stale_columns(attnum_tuples, tables): operator.or_, stale_columns_conditions ) - models.Column.objects.filter(stale_columns_query).delete() + models_deprecated.Column.objects.filter(stale_columns_query).delete() # TODO pass in a cached engine instead of creating a new one @@ -190,14 +190,14 @@ def reflect_constraints_from_database(database): map_of_table_oid_to_constraint_oids[table_oid].append(constraint_oid) table_oids = map_of_table_oid_to_constraint_oids.keys() - tables = models.Table.current_objects.filter(oid__in=table_oids) + tables = models_deprecated.Table.current_objects.filter(oid__in=table_oids) constraint_objs_to_create = [] for table in tables: constraint_oids = map_of_table_oid_to_constraint_oids.get(table.oid, []) for constraint_oid in constraint_oids: - constraint_obj = models.Constraint(oid=constraint_oid, table=table) + constraint_obj = models_deprecated.Constraint(oid=constraint_oid, table=table) constraint_objs_to_create.append(constraint_obj) - models.Constraint.current_objects.bulk_create(constraint_objs_to_create, ignore_conflicts=True) + models_deprecated.Constraint.current_objects.bulk_create(constraint_objs_to_create, ignore_conflicts=True) _delete_stale_dj_constraints(db_constraints, database) engine.dispose() @@ -212,7 +212,7 @@ def _delete_stale_dj_constraints(known_db_constraints, database): for known_db_constraint in known_db_constraints ) - stale_dj_constraints = models.Constraint.current_objects.filter( + stale_dj_constraints = models_deprecated.Constraint.current_objects.filter( ~Q(oid__in=known_db_constraint_oids), table__schema__database=database, ) @@ -224,7 +224,7 @@ def reflect_new_table_constraints(table): engine = create_mathesar_engine(table.schema.database) db_constraints = get_constraints_with_oids(engine, table_oid=table.oid) constraints = [ - models.Constraint.current_objects.get_or_create( + models_deprecated.Constraint.current_objects.get_or_create( oid=db_constraint['oid'], table=table ) diff --git a/mathesar/tests/api/conftest.py b/mathesar/tests/api/conftest.py index 5cc592ee9a..5d0318d51b 100644 --- a/mathesar/tests/api/conftest.py +++ b/mathesar/tests/api/conftest.py @@ -9,7 +9,7 @@ from db.constraints.base import ForeignKeyConstraint, UniqueConstraint from db.tables.operations.select import get_oid_from_table from db.types.base import PostgresType -from mathesar.models.base import Table, DataFile, Column as ServiceLayerColumn +from mathesar.models.deprecated import Table, DataFile, Column as ServiceLayerColumn from db.metadata import get_empty_metadata from mathesar.state import reset_reflection diff --git a/mathesar/tests/api/query/test_query_run.py b/mathesar/tests/api/query/test_query_run.py index de0ada4541..5e477506ef 100644 --- a/mathesar/tests/api/query/test_query_run.py +++ b/mathesar/tests/api/query/test_query_run.py @@ -1,7 +1,7 @@ import pytest from mathesar.api.exceptions.error_codes import ErrorCodes -from mathesar.models.base import Column +from mathesar.models.deprecated import Column run_client_with_status_code = [ ('db_manager_client_factory', 200, 200), diff --git a/mathesar/tests/api/test_column_api_display_options.py b/mathesar/tests/api/test_column_api_display_options.py index 46126530b4..ae421e5ae1 100644 --- a/mathesar/tests/api/test_column_api_display_options.py +++ b/mathesar/tests/api/test_column_api_display_options.py @@ -9,7 +9,7 @@ from db.types.custom.money import MathesarMoney from db.metadata import get_empty_metadata -from mathesar.models.base import Table, Column +from mathesar.models.deprecated import Table, Column @pytest.fixture diff --git a/mathesar/tests/api/test_constraint_api.py b/mathesar/tests/api/test_constraint_api.py index 91256846ca..3258032116 100644 --- a/mathesar/tests/api/test_constraint_api.py +++ b/mathesar/tests/api/test_constraint_api.py @@ -7,7 +7,7 @@ from db.columns.operations.select import get_column_attnum_from_name from db.constraints.base import UniqueConstraint from db.tables.operations.select import get_oid_from_table -from mathesar.models.base import Constraint, Table, Column +from mathesar.models.deprecated import Constraint, Table, Column from mathesar.api.exceptions.error_codes import ErrorCodes from db.metadata import get_empty_metadata from mathesar.state import reset_reflection diff --git a/mathesar/tests/api/test_data_file_api.py b/mathesar/tests/api/test_data_file_api.py index 661adafe08..deec6fe884 100644 --- a/mathesar/tests/api/test_data_file_api.py +++ b/mathesar/tests/api/test_data_file_api.py @@ -8,7 +8,7 @@ from mathesar.api.exceptions.error_codes import ErrorCodes from mathesar.imports import csv -from mathesar.models.base import DataFile +from mathesar.models.deprecated import DataFile from mathesar.errors import InvalidTableError diff --git a/mathesar/tests/api/test_database_api.py b/mathesar/tests/api/test_database_api.py index d8333b9708..6aeb80f931 100644 --- a/mathesar/tests/api/test_database_api.py +++ b/mathesar/tests/api/test_database_api.py @@ -5,7 +5,7 @@ from db.metadata import get_empty_metadata from mathesar.models.users import DatabaseRole from mathesar.state.django import reflect_db_objects -from mathesar.models.base import Table, Schema, Connection +from mathesar.models.deprecated import Table, Schema, Connection from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from db.install import install_mathesar diff --git a/mathesar/tests/api/test_links.py b/mathesar/tests/api/test_links.py index 4b1b1e0644..abd79f42af 100644 --- a/mathesar/tests/api/test_links.py +++ b/mathesar/tests/api/test_links.py @@ -6,7 +6,7 @@ from db.tables.operations.select import get_oid_from_table from db.tables.utils import get_primary_key_column -from mathesar.models.base import Constraint, Table +from mathesar.models.deprecated import Constraint, Table from mathesar.api.exceptions.error_codes import ErrorCodes diff --git a/mathesar/tests/api/test_long_identifiers.py b/mathesar/tests/api/test_long_identifiers.py index d92cd12376..eb6443f00d 100644 --- a/mathesar/tests/api/test_long_identifiers.py +++ b/mathesar/tests/api/test_long_identifiers.py @@ -10,7 +10,7 @@ from mathesar.api.exceptions.database_exceptions import ( exceptions as database_api_exceptions ) -from mathesar.models.base import DataFile +from mathesar.models.deprecated import DataFile from mathesar.tests.api.test_table_api import check_create_table_response, get_expected_name diff --git a/mathesar/tests/api/test_record_api.py b/mathesar/tests/api/test_record_api.py index e67ba8a26c..8fba30e10b 100644 --- a/mathesar/tests/api/test_record_api.py +++ b/mathesar/tests/api/test_record_api.py @@ -14,8 +14,8 @@ from mathesar.api.exceptions.error_codes import ErrorCodes from mathesar.api.utils import follows_json_number_spec from mathesar.functions.operations.convert import rewrite_db_function_spec_column_ids_to_names -from mathesar.models import base as models_base -from mathesar.models.base import compute_default_preview_template +from mathesar.models import deprecated as models_deprecated +from mathesar.models.deprecated import compute_default_preview_template from mathesar.models.query import DBQuery from mathesar.utils.preview import compute_path_prefix, compute_path_str @@ -235,7 +235,7 @@ def test_filter_with_added_columns(create_patents_table, client): table.add_column({"name": new_column_name, "type": new_column_type}) row_values_list = [] # Get a new instance with clean cache, so that the new column is added to the _sa_column list - table = models_base.Table.objects.get(oid=table.oid) + table = models_deprecated.Table.objects.get(oid=table.oid) response_data = client.get(f'/api/db/v0/tables/{table.id}/records/').json() existing_records = response_data['results'] diff --git a/mathesar/tests/api/test_schema_api.py b/mathesar/tests/api/test_schema_api.py index 82166a926b..ac0f02c2f9 100644 --- a/mathesar/tests/api/test_schema_api.py +++ b/mathesar/tests/api/test_schema_api.py @@ -3,7 +3,7 @@ from sqlalchemy import text from db.schemas.utils import get_mathesar_schemas, get_schema_oid_from_name -from mathesar.models.base import Schema +from mathesar.models.deprecated import Schema from mathesar.api.exceptions.error_codes import ErrorCodes from mathesar.models.users import DatabaseRole diff --git a/mathesar/tests/api/test_table_api.py b/mathesar/tests/api/test_table_api.py index 18df6ca1e0..b5ce0b906d 100644 --- a/mathesar/tests/api/test_table_api.py +++ b/mathesar/tests/api/test_table_api.py @@ -14,7 +14,7 @@ from mathesar.state import reset_reflection from mathesar.api.exceptions.error_codes import ErrorCodes -from mathesar.models.base import Column, Table, DataFile +from mathesar.models.deprecated import Column, Table, DataFile @pytest.fixture diff --git a/mathesar/tests/api/test_table_settings_api.py b/mathesar/tests/api/test_table_settings_api.py index adb7131c94..6f1f680a24 100644 --- a/mathesar/tests/api/test_table_settings_api.py +++ b/mathesar/tests/api/test_table_settings_api.py @@ -3,7 +3,7 @@ from sqlalchemy import Table as SATable from db.tables.operations.select import get_oid_from_table -from mathesar.models import base as models_base +from mathesar.models import deprecated as models_deprecated from mathesar.api.exceptions.error_codes import ErrorCodes @@ -31,7 +31,7 @@ def column_test_table(patent_schema, engine): ) db_table.create() db_table_oid = get_oid_from_table(db_table.name, db_table.schema, engine) - table = models_base.Table.current_objects.create(oid=db_table_oid, schema=patent_schema) + table = models_deprecated.Table.current_objects.create(oid=db_table_oid, schema=patent_schema) return table @@ -86,7 +86,7 @@ def test_update_table_settings_permission(create_patents_table, request, client_ table = create_patents_table(table_name) settings_id = table.settings.id client = request.getfixturevalue(client_name)(table.schema) - columns = models_base.Column.objects.filter(table=table).values_list('id', flat=True) + columns = models_deprecated.Column.objects.filter(table=table).values_list('id', flat=True) preview_template = ','.join(f'{{{ column }}}' for column in columns) data = { "preview_settings": { @@ -100,7 +100,7 @@ def test_update_table_settings_permission(create_patents_table, request, client_ def test_update_table_settings(client, column_test_table): - columns = models_base.Column.objects.filter(table=column_test_table).values_list('id', flat=True) + columns = models_deprecated.Column.objects.filter(table=column_test_table).values_list('id', flat=True) preview_template = ','.join(f'{{{ column }}}' for column in columns) settings_id = column_test_table.settings.id column_order = [4, 5, 6] diff --git a/mathesar/tests/api/test_ui_filters_api.py b/mathesar/tests/api/test_ui_filters_api.py index 38e73e01ac..f49516d1d5 100644 --- a/mathesar/tests/api/test_ui_filters_api.py +++ b/mathesar/tests/api/test_ui_filters_api.py @@ -1,4 +1,4 @@ -from mathesar.models.base import Connection +from mathesar.models.deprecated import Connection from mathesar.filters.base import get_available_filters from mathesar.models.users import DatabaseRole diff --git a/mathesar/tests/api/test_ui_types_api.py b/mathesar/tests/api/test_ui_types_api.py index 74450168f5..79d8ab3c46 100644 --- a/mathesar/tests/api/test_ui_types_api.py +++ b/mathesar/tests/api/test_ui_types_api.py @@ -1,5 +1,5 @@ from mathesar.api.display_options import DISPLAY_OPTIONS_BY_UI_TYPE -from mathesar.models.base import Connection +from mathesar.models.deprecated import Connection from mathesar.database.types import get_ui_type_from_id, UIType from db.types.base import PostgresType, MathesarCustomType from mathesar.models.users import DatabaseRole diff --git a/mathesar/tests/api/test_user_api.py b/mathesar/tests/api/test_user_api.py index 8687873ff6..c135081a3f 100644 --- a/mathesar/tests/api/test_user_api.py +++ b/mathesar/tests/api/test_user_api.py @@ -1,7 +1,7 @@ from django.db import transaction from db.schemas.utils import get_schema_oid_from_name -from mathesar.models.base import Connection, Schema +from mathesar.models.deprecated import Connection, Schema from mathesar.models.users import User, DatabaseRole, SchemaRole diff --git a/mathesar/tests/conftest.py b/mathesar/tests/conftest.py index 80b75edfdf..b887648cbc 100644 --- a/mathesar/tests/conftest.py +++ b/mathesar/tests/conftest.py @@ -22,8 +22,8 @@ import mathesar.tests.conftest from mathesar.imports.base import create_table_from_data_file -from mathesar.models.base import Schema, Table, Connection, DataFile -from mathesar.models.base import Column as mathesar_model_column +from mathesar.models.deprecated import Schema, Table, Connection, DataFile +from mathesar.models.deprecated import Column as mathesar_model_column from mathesar.models.users import DatabaseRole, SchemaRole, User from fixtures.utils import create_scoped_fixtures, get_fixture_value diff --git a/mathesar/tests/database/test_types.py b/mathesar/tests/database/test_types.py index f60e5407f7..c74be77335 100644 --- a/mathesar/tests/database/test_types.py +++ b/mathesar/tests/database/test_types.py @@ -1,5 +1,5 @@ from mathesar.database.types import UIType -from mathesar.models.base import Connection +from mathesar.models.deprecated import Connection from db.types.base import known_db_types diff --git a/mathesar/tests/display_options_inference/test_money.py b/mathesar/tests/display_options_inference/test_money.py index 5eaaa95447..5698f4383b 100644 --- a/mathesar/tests/display_options_inference/test_money.py +++ b/mathesar/tests/display_options_inference/test_money.py @@ -3,7 +3,7 @@ from db.columns.operations.select import get_column_attnum_from_name from db.metadata import get_empty_metadata -from mathesar.models.base import DataFile, Table +from mathesar.models.deprecated import DataFile, Table from mathesar.utils.display_options_inference import infer_mathesar_money_display_options create_display_options_test_list = [ diff --git a/mathesar/tests/imports/test_csv.py b/mathesar/tests/imports/test_csv.py index ea88f53607..bdf73a39d3 100644 --- a/mathesar/tests/imports/test_csv.py +++ b/mathesar/tests/imports/test_csv.py @@ -3,7 +3,7 @@ from django.core.files import File from sqlalchemy import text -from mathesar.models.base import DataFile, Schema +from mathesar.models.deprecated import DataFile, Schema from mathesar.errors import InvalidTableError from mathesar.imports.base import create_table_from_data_file from mathesar.imports.csv import get_sv_dialect, get_sv_reader diff --git a/mathesar/tests/imports/test_excel.py b/mathesar/tests/imports/test_excel.py index 6a87736a8a..2854c1ec6f 100644 --- a/mathesar/tests/imports/test_excel.py +++ b/mathesar/tests/imports/test_excel.py @@ -2,7 +2,7 @@ from django.core.files import File -from mathesar.models.base import DataFile, Schema +from mathesar.models.deprecated import DataFile, Schema from mathesar.imports.base import create_table_from_data_file from db.schemas.utils import get_schema_oid_from_name from psycopg.errors import DuplicateTable diff --git a/mathesar/tests/imports/test_json.py b/mathesar/tests/imports/test_json.py index a35b47dc74..689a69a47b 100644 --- a/mathesar/tests/imports/test_json.py +++ b/mathesar/tests/imports/test_json.py @@ -3,7 +3,7 @@ from django.core.files import File from sqlalchemy import text -from mathesar.models.base import DataFile, Schema +from mathesar.models.deprecated import DataFile, Schema from mathesar.imports.base import create_table_from_data_file from db.schemas.operations.create import create_schema_via_sql_alchemy from db.schemas.utils import get_schema_oid_from_name diff --git a/mathesar/tests/test_models.py b/mathesar/tests/test_models.py index d219c619a8..2c4da0dddb 100644 --- a/mathesar/tests/test_models.py +++ b/mathesar/tests/test_models.py @@ -2,7 +2,7 @@ from unittest.mock import patch from django.core.cache import cache -from mathesar.models.base import Connection, Schema, Table, schema_utils +from mathesar.models.deprecated import Connection, Schema, Table, schema_utils from mathesar.utils.models import attempt_dumb_query diff --git a/mathesar/tests/test_multi_db.py b/mathesar/tests/test_multi_db.py index 6c951a67a8..96a3bd5df1 100644 --- a/mathesar/tests/test_multi_db.py +++ b/mathesar/tests/test_multi_db.py @@ -1,7 +1,7 @@ import pytest from django.core.exceptions import ValidationError -from mathesar.models.base import Table, Schema, Connection +from mathesar.models.deprecated import Table, Schema, Connection @pytest.fixture(autouse=True) diff --git a/mathesar/utils/columns.py b/mathesar/utils/columns.py index b172d63203..97075a7d52 100644 --- a/mathesar/utils/columns.py +++ b/mathesar/utils/columns.py @@ -1,4 +1,4 @@ -from mathesar.models.base import Column +from mathesar.models.deprecated import Column # This should be replaced once we have the ColumnMetadata model sorted out. diff --git a/mathesar/utils/connections.py b/mathesar/utils/connections.py index 70a12ec541..95fd15bd79 100644 --- a/mathesar/utils/connections.py +++ b/mathesar/utils/connections.py @@ -1,7 +1,7 @@ """Utilities to help with creating and managing connections in Mathesar.""" from psycopg2.errors import DuplicateSchema from sqlalchemy.exc import OperationalError, ProgrammingError -from mathesar.models.base import Connection +from mathesar.models.deprecated import Connection from db import install, connection as dbconn from mathesar.state import reset_reflection from mathesar.examples.library_dataset import load_library_dataset diff --git a/mathesar/utils/datafiles.py b/mathesar/utils/datafiles.py index 22e16ef69e..e9ab31d6f7 100644 --- a/mathesar/utils/datafiles.py +++ b/mathesar/utils/datafiles.py @@ -13,7 +13,7 @@ from mathesar.errors import URLDownloadError from mathesar.imports.csv import is_valid_csv, get_sv_dialect, get_file_encoding from mathesar.imports.json import is_valid_json, validate_json_format -from mathesar.models.base import DataFile +from mathesar.models.deprecated import DataFile ALLOWED_FILE_FORMATS = ['csv', 'tsv', 'json', 'xls', 'xlsx', 'xlsm', 'xlsb', 'odf', 'ods', 'odt'] diff --git a/mathesar/utils/joins.py b/mathesar/utils/joins.py index 0d3aee1776..cc67105dba 100644 --- a/mathesar/utils/joins.py +++ b/mathesar/utils/joins.py @@ -1,6 +1,6 @@ from db.tables.operations import select as ma_sel from db.metadata import get_empty_metadata -from mathesar.models.base import Table, Column, Constraint +from mathesar.models.deprecated import Table, Column, Constraint TARGET = 'target' FK_PATH = 'fk_path' diff --git a/mathesar/utils/preview.py b/mathesar/utils/preview.py index 7108fb5295..a9f05cc98a 100644 --- a/mathesar/utils/preview.py +++ b/mathesar/utils/preview.py @@ -1,7 +1,7 @@ import re from db.constraints.utils import ConstraintType -from mathesar.models.base import Column, Constraint, Table +from mathesar.models.deprecated import Column, Constraint, Table def _preview_info_by_column_id(fk_constraints, previous_path=None, exising_columns=None): diff --git a/mathesar/utils/schemas.py b/mathesar/utils/schemas.py index 2488c1d752..833cfc33b9 100644 --- a/mathesar/utils/schemas.py +++ b/mathesar/utils/schemas.py @@ -4,7 +4,7 @@ from db.schemas.operations.create import create_schema_via_sql_alchemy from db.schemas.utils import get_schema_oid_from_name, get_mathesar_schemas from mathesar.database.base import create_mathesar_engine -from mathesar.models.base import Schema, Connection +from mathesar.models.deprecated import Schema, Connection def create_schema_and_object(name, connection_id, comment=None): diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index b4f8b4dca7..1772d653d7 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -4,7 +4,7 @@ from db.tables.operations.infer_types import infer_table_column_types from mathesar.database.base import create_mathesar_engine from mathesar.imports.base import create_table_from_data_file -from mathesar.models.base import Table +from mathesar.models.deprecated import Table from mathesar.state.django import reflect_columns_from_tables from mathesar.state import get_cached_metadata diff --git a/mathesar/views.py b/mathesar/views.py index 26807d39d3..3b05ccabf0 100644 --- a/mathesar/views.py +++ b/mathesar/views.py @@ -18,7 +18,7 @@ from mathesar.api.ui.serializers.users import UserSerializer from mathesar.api.utils import is_valid_uuid_v4 from mathesar.database.types import UIType -from mathesar.models.base import Connection, Schema, Table +from mathesar.models.deprecated import Connection, Schema, Table from mathesar.models.query import UIQuery from mathesar.models.shares import SharedTable, SharedQuery from mathesar.state import reset_reflection From 4d4359115437f0b1bc6f7dd8a4a70c7dc95a3361 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 14 Jun 2024 17:44:43 +0800 Subject: [PATCH 0256/1141] Create new models base with base model --- mathesar/models/base.py | 9 +++++++++ mathesar/models/deprecated.py | 9 +-------- mathesar/models/query.py | 3 ++- mathesar/models/shares.py | 2 +- mathesar/models/users.py | 3 ++- 5 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 mathesar/models/base.py diff --git a/mathesar/models/base.py b/mathesar/models/base.py new file mode 100644 index 0000000000..3e157298bd --- /dev/null +++ b/mathesar/models/base.py @@ -0,0 +1,9 @@ +from django.db import models + + +class BaseModel(models.Model): + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + abstract = True diff --git a/mathesar/models/deprecated.py b/mathesar/models/deprecated.py index 0e54d0fa2e..510bc92ca4 100644 --- a/mathesar/models/deprecated.py +++ b/mathesar/models/deprecated.py @@ -44,6 +44,7 @@ from db.records.operations.insert import insert_from_select from db.tables.utils import get_primary_key_column +from mathesar.models.base import BaseModel from mathesar.models.relation import Relation from mathesar.utils import models as model_utils from mathesar.utils.prefetch import PrefetchManager, Prefetcher @@ -57,14 +58,6 @@ NAME_CACHE_INTERVAL = 60 * 5 -class BaseModel(models.Model): - created_at = models.DateTimeField(auto_now_add=True) - updated_at = models.DateTimeField(auto_now=True) - - class Meta: - abstract = True - - class DatabaseObjectManager(PrefetchManager): def get_queryset(self): make_sure_initial_reflection_happened() diff --git a/mathesar/models/query.py b/mathesar/models/query.py index 06eed199e6..5497c78f99 100644 --- a/mathesar/models/query.py +++ b/mathesar/models/query.py @@ -28,8 +28,9 @@ ListOfDictValidator, TransformationsValidator, ) +from mathesar.models.base import BaseModel from mathesar.state.cached_property import cached_property -from mathesar.models.deprecated import BaseModel, Column +from mathesar.models.deprecated import Column from mathesar.models.relation import Relation from mathesar.state import get_cached_metadata diff --git a/mathesar/models/shares.py b/mathesar/models/shares.py index de78b406c3..20f076080a 100644 --- a/mathesar/models/shares.py +++ b/mathesar/models/shares.py @@ -1,7 +1,7 @@ import uuid from django.db import models -from mathesar.models.deprecated import BaseModel +from mathesar.models.base import BaseModel class SharedEntity(BaseModel): diff --git a/mathesar/models/users.py b/mathesar/models/users.py index 693348e558..301ea76314 100644 --- a/mathesar/models/users.py +++ b/mathesar/models/users.py @@ -2,7 +2,8 @@ from django.contrib.auth.models import AbstractUser from django.db import models -from mathesar.models.deprecated import BaseModel, Connection, Schema +from mathesar.models.base import BaseModel +from mathesar.models.deprecated import Connection, Schema class User(AbstractUser): From 0796b6840125b440583f4ea5e9817ea9f03d29e0 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 14 Jun 2024 17:51:42 +0800 Subject: [PATCH 0257/1141] add default auto field setting to get rid of warning --- config/settings/common_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index ce2ce24282..7697ed053c 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -247,6 +247,8 @@ def pipe_delim(pipe_string): MATHESAR_CAPTURE_UNHANDLED_EXCEPTION = decouple_config('CAPTURE_UNHANDLED_EXCEPTION', default=False) MATHESAR_STATIC_NON_CODE_FILES_LOCATION = os.path.join(BASE_DIR, 'mathesar/static/non-code/') +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + # UI source files have to be served by Django in order for static assets to be included during dev mode # https://vitejs.dev/guide/assets.html # https://vitejs.dev/guide/backend-integration.html From de8ee006fd37ddd126295193c6ee6c74cb1cecef Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 14 Jun 2024 19:06:23 +0800 Subject: [PATCH 0258/1141] add new permissions and database models --- ...007_rename_database_connection_and_more.py | 61 ++++++++++++++++++- mathesar/models/base.py | 42 +++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/mathesar/migrations/0007_rename_database_connection_and_more.py b/mathesar/migrations/0007_rename_database_connection_and_more.py index 34a6ad644a..930990b452 100644 --- a/mathesar/migrations/0007_rename_database_connection_and_more.py +++ b/mathesar/migrations/0007_rename_database_connection_and_more.py @@ -1,9 +1,11 @@ # Generated by Django 4.2.11 on 2024-06-13 09:05 +from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import mathesar.models.deprecated +import encrypted_fields.fields +import mathesar.models.deprecated class Migration(migrations.Migration): @@ -31,4 +33,61 @@ class Migration(migrations.Migration): name='column_order', field=models.JSONField(blank=True, default=None, null=True, validators=[mathesar.models.deprecated.validate_column_order]), ), + migrations.CreateModel( + name='Server', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('host', models.CharField(max_length=255)), + ('port', models.IntegerField()), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Database', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=128)), + ('server', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='databases', to='mathesar.server')), + ], + ), + migrations.AddConstraint( + model_name='database', + constraint=models.UniqueConstraint(fields=('name', 'server'), name='unique_database'), + ), + migrations.CreateModel( + name='Role', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=255)), + ('password', encrypted_fields.fields.EncryptedCharField(max_length=255)), + ('server', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='roles', to='mathesar.server')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='UserDatabaseRoleMap', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('database', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mathesar.database')), + ('role', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mathesar.role')), + ('server', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mathesar.server')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.AddConstraint( + model_name='userdatabaserolemap', + constraint=models.UniqueConstraint(fields=('user', 'database'), name='user_one_role_per_database'), + ), ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 3e157298bd..0a60f83c76 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -1,4 +1,5 @@ from django.db import models +from encrypted_fields.fields import EncryptedCharField class BaseModel(models.Model): @@ -7,3 +8,44 @@ class BaseModel(models.Model): class Meta: abstract = True + + +class Server(BaseModel): + host = models.CharField(max_length=255) + port = models.IntegerField() + + +class Database(BaseModel): + name = models.CharField(max_length=128) + server = models.ForeignKey( + 'Server', on_delete=models.CASCADE, related_name='databases' + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["name", "server"], name="unique_database" + ) + ] + + +class Role(BaseModel): + name = models.CharField(max_length=255) + server = models.ForeignKey( + 'Server', on_delete=models.CASCADE, related_name='roles' + ) + password = EncryptedCharField(max_length=255) + + +class UserDatabaseRoleMap(BaseModel): + user = models.ForeignKey('User', on_delete=models.CASCADE) + database = models.ForeignKey('Database', on_delete=models.CASCADE) + role = models.ForeignKey('Role', on_delete=models.CASCADE) + server = models.ForeignKey('Server', on_delete=models.CASCADE) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["user", "database"], name="user_one_role_per_database" + ) + ] From 0411744fd16f24315612d464bdccafe4b9c1cf0a Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 14 Jun 2024 19:03:03 +0530 Subject: [PATCH 0259/1141] finished wiring and created a sql util function --- db/sql/00_msar.sql | 24 +++++++++ db/tables/operations/import_.py | 92 +++++++++++++++++++++++++++++++++ db/tables/operations/select.py | 7 ++- 3 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 db/tables/operations/import_.py diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index a6ee233ad2..e82e49b0a7 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -219,6 +219,30 @@ END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION +msar.get_fully_qualified_relation_name(rel_id oid) RETURNS text AS $$/* +Return the fully-qualified name for a given relation (e.g., table). + +The relation *must* be in the pg_class table to use this function. This function will return NULL if +no corresponding relation can be found. + +Args: + rel_id: The OID of the relation. +*/ +DECLARE + sch_name text; + rel_name text; +BEGIN + SELECT nspname, relname INTO sch_name, rel_name + FROM pg_catalog.pg_class AS pgc + LEFT JOIN pg_catalog.pg_namespace AS pgn + ON pgc.relnamespace = pgn.oid + WHERE pgc.oid = rel_id; + RETURN msar.get_fully_qualified_object_name(sch_name, rel_name); +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.get_relation_name_or_null(rel_id oid) RETURNS text AS $$/* Return the name for a given relation (e.g., table), qualified or quoted as appropriate. diff --git a/db/tables/operations/import_.py b/db/tables/operations/import_.py new file mode 100644 index 0000000000..0b4a114e64 --- /dev/null +++ b/db/tables/operations/import_.py @@ -0,0 +1,92 @@ +import tempfile +import clevercsv as csv +from psycopg import sql +from db.tables.operations.create import prepare_table_for_import +from db.tables.operations.select import get_fully_qualified_relation_name +from db.encoding_utils import get_sql_compatible_encoding +from mathesar.models.base import DataFile +from mathesar.imports.csv import get_file_encoding, get_sv_reader, process_column_names + + +def import_csv(data_file_id, table_name, schema_oid, conn, comment=None): + data_file = DataFile.objects.get(id=data_file_id) + file_path = data_file.file.path + header = data_file.header + dialect = csv.dialect.SimpleDialect( + data_file.delimiter, + data_file.quotechar, + data_file.escapechar + ) + encoding = get_file_encoding(data_file.file) + with open(file_path, 'rb') as csv_file: + csv_reader = get_sv_reader(csv_file, header, dialect) + column_names = process_column_names(csv_reader.fieldnames) + table_oid = prepare_table_for_import( + table_name, + schema_oid, + column_names, + conn, + comment + ) + insert_csv_records( + table_oid, + conn, + file_path, + column_names, + header, + dialect.delimiter, + dialect.escapechar, + dialect.quotechar, + encoding + ) + return table_oid + + +def insert_csv_records( + table_oid, + conn, + file_path, + column_names, + header, + delimiter=None, + escape=None, + quote=None, + encoding=None +): + conversion_encoding, sql_encoding = get_sql_compatible_encoding(encoding) + schema_name, table_name = get_fully_qualified_relation_name(table_oid, conn).split('.') + formatted_columns = sql.SQL(",").join(sql.Identifier(column_name) for column_name in column_names) + copy_sql = sql.SQL( + "COPY {schema_name}.{table_name} ({formatted_columns}) FROM STDIN CSV {header} {delimiter} {escape} {quote} {encoding}" + ).format( + schema_name=sql.Identifier(schema_name), + table_name=sql.Identifier(table_name), + formatted_columns=formatted_columns, + header=sql.SQL("HEADER" if header else ""), + delimiter=sql.SQL(f"DELIMITER E'{delimiter}'" if delimiter else ""), + escape=sql.SQL(f"ESCAPE '{escape}'" if escape else ""), + quote=sql.SQL( + ("QUOTE ''''" if quote == "'" else f"QUOTE '{quote}'") + if quote + else "" + ), + encoding=sql.SQL(f"ENCODING '{sql_encoding}'" if sql_encoding else ""), + ) + cursor = conn.cursor() + with open(file_path, 'r', encoding=encoding) as csv_file: + if conversion_encoding == encoding: + with cursor.copy(copy_sql) as copy: + while data := csv_file.read(): + copy.write(data) + else: + # File needs to be converted to compatible database supported encoding + with tempfile.SpooledTemporaryFile(mode='wb+', encoding=conversion_encoding) as temp_file: + while True: + contents = csv_file.read().encode(conversion_encoding, "replace") + if not contents: + break + temp_file.write(contents) + temp_file.seek(0) + with cursor.copy(copy_sql) as copy: + while data := temp_file.read(): + copy.write(data) diff --git a/db/tables/operations/select.py b/db/tables/operations/select.py index 69149ea469..1f3cc96448 100644 --- a/db/tables/operations/select.py +++ b/db/tables/operations/select.py @@ -59,12 +59,11 @@ def get_table_info(schema, conn): return exec_msar_func(conn, 'get_table_info', schema).fetchone()[0] -def get_relation_name(table_oid, conn): +def get_fully_qualified_relation_name(table_oid, conn): """ - Return a fully qualified table name for a given table_oid - when table is not included in the search path. + Return a fully qualified table name. """ - return exec_msar_func(conn, 'get_relation_name_or_null', table_oid).fetchone()[0] + return exec_msar_func(conn, 'get_fully_qualified_relation_name', table_oid).fetchone()[0] def reflect_table(name, schema, engine, metadata, connection_to_use=None, keep_existing=False): From a9542968b06e5f46d829ec959499f38697a38ea5 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 14 Jun 2024 19:20:24 +0530 Subject: [PATCH 0260/1141] add tests and docstrings --- db/tables/operations/select.py | 3 +++ mathesar/rpc/tables.py | 13 +++++++++++++ mathesar/tests/rpc/test_endpoints.py | 5 +++++ mathesar/tests/rpc/test_tables.py | 27 +++++++++++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/db/tables/operations/select.py b/db/tables/operations/select.py index 1f3cc96448..9008b25030 100644 --- a/db/tables/operations/select.py +++ b/db/tables/operations/select.py @@ -62,6 +62,9 @@ def get_table_info(schema, conn): def get_fully_qualified_relation_name(table_oid, conn): """ Return a fully qualified table name. + + Args: + table_oid: The table oid for which we want fully qualified name. """ return exec_msar_func(conn, 'get_fully_qualified_relation_name', table_oid).fetchone()[0] diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 783e3262e4..af2ad0b54e 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -178,6 +178,19 @@ def patch( def import_( *, data_file_id: int, table_name: str, schema_oid: int, database_id: int, comment=None, **kwargs ) -> int: + """ + Import a CSV/TSV into a table. + + Args: + data_file_id: The Django id of the DataFile containing desired CSV/TSV. + table_name: Name of the table to be imported. + schema_oid: Identity of the schema in the user's database. + database_id: The Django id of the database containing the table. + comment: The comment for the new table. + + Returns: + The `oid` of the created table. + """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: return import_csv(data_file_id, table_name, schema_oid, conn, comment) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 3644b808b5..8541233ea4 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -73,6 +73,11 @@ tables.patch, "tables.patch", [user_is_authenticated] + ), + ( + tables.import_, + "tables.import", + [user_is_authenticated] ) ] diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index 76da0bbb1d..76471c2da9 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -192,3 +192,30 @@ def mock_table_patch(_table_oid, _table_data_dict, conn): request=request ) assert altered_table_name == 'newtabname' + + +def test_tables_import(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + schema_oid = 2200 + data_file_id = 10 + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_table_import(_data_file_id, table_name, _schema_oid, conn, comment): + if _schema_oid != schema_oid and _data_file_id != data_file_id: + raise AssertionError('incorrect parameters passed') + return 1964474 + monkeypatch.setattr(tables, 'connect', mock_connect) + monkeypatch.setattr(tables, 'import_csv', mock_table_import) + actual_table_oid = tables.import_(data_file_id=10, table_name='imported_table', schema_oid=2200, database_id=11, request=request) + assert actual_table_oid == 1964474 From 4df1557d6c59146510a15ad77643d33a6bc6fd20 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 14 Jun 2024 19:24:42 +0530 Subject: [PATCH 0261/1141] fix function signature --- mathesar/rpc/tables.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index e27740fa56..1516e12ad6 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -176,7 +176,13 @@ def patch( @http_basic_auth_login_required @handle_rpc_exceptions def import_( - *, data_file_id: int, table_name: str, schema_oid: int, database_id: int, comment=None, **kwargs + *, + data_file_id: int, + table_name: str, + schema_oid: int, + database_id: int, + comment: str = None, + **kwargs ) -> int: """ Import a CSV/TSV into a table. From a9839a03661b1ffaa03ceb6f8cc54a1e1b154df0 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 14 Jun 2024 19:29:19 +0530 Subject: [PATCH 0262/1141] restore csv.py --- mathesar/imports/csv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mathesar/imports/csv.py b/mathesar/imports/csv.py index 93478b8357..e1b45236c5 100644 --- a/mathesar/imports/csv.py +++ b/mathesar/imports/csv.py @@ -11,6 +11,7 @@ from mathesar.imports.utils import get_alternate_column_names, process_column_names from db.constants import COLUMN_NAME_TEMPLATE from psycopg2.errors import IntegrityError, DataError + from mathesar.state import reset_reflection # The user-facing documentation replicates these delimiter characters. If you From dc4e54cfcc24d051e59b0376930a172806624237 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 14 Jun 2024 19:31:43 +0530 Subject: [PATCH 0263/1141] imporve test formatting --- mathesar/tests/rpc/test_tables.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index 76471c2da9..d0761730e5 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -217,5 +217,11 @@ def mock_table_import(_data_file_id, table_name, _schema_oid, conn, comment): return 1964474 monkeypatch.setattr(tables, 'connect', mock_connect) monkeypatch.setattr(tables, 'import_csv', mock_table_import) - actual_table_oid = tables.import_(data_file_id=10, table_name='imported_table', schema_oid=2200, database_id=11, request=request) - assert actual_table_oid == 1964474 + imported_table_oid = tables.import_( + data_file_id=10, + table_name='imported_table', + schema_oid=2200, + database_id=11, + request=request + ) + assert imported_table_oid == 1964474 From dac974f1b8bdccd69eab2a6e1a3ffec952b180ad Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 17 Jun 2024 15:26:08 +0800 Subject: [PATCH 0264/1141] move UIQuery model to Exploration --- mathesar/admin.py | 4 ++-- mathesar/api/db/viewsets/queries.py | 12 +++++----- mathesar/api/dj_filters.py | 6 ++--- mathesar/api/exceptions/error_codes.py | 2 +- .../validation_exceptions/exceptions.py | 6 ++--- mathesar/api/pagination.py | 4 ++-- mathesar/api/serializers/queries.py | 24 +++++++++---------- mathesar/api/utils.py | 6 ++--- ...e.py => 0007_users_permissions_remodel.py} | 1 + mathesar/models/query.py | 4 ++-- mathesar/models/shares.py | 2 +- mathesar/rpc/exceptions/error_codes.py | 2 +- mathesar/tests/api/query/conftest.py | 4 ++-- mathesar/tests/api/query/test_query_api.py | 6 ++--- .../tests/api/query/test_query_records_api.py | 4 ++-- mathesar/tests/api/test_table_api.py | 4 ++-- mathesar/tests/query/test_base.py | 4 ++-- mathesar/views.py | 4 ++-- 18 files changed, 50 insertions(+), 49 deletions(-) rename mathesar/migrations/{0007_rename_database_connection_and_more.py => 0007_users_permissions_remodel.py} (98%) diff --git a/mathesar/admin.py b/mathesar/admin.py index 6eca4ac3d5..6afc0e326d 100644 --- a/mathesar/admin.py +++ b/mathesar/admin.py @@ -3,7 +3,7 @@ from mathesar.models.deprecated import Table, Schema, DataFile from mathesar.models.users import User -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration from mathesar.models.shares import SharedTable, SharedQuery @@ -24,6 +24,6 @@ class MathesarUserAdmin(UserAdmin): admin.site.register(Schema) admin.site.register(DataFile) admin.site.register(User, MathesarUserAdmin) -admin.site.register(UIQuery) +admin.site.register(Exploration) admin.site.register(SharedTable) admin.site.register(SharedQuery) diff --git a/mathesar/api/db/viewsets/queries.py b/mathesar/api/db/viewsets/queries.py index b15b23c0df..c9549fd238 100644 --- a/mathesar/api/db/viewsets/queries.py +++ b/mathesar/api/db/viewsets/queries.py @@ -9,13 +9,13 @@ from rest_framework.decorators import action from mathesar.api.db.permissions.query import QueryAccessPolicy -from mathesar.api.dj_filters import UIQueryFilter +from mathesar.api.dj_filters import ExplorationFilter from mathesar.api.exceptions.query_exceptions.exceptions import DeletedColumnAccess, DeletedColumnAccessAPIException from mathesar.api.pagination import DefaultLimitOffsetPagination, TableLimitOffsetPagination from mathesar.api.serializers.queries import BaseQuerySerializer, QuerySerializer from mathesar.api.serializers.records import RecordListParameterSerializer -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration class QueryViewSet( @@ -30,7 +30,7 @@ class QueryViewSet( serializer_class = QuerySerializer pagination_class = DefaultLimitOffsetPagination filter_backends = (filters.DjangoFilterBackend,) - filterset_class = UIQueryFilter + filterset_class = ExplorationFilter permission_classes = [IsAuthenticatedOrReadOnly] access_policy = QueryAccessPolicy @@ -55,10 +55,10 @@ def _get_scoped_queryset(self): if should_queryset_be_scoped: queryset = self.access_policy.scope_queryset( self.request, - UIQuery.objects.all() + Exploration.objects.all() ) else: - queryset = UIQuery.objects.all() + queryset = Exploration.objects.all() return queryset @action(methods=['get'], detail=True) @@ -119,7 +119,7 @@ def run(self, request): paginator = TableLimitOffsetPagination() input_serializer = BaseQuerySerializer(data=request.data, context={'request': request}) input_serializer.is_valid(raise_exception=True) - query = UIQuery(**input_serializer.validated_data) + query = Exploration(**input_serializer.validated_data) try: query.replace_transformations_with_processed_transformations() query.add_defaults_to_display_names() diff --git a/mathesar/api/dj_filters.py b/mathesar/api/dj_filters.py index bb850f8ebb..70d2cddb4b 100644 --- a/mathesar/api/dj_filters.py +++ b/mathesar/api/dj_filters.py @@ -2,7 +2,7 @@ from django_property_filter import PropertyFilterSet, PropertyBaseInFilter, PropertyCharFilter, PropertyOrderingFilter from mathesar.models.deprecated import Schema, Table, Connection, DataFile -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration class CharInFilter(PropertyBaseInFilter, PropertyCharFilter): @@ -77,7 +77,7 @@ class Meta: fields = ['name', 'schema', 'created_at', 'updated_at', 'import_verified'] -class UIQueryFilter(PropertyFilterSet): +class ExplorationFilter(PropertyFilterSet): database = CharInFilter(field_name='base_table__schema__database__name', lookup_expr='in') name = CharInFilter(field_name='name', lookup_expr='in') @@ -90,5 +90,5 @@ class UIQueryFilter(PropertyFilterSet): ) class Meta: - model = UIQuery + model = Exploration fields = ['name'] diff --git a/mathesar/api/exceptions/error_codes.py b/mathesar/api/exceptions/error_codes.py index c5fd4749ce..d4abf2b1ea 100644 --- a/mathesar/api/exceptions/error_codes.py +++ b/mathesar/api/exceptions/error_codes.py @@ -64,7 +64,7 @@ class ErrorCodes(Enum): DeletedColumnAccess = 4418 IncorrectOldPassword = 4419 EditingPublicSchema = 4421 - DuplicateUIQueryInSchema = 4422 + DuplicateExplorationInSchema = 4422 IdentifierTooLong = 4423 DynamicDefaultAlterationToStaticDefault = 4424 InvalidJSONFormat = 4425 diff --git a/mathesar/api/exceptions/validation_exceptions/exceptions.py b/mathesar/api/exceptions/validation_exceptions/exceptions.py index b99e1e1f3b..9187c717aa 100644 --- a/mathesar/api/exceptions/validation_exceptions/exceptions.py +++ b/mathesar/api/exceptions/validation_exceptions/exceptions.py @@ -2,12 +2,12 @@ from mathesar.api.exceptions.validation_exceptions.base_exceptions import MathesarValidationException -class DuplicateUIQueryInSchemaAPIException(MathesarValidationException): - error_code = ErrorCodes.DuplicateUIQueryInSchema.value +class DuplicateExplorationInSchemaAPIException(MathesarValidationException): + error_code = ErrorCodes.DuplicateExplorationInSchema.value def __init__( self, - message="UIQuery names must be unique per schema", + message="Exploration names must be unique per schema", field=None, details=None, ): diff --git a/mathesar/api/pagination.py b/mathesar/api/pagination.py index ce0f137b57..fad6397ea2 100644 --- a/mathesar/api/pagination.py +++ b/mathesar/api/pagination.py @@ -6,7 +6,7 @@ from db.records.operations.group import GroupBy from mathesar.api.utils import get_table_or_404, process_annotated_records from mathesar.models.deprecated import Column, Table -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration from mathesar.utils.preview import get_preview_info @@ -89,7 +89,7 @@ def paginate_queryset( table_columns = [{'id': column.id, 'alias': column.name} for column in columns_query] columns_to_fetch = table_columns + preview_columns - query = UIQuery(name="preview", base_table=table, initial_columns=columns_to_fetch) + query = Exploration(name="preview", base_table=table, initial_columns=columns_to_fetch) else: query = table records = query.get_records( diff --git a/mathesar/api/serializers/queries.py b/mathesar/api/serializers/queries.py index 2e4187d6b0..e49be3cea7 100644 --- a/mathesar/api/serializers/queries.py +++ b/mathesar/api/serializers/queries.py @@ -7,9 +7,9 @@ from mathesar.api.db.permissions.query_table import QueryTableAccessPolicy from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin -from mathesar.api.exceptions.validation_exceptions.exceptions import DuplicateUIQueryInSchemaAPIException +from mathesar.api.exceptions.validation_exceptions.exceptions import DuplicateExplorationInSchemaAPIException from mathesar.models.deprecated import Table -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration class BaseQuerySerializer(MathesarErrorMessageMixin, serializers.ModelSerializer): @@ -20,7 +20,7 @@ class BaseQuerySerializer(MathesarErrorMessageMixin, serializers.ModelSerializer ) class Meta: - model = UIQuery + model = Exploration fields = ['schema', 'initial_columns', 'transformations', 'base_table', 'display_names'] def get_schema(self, uiquery): @@ -48,9 +48,9 @@ def _validate_uniqueness(self, attrs): if base_table: schema = base_table.schema is_duplicate_q = self._get_is_duplicate_q(name, schema) - duplicates = UIQuery.objects.filter(is_duplicate_q) + duplicates = Exploration.objects.filter(is_duplicate_q) if duplicates.exists(): - raise DuplicateUIQueryInSchemaAPIException(field='name') + raise DuplicateExplorationInSchemaAPIException(field='name') def _get_is_duplicate_q(self, name, schema): has_same_name_q = Q(name=name) @@ -71,28 +71,28 @@ class QuerySerializer(BaseQuerySerializer): columns_url = serializers.SerializerMethodField('get_columns_url') class Meta: - model = UIQuery + model = Exploration fields = '__all__' def get_records_url(self, obj): - if isinstance(obj, UIQuery) and obj.pk is not None: - # Only get records_url if we are serializing an existing persisted UIQuery + if isinstance(obj, Exploration) and obj.pk is not None: + # Only get records_url if we are serializing an existing persisted Exploration request = self.context['request'] return request.build_absolute_uri(reverse('query-records', kwargs={'pk': obj.pk})) else: return None def get_columns_url(self, obj): - if isinstance(obj, UIQuery) and obj.pk is not None: - # Only get columns_url if we are serializing an existing persisted UIQuery + if isinstance(obj, Exploration) and obj.pk is not None: + # Only get columns_url if we are serializing an existing persisted Exploration request = self.context['request'] return request.build_absolute_uri(reverse('query-columns', kwargs={'pk': obj.pk})) else: return None def get_results_url(self, obj): - if isinstance(obj, UIQuery) and obj.pk is not None: - # Only get records_url if we are serializing an existing persisted UIQuery + if isinstance(obj, Exploration) and obj.pk is not None: + # Only get records_url if we are serializing an existing persisted Exploration request = self.context['request'] return request.build_absolute_uri(reverse('query-results', kwargs={'pk': obj.pk})) else: diff --git a/mathesar/api/utils.py b/mathesar/api/utils.py index d7a24d8927..be83a84173 100644 --- a/mathesar/api/utils.py +++ b/mathesar/api/utils.py @@ -7,7 +7,7 @@ from db.records.operations import group from mathesar.api.exceptions.error_codes import ErrorCodes from mathesar.models.deprecated import Table -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration from mathesar.utils.preview import column_alias_from_preview_template from mathesar.api.exceptions.generic_exceptions.base_exceptions import BadDBCredentials import psycopg @@ -40,8 +40,8 @@ def get_table_or_404(pk): def get_query_or_404(pk): try: - query = UIQuery.objects.get(id=pk) - except UIQuery.DoesNotExist: + query = Exploration.objects.get(id=pk) + except Exploration.DoesNotExist: raise generic_api_exceptions.NotFoundAPIException( NotFound, error_code=ErrorCodes.QueryNotFound.value, diff --git a/mathesar/migrations/0007_rename_database_connection_and_more.py b/mathesar/migrations/0007_users_permissions_remodel.py similarity index 98% rename from mathesar/migrations/0007_rename_database_connection_and_more.py rename to mathesar/migrations/0007_users_permissions_remodel.py index 930990b452..9f3352beb7 100644 --- a/mathesar/migrations/0007_rename_database_connection_and_more.py +++ b/mathesar/migrations/0007_users_permissions_remodel.py @@ -90,4 +90,5 @@ class Migration(migrations.Migration): model_name='userdatabaserolemap', constraint=models.UniqueConstraint(fields=('user', 'database'), name='user_one_role_per_database'), ), + migrations.RenameModel(old_name='UIQuery', new_name='Exploration'), ] diff --git a/mathesar/models/query.py b/mathesar/models/query.py index 5497c78f99..a82450f4d4 100644 --- a/mathesar/models/query.py +++ b/mathesar/models/query.py @@ -35,7 +35,7 @@ from mathesar.state import get_cached_metadata -class UIQuery(BaseModel, Relation): +class Exploration(BaseModel, Relation): name = models.CharField( max_length=128, ) @@ -176,7 +176,7 @@ def replace_transformations_with_processed_transformations(self): Whereas before the transformations attribute was a one-way flow from the client, now it's something that the backend may redefine. This a significant complication of the - data flow. For example, if you replace transformations on a saved UIQuery and save it + data flow. For example, if you replace transformations on a saved Exploration and save it again, we must trigger a reflection, which can have a performance impact. Also, frontend must expect that certain transformations might alter the transformation pipeline, which would then need reflecting by frontend; that might be a breaking change. diff --git a/mathesar/models/shares.py b/mathesar/models/shares.py index 20f076080a..15f9cc3cda 100644 --- a/mathesar/models/shares.py +++ b/mathesar/models/shares.py @@ -24,5 +24,5 @@ class SharedTable(SharedEntity): class SharedQuery(SharedEntity): query = models.ForeignKey( - 'UIQuery', on_delete=models.CASCADE, related_name='shared_query' + 'Exploration', on_delete=models.CASCADE, related_name='shared_query' ) diff --git a/mathesar/rpc/exceptions/error_codes.py b/mathesar/rpc/exceptions/error_codes.py index 52b4925196..af25368002 100644 --- a/mathesar/rpc/exceptions/error_codes.py +++ b/mathesar/rpc/exceptions/error_codes.py @@ -516,7 +516,7 @@ def get_error_code(err: Exception) -> int: "DictHasBadKeys": -28007, "DistinctColumnRequiredAPIException": -28008, "DoesNotExist": -28009, - "DuplicateUIQueryInSchemaAPIException": -28010, + "DuplicateExplorationInSchemaAPIException": -28010, "EditingPublicSchemaIsDisallowed": -28011, "GenericAPIException": -28012, "IncompatibleFractionDigitValuesAPIException": -28013, diff --git a/mathesar/tests/api/query/conftest.py b/mathesar/tests/api/query/conftest.py index 711119f6db..540680ca1e 100644 --- a/mathesar/tests/api/query/conftest.py +++ b/mathesar/tests/api/query/conftest.py @@ -1,5 +1,5 @@ import pytest -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration @pytest.fixture @@ -37,7 +37,7 @@ def _create(schema_name=schema_name): 'col1': dict(a=1), 'col2': dict(b=2), } - ui_query = UIQuery.objects.create( + ui_query = Exploration.objects.create( base_table=base_table, initial_columns=initial_columns, display_options=display_options, diff --git a/mathesar/tests/api/query/test_query_api.py b/mathesar/tests/api/query/test_query_api.py index e0c39e6fff..2c2d57acfa 100644 --- a/mathesar/tests/api/query/test_query_api.py +++ b/mathesar/tests/api/query/test_query_api.py @@ -1,6 +1,6 @@ import pytest -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration @pytest.fixture @@ -330,7 +330,7 @@ def test_update_based_on_permission( ): base_table = create_patents_table(table_name=get_uid()) different_schema_base_table = create_patents_table(table_name=get_uid(), schema_name='Private Schema') - ui_query = UIQuery.objects.create( + ui_query = Exploration.objects.create( name="Query1", base_table=base_table, initial_columns=[ @@ -342,7 +342,7 @@ def test_update_based_on_permission( } ] ) - different_schema_ui_query = UIQuery.objects.create( + different_schema_ui_query = Exploration.objects.create( name="Query2", base_table=different_schema_base_table, initial_columns=[ diff --git a/mathesar/tests/api/query/test_query_records_api.py b/mathesar/tests/api/query/test_query_records_api.py index f35d6c142c..67a0681efc 100644 --- a/mathesar/tests/api/query/test_query_records_api.py +++ b/mathesar/tests/api/query/test_query_records_api.py @@ -1,7 +1,7 @@ import pytest import json -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration @pytest.fixture @@ -30,7 +30,7 @@ def joining_patents_query(academics_ma_tables): 'name': dict(a=1), 'institution_name': dict(b=2), } - ui_query = UIQuery.objects.create( + ui_query = Exploration.objects.create( base_table=academics_table, initial_columns=initial_columns, display_options=display_options, diff --git a/mathesar/tests/api/test_table_api.py b/mathesar/tests/api/test_table_api.py index b5ce0b906d..55e7fdbc73 100644 --- a/mathesar/tests/api/test_table_api.py +++ b/mathesar/tests/api/test_table_api.py @@ -10,7 +10,7 @@ from db.types.base import PostgresType, MathesarCustomType from db.metadata import get_empty_metadata from mathesar.models.users import DatabaseRole, SchemaRole -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration from mathesar.state import reset_reflection from mathesar.api.exceptions.error_codes import ErrorCodes @@ -1876,7 +1876,7 @@ def test_table_ui_dependency(client, create_patents_table, get_uid): }, ], } - query = UIQuery.objects.create(**query_data) + query = Exploration.objects.create(**query_data) response = client.get(f'/api/db/v0/tables/{base_table.id}/ui_dependents/') response_data = response.json() expected_response = { diff --git a/mathesar/tests/query/test_base.py b/mathesar/tests/query/test_base.py index 16eb9c637a..c8611843ba 100644 --- a/mathesar/tests/query/test_base.py +++ b/mathesar/tests/query/test_base.py @@ -1,4 +1,4 @@ -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration from db.queries.base import DBQuery, InitialColumn from db.transforms import base as transforms_base @@ -49,7 +49,7 @@ def test_convert_to_db_query(create_patents_table, get_uid): transforms_base.Offset(15), ] name = "some query" - ui_query = UIQuery( + ui_query = Exploration( name=name, base_table=base_table_dj, initial_columns=initial_columns_json, diff --git a/mathesar/views.py b/mathesar/views.py index 3b05ccabf0..e88b596881 100644 --- a/mathesar/views.py +++ b/mathesar/views.py @@ -19,7 +19,7 @@ from mathesar.api.utils import is_valid_uuid_v4 from mathesar.database.types import UIType from mathesar.models.deprecated import Connection, Schema, Table -from mathesar.models.query import UIQuery +from mathesar.models.query import Exploration from mathesar.models.shares import SharedTable, SharedQuery from mathesar.state import reset_reflection from mathesar import __version__ @@ -96,7 +96,7 @@ def get_table_list(request, schema): def get_queries_list(request, schema): if schema is None: return [] - qs = UIQuery.objects.filter(base_table__schema=schema) + qs = Exploration.objects.filter(base_table__schema=schema) permission_restricted_qs = QueryAccessPolicy.scope_queryset(request, qs) query_serializer = QuerySerializer( From e6571e61973261828efc58715b690bcec669a280 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 17 Jun 2024 16:42:05 +0800 Subject: [PATCH 0265/1141] add multicolumn integrity enforcers --- .../0007_users_permissions_remodel.py | 39 ++++++++++++++++--- mathesar/models/base.py | 10 +++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/mathesar/migrations/0007_users_permissions_remodel.py b/mathesar/migrations/0007_users_permissions_remodel.py index 9f3352beb7..8ca607bd8f 100644 --- a/mathesar/migrations/0007_users_permissions_remodel.py +++ b/mathesar/migrations/0007_users_permissions_remodel.py @@ -14,10 +14,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RenameModel( - old_name='Database', - new_name='Connection', - ), + migrations.RenameModel(old_name='Database', new_name='Connection'), + migrations.RenameModel(old_name='UIQuery', new_name='Exploration'), migrations.AlterField( model_name='databaserole', name='database', @@ -60,6 +58,10 @@ class Migration(migrations.Migration): model_name='database', constraint=models.UniqueConstraint(fields=('name', 'server'), name='unique_database'), ), + migrations.AddConstraint( + model_name='database', + constraint=models.UniqueConstraint(fields=('id', 'server'), name='database_id_server_index'), + ), migrations.CreateModel( name='Role', fields=[ @@ -74,6 +76,10 @@ class Migration(migrations.Migration): 'abstract': False, }, ), + migrations.AddConstraint( + model_name='role', + constraint=models.UniqueConstraint(fields=('id', 'server'), name='role_id_server_index'), + ), migrations.CreateModel( name='UserDatabaseRoleMap', fields=[ @@ -90,5 +96,28 @@ class Migration(migrations.Migration): model_name='userdatabaserolemap', constraint=models.UniqueConstraint(fields=('user', 'database'), name='user_one_role_per_database'), ), - migrations.RenameModel(old_name='UIQuery', new_name='Exploration'), + migrations.RunSQL( + sql=""" + ALTER TABLE mathesar_userdatabaserolemap + ADD CONSTRAINT userdatabaserolemap_database_server_integrity + FOREIGN KEY (database_id, server_id) + REFERENCES mathesar_database(id, server_id); + """, + reverse_sql=""" + ALTER TABLE mathesar_userdatabaserolemap + DROP CONSTRAINT userdatabaserolemap_database_server_integrity; + """ + ), + migrations.RunSQL( + sql=""" + ALTER TABLE mathesar_userdatabaserolemap + ADD CONSTRAINT userdatabaserolemap_role_server_integrity + FOREIGN KEY (role_id, server_id) + REFERENCES mathesar_role(id, server_id); + """, + reverse_sql=""" + ALTER TABLE mathesar_userdatabaserolemap + DROP CONSTRAINT userdatabaserolemap_role_server_integrity; + """ + ), ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 0a60f83c76..1086f3c87a 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -25,6 +25,9 @@ class Meta: constraints = [ models.UniqueConstraint( fields=["name", "server"], name="unique_database" + ), + models.UniqueConstraint( + fields=["id", "server"], name="database_id_server_index" ) ] @@ -36,6 +39,13 @@ class Role(BaseModel): ) password = EncryptedCharField(max_length=255) + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["id", "server"], name="role_id_server_index" + ) + ] + class UserDatabaseRoleMap(BaseModel): user = models.ForeignKey('User', on_delete=models.CASCADE) From 4afc251f5b5d21d0e157d2ff12fdb76a45ef1215 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 17 Jun 2024 17:18:38 +0800 Subject: [PATCH 0266/1141] fix linter error --- mathesar/migrations/0007_users_permissions_remodel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mathesar/migrations/0007_users_permissions_remodel.py b/mathesar/migrations/0007_users_permissions_remodel.py index 8ca607bd8f..f9d69b6212 100644 --- a/mathesar/migrations/0007_users_permissions_remodel.py +++ b/mathesar/migrations/0007_users_permissions_remodel.py @@ -7,6 +7,7 @@ import mathesar.models.deprecated + class Migration(migrations.Migration): dependencies = [ @@ -15,7 +16,6 @@ class Migration(migrations.Migration): operations = [ migrations.RenameModel(old_name='Database', new_name='Connection'), - migrations.RenameModel(old_name='UIQuery', new_name='Exploration'), migrations.AlterField( model_name='databaserole', name='database', @@ -31,6 +31,7 @@ class Migration(migrations.Migration): name='column_order', field=models.JSONField(blank=True, default=None, null=True, validators=[mathesar.models.deprecated.validate_column_order]), ), + migrations.RenameModel(old_name='UIQuery', new_name='Exploration'), migrations.CreateModel( name='Server', fields=[ From 59693911fef66575aa96fce16a40213e50fec28a Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 17 Jun 2024 18:11:50 +0800 Subject: [PATCH 0267/1141] update pandas to sort out build error --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5ae8c5bd9e..cf6bc29440 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,6 @@ SQLAlchemy-Utils==0.38.2 thefuzz==0.19.0 whitenoise==6.4.0 gunicorn==20.1.0 -pandas==2.0.2 +pandas==2.2.2 openpyxl==3.1.2 pyxlsb==1.0.10 From 3f1f84f1bbb26d2266ee27d09d2c1a79de63cf3e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 17 Jun 2024 12:09:29 -0400 Subject: [PATCH 0268/1141] Add schemas.patch RPC function --- db/schemas/operations/alter.py | 47 +++----- db/sql/00_msar.sql | 125 +++++++++------------- db/sql/test_00_msar.sql | 61 +++++------ db/tests/schemas/operations/test_alter.py | 28 ----- docs/docs/api/rpc.md | 2 + mathesar/rpc/schemas.py | 34 +++++- mathesar/tests/rpc/test_endpoints.py | 5 + mathesar/utils/models.py | 6 +- 8 files changed, 135 insertions(+), 173 deletions(-) delete mode 100644 db/tests/schemas/operations/test_alter.py diff --git a/db/schemas/operations/alter.py b/db/schemas/operations/alter.py index 919997eff0..994cdb1fae 100644 --- a/db/schemas/operations/alter.py +++ b/db/schemas/operations/alter.py @@ -1,46 +1,31 @@ -from db.connection import execute_msar_func_with_engine +import json -SUPPORTED_SCHEMA_ALTER_ARGS = {'name', 'description'} +from db.connection import execute_msar_func_with_engine, exec_msar_func -def rename_schema(schema_name, engine, rename_to): +def patch_schema_via_sql_alchemy(schema_name, engine, patch): """ - Rename an existing schema. + Patch a schema using a SQLAlchemy engine. Args: schema_name: Name of the schema to change. engine: SQLAlchemy engine object for connecting. - rename_to: New schema name. - - Returns: - Returns a string giving the command that was run. + patch: A dict mapping the following fields to new values: + - 'name' (optional): New name for the schema. + - 'description' (optional): New description for the schema. """ - if rename_to == schema_name: - return - return execute_msar_func_with_engine( - engine, 'rename_schema', schema_name, rename_to - ).fetchone()[0] + execute_msar_func_with_engine(engine, "patch_schema", schema_name, json.dumps(patch)) -def comment_on_schema(schema_name, engine, comment): +def patch_schema(schema_oid, conn, patch): """ - Change description of a schema. + Patch a schema using a psycopg connection. Args: - schema_name: The name of the schema whose comment we will change. - comment: The new comment. - engine: SQLAlchemy engine object for connecting. - - Returns: - Returns a string giving the command that was run. + schema_oid: The OID of the schema to change. + conn: a psycopg connection + patch: A dict mapping the following fields to new values: + - 'name' (optional): New name for the schema. + - 'description' (optional): New description for the schema. """ - return execute_msar_func_with_engine( - engine, 'comment_on_schema', schema_name, comment - ).fetchone()[0] - - -def alter_schema(name, engine, update_data): - if "description" in update_data: - comment_on_schema(name, engine, update_data["description"]) - if "name" in update_data: - rename_schema(name, engine, update_data["name"]) + exec_msar_func(conn, "patch_schema", schema_oid, json.dumps(patch)) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 52e69e92e9..f242b84797 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -162,18 +162,26 @@ SELECT oid FROM pg_namespace WHERE nspname=sch_name; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION __msar.get_schema_name(sch_id oid) RETURNS TEXT AS $$/* -Return the QUOTED name for a given schema. +CREATE OR REPLACE FUNCTION msar.get_schema_name(sch_id oid) RETURNS TEXT AS $$/* +Return the UNQUOTED name for a given schema. -The schema *must* be in the pg_namespace table to use this function. +Raises an exception if the schema is not found. Args: sch_id: The OID of the schema. */ +DECLARE sch_name text; BEGIN - RETURN sch_id::regnamespace::text; + SELECT nspname INTO sch_name FROM pg_namespace WHERE oid=sch_id; + + IF sch_name IS NULL THEN + RAISE EXCEPTION 'No schema with OID % exists.', sch_id + USING ERRCODE = '3F000'; -- invalid_schema_name + END IF; + + RETURN sch_name; END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +$$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION @@ -834,95 +842,71 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- - --- Rename schema ----------------------------------------------------------------------------------- - -CREATE OR REPLACE FUNCTION -__msar.rename_schema(old_sch_name text, new_sch_name text) RETURNS TEXT AS $$/* -Change a schema's name, returning the command executed. +DROP FUNCTION IF EXISTS msar.rename_schema(oid, text); +CREATE OR REPLACE FUNCTION msar.rename_schema(sch_id oid, new_sch_name text) RETURNS void AS $$/* +Change a schema's name Args: - old_sch_name: A properly quoted original schema name - new_sch_name: A properly quoted new schema name + sch_id: The OID of the schema to rename + new_sch_name: A new for the schema, UNQUOTED */ DECLARE - cmd_template text; -BEGIN - cmd_template := 'ALTER SCHEMA %s RENAME TO %s'; - RETURN __msar.exec_ddl(cmd_template, old_sch_name, new_sch_name); -END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; - - -CREATE OR REPLACE FUNCTION -msar.rename_schema(old_sch_name text, new_sch_name text) RETURNS TEXT AS $$/* -Change a schema's name, returning the command executed. - -Args: - old_sch_name: An unquoted original schema name - new_sch_name: An unquoted new schema name -*/ -BEGIN - RETURN __msar.rename_schema(quote_ident(old_sch_name), quote_ident(new_sch_name)); -END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; - - -CREATE OR REPLACE FUNCTION msar.rename_schema(sch_id oid, new_sch_name text) RETURNS TEXT AS $$/* -Change a schema's name, returning the command executed. - -Args: - sch_id: The OID of the original schema - new_sch_name: An unquoted new schema name -*/ + old_sch_name text := msar.get_schema_name(sch_id); BEGIN - RETURN __msar.rename_schema(__msar.get_schema_name(sch_id), quote_ident(new_sch_name)); + IF old_sch_name = new_sch_name THEN + -- Return early if the names are the same. This avoids an error from Postgres. + RETURN; + END IF; + EXECUTE format('ALTER SCHEMA %I RENAME TO %I', old_sch_name, new_sch_name); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; --- Comment on schema ------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION msar.set_schema_description( + sch_id oid, + description text +) RETURNS void AS $$/* +Set the PostgreSQL description (aka COMMENT) of a schema. -CREATE OR REPLACE FUNCTION -__msar.comment_on_schema(sch_name text, comment_ text) RETURNS TEXT AS $$/* -Change the description of a schema, returning command executed. +Descriptions are removed by passing an empty string. Passing a NULL description will cause +this function to return NULL withou doing anything. Args: - sch_name: The QUOTED name of the schema whose comment we will change. - comment_: The new comment, QUOTED + sch_id: The OID of the schema. + description: The new description, UNQUOTED */ -DECLARE - cmd_template text; BEGIN - cmd_template := 'COMMENT ON SCHEMA %s IS %s'; - RETURN __msar.exec_ddl(cmd_template, sch_name, comment_); + EXECUTE format('COMMENT ON SCHEMA %I IS %L', msar.get_schema_name(sch_id), description); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION -msar.comment_on_schema(sch_name text, comment_ text) RETURNS TEXT AS $$/* -Change the description of a schema, returning command executed. +CREATE OR REPLACE FUNCTION msar.patch_schema(sch_id oid, patch jsonb) RETURNS void AS $$/* +Modify a schema according to the given patch. Args: - sch_name: The UNQUOTED name of the schema whose comment we will change. - comment_: The new comment, UNQUOTED + sch_id: The OID of the schema. + patch: A JSONB object with the following keys: + - name: (optional) The new name of the schema + - description: (optional) The new description of the schema. To remove a description, pass an + empty string. Passing a NULL description will have no effect on the description. */ BEGIN - RETURN __msar.comment_on_schema(quote_ident(sch_name), quote_literal(comment_)); + PERFORM msar.rename_schema(sch_id, patch->>'name'); + PERFORM msar.set_schema_description(sch_id, patch->>'description'); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION msar.comment_on_schema(sch_id oid, comment_ text) RETURNS TEXT AS $$/* -Change the description of a schema, returning command executed. +CREATE OR REPLACE FUNCTION msar.patch_schema(sch_name text, patch jsonb) RETURNS void AS $$/* +Modify a schema according to the given patch. Args: - sch_id: The OID of the schema. - comment_: The new comment, UNQUOTED + sch_id: The name of the schema. + patch: A JSONB object as specified by msar.patch_schema(sch_id oid, patch jsonb) */ BEGIN - RETURN __msar.comment_on_schema(__msar.get_schema_name(sch_id), quote_literal(comment_)); + PERFORM msar.patch_schema(msar.get_schema_oid(sch_name), patch); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -979,7 +963,7 @@ DECLARE schema_oid oid; BEGIN EXECUTE 'CREATE SCHEMA ' || quote_ident(sch_name); schema_oid := msar.get_schema_oid(sch_name); - PERFORM msar.comment_on_schema(schema_oid, description); + PERFORM msar.set_schema_description(schema_oid, description); RETURN schema_oid; END; $$ LANGUAGE plpgsql; @@ -1022,15 +1006,8 @@ Args: sch_id: The OID of the schema to drop cascade_: When true, dependent objects will be dropped automatically */ -DECLARE - sch_name text; BEGIN - SELECT nspname INTO sch_name FROM pg_namespace WHERE oid = sch_id; - IF sch_name IS NULL THEN - RAISE EXCEPTION 'No schema with OID % exists.', sch_id - USING ERRCODE = '3F000'; -- invalid_schema_name - END IF; - PERFORM msar.drop_schema(sch_name, cascade_); + PERFORM msar.drop_schema(msar.get_schema_name(sch_id), cascade_); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -2339,7 +2316,7 @@ DECLARE column_defs __msar.col_def[]; constraint_defs __msar.con_def[]; BEGIN - fq_table_name := format('%s.%s', __msar.get_schema_name(sch_oid), quote_ident(tab_name)); + fq_table_name := format('%I.%I', msar.get_schema_name(sch_oid), tab_name); column_defs := msar.process_col_def_jsonb(0, col_defs, false, true); constraint_defs := msar.process_con_def_jsonb(0, con_defs); PERFORM __msar.add_table(fq_table_name, column_defs, constraint_defs); diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index db10bb73e5..9bce00d3cf 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -1310,47 +1310,40 @@ END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION __setup_alter_schema() RETURNS SETOF TEXT AS $$ +CREATE OR REPLACE FUNCTION test_patch_schema() RETURNS SETOF TEXT AS $$ +DECLARE sch_oid oid; BEGIN - CREATE SCHEMA alter_me; -END; -$$ LANGUAGE plpgsql; - + CREATE SCHEMA foo; + SELECT msar.get_schema_oid('foo') INTO sch_oid; -CREATE OR REPLACE FUNCTION test_rename_schema() RETURNS SETOF TEXT AS $$ -BEGIN - PERFORM __setup_alter_schema(); - PERFORM msar.rename_schema( - old_sch_name => 'alter_me', - new_sch_name => 'altered' - ); - RETURN NEXT hasnt_schema('alter_me'); + PERFORM msar.patch_schema('foo', '{"name": "altered"}'); + RETURN NEXT hasnt_schema('foo'); RETURN NEXT has_schema('altered'); -END; -$$ LANGUAGE plpgsql; + RETURN NEXT is(obj_description(sch_oid), NULL); + RETURN NEXT is(msar.get_schema_name(sch_oid), 'altered'); + PERFORM msar.patch_schema(sch_oid, '{"description": "yay"}'); + RETURN NEXT is(obj_description(sch_oid), 'yay'); -CREATE OR REPLACE FUNCTION test_rename_schema_using_oid() RETURNS SETOF TEXT AS $$ -BEGIN - PERFORM __setup_alter_schema(); - PERFORM msar.rename_schema( - sch_id => 'alter_me'::regnamespace::oid, - new_sch_name => 'altered' - ); - RETURN NEXT hasnt_schema('alter_me'); - RETURN NEXT has_schema('altered'); -END; -$$ LANGUAGE plpgsql; + -- Edge case: setting the description to null doesn't actually remove it. This behavior is + -- debatable. I did it this way because it was easier to implement. + PERFORM msar.patch_schema(sch_oid, '{"description": null}'); + RETURN NEXT is(obj_description(sch_oid), 'yay'); + -- Description is removed when an empty string is passed. + PERFORM msar.patch_schema(sch_oid, '{"description": ""}'); + RETURN NEXT is(obj_description(sch_oid), NULL); -CREATE OR REPLACE FUNCTION test_comment_on_schema() RETURNS SETOF TEXT AS $$ -BEGIN - PERFORM __setup_alter_schema(); - PERFORM msar.comment_on_schema( - sch_name => 'alter_me', - comment_ => 'test comment' - ); - RETURN NEXT is(obj_description('alter_me'::regnamespace::oid), 'test comment'); + PERFORM msar.patch_schema(sch_oid, '{"name": "NEW", "description": "WOW"}'); + RETURN NEXT has_schema('NEW'); + RETURN NEXT is(msar.get_schema_name(sch_oid), 'NEW'); + RETURN NEXT is(obj_description(sch_oid), 'WOW'); + + -- Patching should be idempotent + PERFORM msar.patch_schema(sch_oid, '{"name": "NEW", "description": "WOW"}'); + RETURN NEXT has_schema('NEW'); + RETURN NEXT is(msar.get_schema_name(sch_oid), 'NEW'); + RETURN NEXT is(obj_description(sch_oid), 'WOW'); END; $$ LANGUAGE plpgsql; diff --git a/db/tests/schemas/operations/test_alter.py b/db/tests/schemas/operations/test_alter.py deleted file mode 100644 index 3cc3e7c995..0000000000 --- a/db/tests/schemas/operations/test_alter.py +++ /dev/null @@ -1,28 +0,0 @@ -from unittest.mock import patch -import db.schemas.operations.alter as sch_alter - - -def test_rename_schema(engine_with_schema): - engine = engine_with_schema - with patch.object(sch_alter, 'execute_msar_func_with_engine') as mock_exec: - sch_alter.rename_schema('rename_me', engine, rename_to='renamed') - call_args = mock_exec.call_args_list[0][0] - assert call_args[0] == engine - assert call_args[1] == "rename_schema" - assert call_args[2] == "rename_me" - assert call_args[3] == "renamed" - - -def test_comment_on_schema(engine_with_schema): - engine = engine_with_schema - with patch.object(sch_alter, 'execute_msar_func_with_engine') as mock_exec: - sch_alter.comment_on_schema( - schema_name='comment_on_me', - engine=engine, - comment='This is a comment' - ) - call_args = mock_exec.call_args_list[0][0] - assert call_args[0] == engine - assert call_args[1] == "comment_on_schema" - assert call_args[2] == "comment_on_me" - assert call_args[3] == "This is a comment" diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 1f2e5e0e30..2086e7d825 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -54,7 +54,9 @@ To use an RPC function: - list_ - add - delete + - patch - SchemaInfo + - SchemaPatch --- diff --git a/mathesar/rpc/schemas.py b/mathesar/rpc/schemas.py index 890f5c5747..66bd1e4b97 100644 --- a/mathesar/rpc/schemas.py +++ b/mathesar/rpc/schemas.py @@ -10,6 +10,7 @@ from db.schemas.operations.create import create_schema from db.schemas.operations.select import get_schemas from db.schemas.operations.drop import drop_schema_via_oid +from db.schemas.operations.alter import patch_schema from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -30,6 +31,16 @@ class SchemaInfo(TypedDict): table_count: int +class SchemaPatch(TypedDict): + """ + Attributes: + name: The name of the schema + description: A description of the schema + """ + name: Optional[str] + description: Optional[str] + + @rpc_method(name="schemas.add") @http_basic_auth_login_required @handle_rpc_exceptions @@ -82,13 +93,30 @@ def list_(*, database_id: int, **kwargs) -> list[SchemaInfo]: @rpc_method(name="schemas.delete") @http_basic_auth_login_required @handle_rpc_exceptions -def delete(*, schema_id: int, database_id: int, **kwargs) -> None: +def delete(*, schema_oid: int, database_id: int, **kwargs) -> None: """ Delete a schema, given its OID. Args: - schema_id: The OID of the schema to delete. + schema_oid: The OID of the schema to delete. database_id: The Django id of the database containing the schema. """ with connect(database_id, kwargs.get(REQUEST_KEY).user) as conn: - drop_schema_via_oid(conn, schema_id) + drop_schema_via_oid(conn, schema_oid) + + +@rpc_method(name="schemas.patch") +@http_basic_auth_login_required +@handle_rpc_exceptions +def patch(*, schema_oid: int, database_id: int, patch: SchemaPatch, **kwargs) -> None: + """ + Patch a schema, given its OID. + + Args: + schema_oid: The OID of the schema to delete. + database_id: The Django id of the database containing the schema. + patch: A SchemaPatch object containing the fields to update. + """ + with connect(database_id, kwargs.get(REQUEST_KEY).user) as conn: + patch_schema(schema_oid, conn, patch) + diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index c290cac843..d51ac78fa9 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -59,6 +59,11 @@ "schemas.delete", [user_is_authenticated] ), + ( + schemas.patch, + "schemas.patch", + [user_is_authenticated] + ), ( tables.list_, "tables.list", diff --git a/mathesar/utils/models.py b/mathesar/utils/models.py index 6072f95bd5..85904a0847 100644 --- a/mathesar/utils/models.py +++ b/mathesar/utils/models.py @@ -7,7 +7,7 @@ from rest_framework.exceptions import ValidationError from db.tables.operations.alter import alter_table, SUPPORTED_TABLE_ALTER_ARGS -from db.schemas.operations.alter import alter_schema, SUPPORTED_SCHEMA_ALTER_ARGS +from db.schemas.operations.alter import patch_schema_via_sql_alchemy from db.columns.exceptions import InvalidTypeError from mathesar.api.exceptions.error_codes import ErrorCodes @@ -47,12 +47,12 @@ def update_sa_schema(schema, validated_data): ErrorCodes.UnsupportedAlter.value, message=f'Updating {arg} for schema is not supported.' ) - for arg in set(validated_data) - SUPPORTED_SCHEMA_ALTER_ARGS] + for arg in set(validated_data) - {'name', 'description'}] if errors: raise base_api_exceptions.GenericAPIException(errors, status_code=status.HTTP_400_BAD_REQUEST) if errors: raise ValidationError(errors) - alter_schema(schema.name, schema._sa_engine, validated_data) + patch_schema_via_sql_alchemy(schema.name, schema._sa_engine, validated_data) def ensure_cached_engine_ready(engine): From ecc6fe74da75bdfa56a2fee3538fe42daebbc758 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 17 Jun 2024 20:48:52 -0400 Subject: [PATCH 0269/1141] Fix linting error --- mathesar/rpc/schemas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mathesar/rpc/schemas.py b/mathesar/rpc/schemas.py index 66bd1e4b97..a372f3b8e0 100644 --- a/mathesar/rpc/schemas.py +++ b/mathesar/rpc/schemas.py @@ -119,4 +119,3 @@ def patch(*, schema_oid: int, database_id: int, patch: SchemaPatch, **kwargs) -> """ with connect(database_id, kwargs.get(REQUEST_KEY).user) as conn: patch_schema(schema_oid, conn, patch) - From 85c87813906b4077208b37d03e9c233491b3ce87 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 18 Jun 2024 15:59:00 +0800 Subject: [PATCH 0270/1141] add connect method to map model, fix constraint --- .../migrations/0007_users_permissions_remodel.py | 4 ++++ mathesar/models/base.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mathesar/migrations/0007_users_permissions_remodel.py b/mathesar/migrations/0007_users_permissions_remodel.py index f9d69b6212..7ee5e4a67b 100644 --- a/mathesar/migrations/0007_users_permissions_remodel.py +++ b/mathesar/migrations/0007_users_permissions_remodel.py @@ -77,6 +77,10 @@ class Migration(migrations.Migration): 'abstract': False, }, ), + migrations.AddConstraint( + model_name='role', + constraint=models.UniqueConstraint(fields=('name', 'server'), name='unique_role'), + ), migrations.AddConstraint( model_name='role', constraint=models.UniqueConstraint(fields=('id', 'server'), name='role_id_server_index'), diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 1086f3c87a..971837219f 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -1,5 +1,6 @@ from django.db import models from encrypted_fields.fields import EncryptedCharField +import psycopg class BaseModel(models.Model): @@ -41,6 +42,9 @@ class Role(BaseModel): class Meta: constraints = [ + models.UniqueConstraint( + fields=["name", "server"], name="unique_role" + ), models.UniqueConstraint( fields=["id", "server"], name="role_id_server_index" ) @@ -59,3 +63,14 @@ class Meta: fields=["user", "database"], name="user_one_role_per_database" ) ] + + + @property + def connection(self): + return psycopg.connect( + host=self.server.host, + port=self.server.port, + dbname=self.database.name, + user=self.role.name, + password=self.role.password, + ) From 4bd3487d39eac3d26bff4b55560b09a0a008b6e6 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 18 Jun 2024 16:12:35 +0800 Subject: [PATCH 0271/1141] add python function to migrate connections to new models --- mathesar/utils/permissions.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 mathesar/utils/permissions.py diff --git a/mathesar/utils/permissions.py b/mathesar/utils/permissions.py new file mode 100644 index 0000000000..c9272a036b --- /dev/null +++ b/mathesar/utils/permissions.py @@ -0,0 +1,20 @@ +from mathesar.models.base import Server, Database, Role, UserDatabaseRoleMap +from mathesar.models.deprecated import Connection +from mathesar.models.users import User + + +def create_user_database_role_map(connection_id, user_id): + """Move data from old-style connection model to new models.""" + conn = Connection.current_objects.get(id=connection_id) + + server = Server.objects.get_or_create(host=conn.host, port=conn.port)[0] + database = Database.objects.get_or_create(name=conn.db_name, server=server)[0] + role = Role.objects.get_or_create( + name=conn.username, server=server, password=conn.password + )[0] + return UserDatabaseRoleMap.objects.create( + user=User.objects.get(id=user_id), + database=database, + role=role, + server=server + ) From 386e93fa7a8758bcf9319a4b03f7b5063f766844 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 18 Jun 2024 16:17:12 +0800 Subject: [PATCH 0272/1141] add RPC utility function for migrating old connections --- mathesar/rpc/connections.py | 21 ++++++++++++++++++++- mathesar/tests/rpc/test_endpoints.py | 5 +++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/mathesar/rpc/connections.py b/mathesar/rpc/connections.py index 7972992a47..59be60a74f 100644 --- a/mathesar/rpc/connections.py +++ b/mathesar/rpc/connections.py @@ -6,7 +6,7 @@ from modernrpc.core import rpc_method from modernrpc.auth.basic import http_basic_auth_superuser_required -from mathesar.utils import connections +from mathesar.utils import connections, permissions from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -125,3 +125,22 @@ def add_from_scratch( user, password, host, port, nickname, database, sample_data ) return DBModelReturn.from_db_model(db_model) + + +@rpc_method(name='connections.grant_access_to_user') +@http_basic_auth_superuser_required +@handle_rpc_exceptions +def grant_access_to_user(*, connection_id: int, user_id: int): + """ + Migrate a connection to new models, grant access for a user. + + This function is designed to be temporary, and should probably be + removed once we have completed the new users and permissions setup + for beta. You pass any conneciton id and user id. The function will + fill the required models as needed. + + Args: + connection_id: The Django id of an old-style connection. + user_id: The Django id of a user. + """ + permissions.create_user_database_role_map(connection_id, user_id) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index c290cac843..35a7cd75e1 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -44,6 +44,11 @@ "connections.add_from_scratch", [user_is_superuser] ), + ( + connections.grant_access_to_user, + "connections.grant_access_to_user", + [user_is_superuser] + ), ( schemas.add, "schemas.add", From 000f3cd1606ef6f346533f866e00d47fc36c3a93 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 18 Jun 2024 16:17:44 +0800 Subject: [PATCH 0273/1141] use new models to get connections for db access --- mathesar/rpc/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mathesar/rpc/utils.py b/mathesar/rpc/utils.py index 57118f0f9b..c307a8986c 100644 --- a/mathesar/rpc/utils.py +++ b/mathesar/rpc/utils.py @@ -1,14 +1,14 @@ -from mathesar.database.base import get_psycopg_connection -from mathesar.models.deprecated import Connection +from mathesar.models.base import UserDatabaseRoleMap -def connect(conn_id, user): +def connect(database_id, user): """ Return a psycopg connection, given a Connection model id. Args: conn_id: The Django id corresponding to the Connection. """ - print("User is: ", user) - conn_model = Connection.current_objects.get(id=conn_id) - return get_psycopg_connection(conn_model) + user_database_role = UserDatabaseRoleMap.objects.get( + user=user, database__id=database_id + ) + return user_database_role.connection From effa934b47d2f6428d2171c0cd16acfa131ca5fc Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 18 Jun 2024 16:47:59 +0800 Subject: [PATCH 0274/1141] implement documentation for new RPC function --- docs/docs/api/rpc.md | 1 + mathesar/rpc/connections.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 1f2e5e0e30..b149a9045d 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -44,6 +44,7 @@ To use an RPC function: members: - add_from_known_connection - add_from_scratch + - grant_access_to_user - DBModelReturn --- diff --git a/mathesar/rpc/connections.py b/mathesar/rpc/connections.py index 59be60a74f..578b56f061 100644 --- a/mathesar/rpc/connections.py +++ b/mathesar/rpc/connections.py @@ -132,7 +132,7 @@ def add_from_scratch( @handle_rpc_exceptions def grant_access_to_user(*, connection_id: int, user_id: int): """ - Migrate a connection to new models, grant access for a user. + Migrate a connection to new models and grant access to a user. This function is designed to be temporary, and should probably be removed once we have completed the new users and permissions setup From 68a82c312c1208f14a96dd10c17aadeb1f823f44 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 18 Jun 2024 16:49:12 +0800 Subject: [PATCH 0275/1141] fix formatting, make flake8 happy --- mathesar/models/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 971837219f..bb4dd5e8b1 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -64,7 +64,6 @@ class Meta: ) ] - @property def connection(self): return psycopg.connect( From edd9d0a82a57ddd4d2f5d597feab337bf96c530b Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 18 Jun 2024 17:02:31 +0530 Subject: [PATCH 0276/1141] fix python imports --- db/tables/operations/import_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/tables/operations/import_.py b/db/tables/operations/import_.py index 0b4a114e64..52f02d2fe8 100644 --- a/db/tables/operations/import_.py +++ b/db/tables/operations/import_.py @@ -4,7 +4,7 @@ from db.tables.operations.create import prepare_table_for_import from db.tables.operations.select import get_fully_qualified_relation_name from db.encoding_utils import get_sql_compatible_encoding -from mathesar.models.base import DataFile +from mathesar.models.deprecated import DataFile from mathesar.imports.csv import get_file_encoding, get_sv_reader, process_column_names From bc766fcc80c8fe9d3da1b378867076212a0c9b8e Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 18 Jun 2024 21:43:25 +0800 Subject: [PATCH 0277/1141] add unique constraint on server --- mathesar/migrations/0007_users_permissions_remodel.py | 4 ++++ mathesar/models/base.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/mathesar/migrations/0007_users_permissions_remodel.py b/mathesar/migrations/0007_users_permissions_remodel.py index 7ee5e4a67b..1f790a1622 100644 --- a/mathesar/migrations/0007_users_permissions_remodel.py +++ b/mathesar/migrations/0007_users_permissions_remodel.py @@ -45,6 +45,10 @@ class Migration(migrations.Migration): 'abstract': False, }, ), + migrations.AddConstraint( + model_name='server', + constraint=models.UniqueConstraint(fields=('host', 'port'), name='unique_server'), + ), migrations.CreateModel( name='Database', fields=[ diff --git a/mathesar/models/base.py b/mathesar/models/base.py index bb4dd5e8b1..de7454dd6d 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -15,6 +15,13 @@ class Server(BaseModel): host = models.CharField(max_length=255) port = models.IntegerField() + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["host", "port"], name="unique_server" + ), + ] + class Database(BaseModel): name = models.CharField(max_length=128) From defc2a1201864261dcd5314af9c9016c0af9c901 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 18 Jun 2024 21:43:55 +0800 Subject: [PATCH 0278/1141] correct documentation for function --- mathesar/rpc/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mathesar/rpc/utils.py b/mathesar/rpc/utils.py index c307a8986c..bffa807a73 100644 --- a/mathesar/rpc/utils.py +++ b/mathesar/rpc/utils.py @@ -3,10 +3,11 @@ def connect(database_id, user): """ - Return a psycopg connection, given a Connection model id. + Get a psycopg database connection. Args: - conn_id: The Django id corresponding to the Connection. + database_id: The Django id of the Database used for connecting. + user: A user model instance who'll connect to the database. """ user_database_role = UserDatabaseRoleMap.objects.get( user=user, database__id=database_id From 3b32521e9c6fe8bb3ff7b8eeceb5206e225826bd Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 16:20:09 -0400 Subject: [PATCH 0279/1141] Improve SQL code docs and begin SQL standards --- db/sql/00_msar.sql | 55 ------------------------------------------ db/sql/README.md | 47 ++++++++++++++++++++++++++++++++++++ db/sql/STANDARDS.md | 59 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 55 deletions(-) create mode 100644 db/sql/README.md create mode 100644 db/sql/STANDARDS.md diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 52e69e92e9..bbc026ec6e 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1,58 +1,3 @@ -/* -This script defines a number of functions to be used for manipulating database objects (tables, -columns, schemas) using Data Definition Language style queries. - -These are the schemas where the new functions will generally live: - - __msar: These functions aren't designed to be used except by other Mathesar functions. - Generally need preformatted strings as input, won't do quoting, etc. - msar: These functions are designed to be used more easily. They'll format strings, quote - identifiers, and so on. - -The reason they're so abbreviated is to avoid namespace clashes, and also because making them longer -would make using them quite tedious, since they're everywhere. - -The functions should each be overloaded to accept at a minimum the 'fixed' ID of a given object, as -well as its name identifer(s). - -- Schemas should be identified by one of the following: - - OID, or - - Name -- Tables should be identified by one of the following: - - OID, or - - Schema, Name pair (unquoted) -- Columns should be identified by one of the following: - - OID, ATTNUM pair, or - - Schema, Table Name, Column Name triple (unquoted), or - - Table OID, Column Name pair (optional). - -Note that these identification schemes apply to the public-facing functions in the `msar` namespace, -not necessarily the internal `__msar` functions. - -NAMING CONVENTIONS - -Because function signatures are used informationally in command-generated tables, horizontal space -needs to be conserved. As a compromise between readability and terseness, we use the following -conventions in variable naming: - -attribute -> att -schema -> sch -table -> tab -column -> col -constraint -> con -object -> obj -relation -> rel - -Textual names will have the suffix _name, and numeric identifiers will have the suffix _id. - -So, the OID of a table will be tab_id and the name of a column will be col_name. The attnum of a -column will be col_id. - -Generally, we'll use snake_case for legibility and to avoid collisions with internal PostgreSQL -naming conventions. - -*/ - CREATE SCHEMA IF NOT EXISTS __msar; CREATE SCHEMA IF NOT EXISTS msar; diff --git a/db/sql/README.md b/db/sql/README.md new file mode 100644 index 0000000000..08861b9904 --- /dev/null +++ b/db/sql/README.md @@ -0,0 +1,47 @@ +# SQL code + +A substantial amount of Mathesar's application logic is implemented directly in the PostgreSQL database layer. This directory holds the code for that logic, as written in PL/pgSQL. + +Also see our [SQL code standards](./STANDARDS.md) when making changes to the SQL code. + + +## Schemas + +Mathesar installs multiple schemas in the PostgreSQL database for itself to run. We use these internal schemas to hold our custom functions and types. + +### msar + +This is the main schema where we define Mathesar-specific functionality. + +### __msar + +This is a legacy schema that is now **deprecated**. Try to avoid using it if you can. + +It was originally intended for private use within the Mathesar SQL layer (not to be called by the service layer). So if you do use this schema, don't call its functions from within the service layer. + +### mathesar_types + +This schema holds types which the user might utilize in their own tables as well as types for our internal use. + + +## Testing + +SQL code is tested using [pgTAP](https://pgtap.org/). + +- Run all tests: + + ``` + docker exec mathesar_dev_db /bin/bash /sql/run_tests.sh -v + ``` + +- Run tests having names which contain `foo_bar`: + + ``` + docker exec mathesar_dev_db /bin/bash /sql/run_tests.sh -v -x foo_bar + ``` + + +## When modifying the set of Mathesar-internal schemas + +The names of all schemas managed by Mathesar in this SQL is also duplicated in [constants.py](../constants.py). Any changes to these schema name (e.g. adding a new internal schema) must be propagated there too. + diff --git a/db/sql/STANDARDS.md b/db/sql/STANDARDS.md new file mode 100644 index 0000000000..7310f20697 --- /dev/null +++ b/db/sql/STANDARDS.md @@ -0,0 +1,59 @@ +# SQL Code Standards + +## Naming conventions + +Because function signatures are used informationally in command-generated tables, horizontal space needs to be conserved. As a compromise between readability and terseness, we use the following conventions in variable naming: + +| Object | Naming abbreviation | +| -- | -- | +| attribute | `att` | +| schema | `sch` | +| table | `tab` | +| column | `col` | +| constraint | `con` | +| object | `obj` | +| relation | `rel` | + +Textual names have the suffix `_name`, and numeric identifiers have the suffix `_id`. + +Examples + +- The OID of a table is `tab_id` +- The name of a column is `col_name` +- The attnum of a column is `col_id` + +## Casing + +We use `snake_case` for basically everything — schema names, function names, variables, and types. + +## Code documentation + +Every function should have a docstring-style code documentation block. Follow the syntax of existing functions when writing new ones. + +## Quoting, escaping, SQL injection, and security + +As of mid-2024, Mathesar is in the midst of a gradual transition from one pattern of quoting to another. + +- **The old pattern** is used for all functions within the (deprecated) `__msar` schema and will eventually be refactored out. + + In this pattern, if the name of a database object is accepted as a function argument, stored as an intermediate variable, or returned from a function, then that name is _quoted_ in preparation for it to eventually be used in an SQL statement. For example, a table name of `foo bar` would be passed around as `"foo bar"`. + +- **The new pattern** is used for all functions within the `msar` schema, and will be used going forward. + + In this pattern all names are passed around unquoted for as long as possible. Like above, this applies to names in function arguments, intermediate variables, and return values. They are only quoted at the latest possible point in their execution path, i.e. when they are put into SQL. + + One way to think about this pattern is: + + **If it _can_ be unquoted, then it _should_ be unquoted.** + + For example, if you're dealing with a plain table name such as `foo bar`, then leave it unquoted. If you need to qualify that table name with a schema too, then either store both values unquoted in some composite type (e.g. jsonb, tuple, custom type, etc) or store them _quoted_ in a string like `"my schema"."foo bar"`. That string represents a fragment of SQL, and because of the dot there is no way to leave the values unquoted. With fragments of SQL like this, utilize descriptive naming and helpful code comments to be extra clear about when a string represents an SQL fragment. Try not to pass around too many fragments of SQL if you can avoid it. Prefer to pass around raw, unquoted values when possible/easy. + +From [OWASP](https://owasp.org/www-project-proactive-controls/v3/en/c4-encode-escape-data): + +> Output encoding is best applied **just before** the content is passed to the target interpreter. If this defense is performed too early in the processing of a request then the encoding or escaping may interfere with the use of the content in other parts of the program. + +## System catalog qualification + +Always qualify system catalog tables by prefixing them with `pg_catalog.`. If you don't, then user-defined tables can shadow the system catalog tables, breaking core functionality. + + From 874868b428697eca5d35aa246f32ef1005caf56e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 16:22:58 -0400 Subject: [PATCH 0280/1141] Rename msar.get_fully_qualified_object_name --- db/sql/00_msar.sql | 38 ++++++++++++-------------------------- db/sql/20_msar_views.sql | 2 +- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index bbc026ec6e..710e92704d 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -122,21 +122,7 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -__msar.get_fully_qualified_object_name(sch_name text, obj_name text) RETURNS text AS $$/* -Return the fully-qualified name for a given database object (e.g., table). - -Args: - sch_name: The schema of the object, quoted. - obj_name: The name of the object, unqualified and quoted. -*/ -BEGIN - RETURN format('%s.%s', sch_name, obj_name); -END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; - - -CREATE OR REPLACE FUNCTION -msar.get_fully_qualified_object_name(sch_name text, obj_name text) RETURNS text AS $$/* +msar.build_qualified_name_sql(sch_name text, obj_name text) RETURNS text AS $$/* Return the fully-qualified, properly quoted, name for a given database object (e.g., table). Args: @@ -144,7 +130,7 @@ Args: obj_name: The name of the object, unqualified and unquoted. */ BEGIN - RETURN __msar.get_fully_qualified_object_name(quote_ident(sch_name), quote_ident(obj_name)); + RETURN format('%I.%I', sch_name, obj_name); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -199,7 +185,7 @@ Args: rel_name: The name of the relation, unqualified and unquoted. */ BEGIN - RETURN msar.get_fully_qualified_object_name(sch_name, rel_name)::regclass::oid; + RETURN msar.build_qualified_name_sql(sch_name, rel_name)::regclass::oid; END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1032,7 +1018,7 @@ Args: */ DECLARE fullname text; BEGIN - fullname := msar.get_fully_qualified_object_name(sch_name, old_tab_name); + fullname := msar.build_qualified_name_sql(sch_name, old_tab_name); RETURN __msar.rename_table(fullname, quote_ident(new_tab_name)); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1078,7 +1064,7 @@ Args: comment_: The new comment. */ SELECT __msar.comment_on_table( - msar.get_fully_qualified_object_name(sch_name, tab_name), + msar.build_qualified_name_sql(sch_name, tab_name), quote_literal(comment_) ); $$ LANGUAGE SQL; @@ -1176,7 +1162,7 @@ Args: */ DECLARE qualified_tab_name text; BEGIN - qualified_tab_name := msar.get_fully_qualified_object_name(sch_name, tab_name); + qualified_tab_name := msar.build_qualified_name_sql(sch_name, tab_name); RETURN __msar.update_pk_sequence_to_latest(qualified_tab_name, quote_ident(col_name)); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1235,7 +1221,7 @@ DECLARE prepared_col_names text[]; DECLARE fully_qualified_tab_name text; BEGIN SELECT array_agg(quote_ident(col)) FROM unnest(col_names) AS col INTO prepared_col_names; - fully_qualified_tab_name := msar.get_fully_qualified_object_name(sch_name, tab_name); + fully_qualified_tab_name := msar.build_qualified_name_sql(sch_name, tab_name); RETURN __msar.drop_columns(fully_qualified_tab_name, variadic prepared_col_names); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1531,7 +1517,7 @@ SELECT COALESCE( -- Second choice is the type specified by string IDs. __msar.get_formatted_base_type( COALESCE( - msar.get_fully_qualified_object_name(typ_jsonb ->> 'schema', typ_jsonb ->> 'name'), + msar.build_qualified_name_sql(typ_jsonb ->> 'schema', typ_jsonb ->> 'name'), typ_jsonb ->> 'name', 'text' -- We fall back to 'text' when input is null or empty. ), @@ -1892,7 +1878,7 @@ SELECT array_agg( -- Build the relation name where the constraint will be applied. Prefer numeric ID. COALESCE( __msar.get_relation_name((con_create_obj -> 'fkey_relation_id')::integer::oid), - msar.get_fully_qualified_object_name( + msar.build_qualified_name_sql( con_create_obj ->> 'fkey_relation_schema', con_create_obj ->> 'fkey_relation_name' ) ), @@ -2165,7 +2151,7 @@ Args: */ DECLARE qualified_tab_name text; BEGIN - qualified_tab_name := msar.get_fully_qualified_object_name(sch_name, tab_name); + qualified_tab_name := msar.build_qualified_name_sql(sch_name, tab_name); RETURN __msar.drop_table(qualified_tab_name, cascade_, if_exists); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -2209,7 +2195,7 @@ Args: */ BEGIN RETURN __msar.drop_constraint( - msar.get_fully_qualified_object_name(sch_name, tab_name), quote_ident(con_name) + msar.build_qualified_name_sql(sch_name, tab_name), quote_ident(con_name) ); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -2673,7 +2659,7 @@ Args: comment_: The new comment. */ SELECT __msar.comment_on_column( - msar.get_fully_qualified_object_name(sch_name, tab_name), + msar.build_qualified_name_sql(sch_name, tab_name), quote_ident(col_name), quote_literal(comment_) ); diff --git a/db/sql/20_msar_views.sql b/db/sql/20_msar_views.sql index ad15a60284..cccb08b737 100644 --- a/db/sql/20_msar_views.sql +++ b/db/sql/20_msar_views.sql @@ -32,7 +32,7 @@ Args: tab_id: The OID of the table whose associated view we want to name. */ BEGIN - RETURN msar.get_fully_qualified_object_name('msar_views', format('mv%s', tab_id)); + RETURN msar.build_qualified_name_sql('msar_views', format('mv%s', tab_id)); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; From d715b14e7e7ce6fd315f7330d476d8828f36104d Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 16:23:54 -0400 Subject: [PATCH 0281/1141] Rename msar.build_unique_column_name_unquoted --- db/sql/00_msar.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 710e92704d..4f0e27da9c 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1307,7 +1307,7 @@ $$ LANGUAGE sql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.build_unique_column_name_unquoted(tab_id oid, col_name text) RETURNS text AS $$/* +msar.build_unique_column_name(tab_id oid, col_name text) RETURNS text AS $$/* Get a unique column name based on the given name. Args: @@ -1342,7 +1342,7 @@ will be of the form: _id. Then, we apply some logic to ensure the res */ BEGIN fk_col_name := COALESCE(fk_col_name, format('%s_id', frel_name)); - RETURN msar.build_unique_column_name_unquoted(tab_id, fk_col_name); + RETURN msar.build_unique_column_name(tab_id, fk_col_name); END; $$ LANGUAGE plpgsql; From 51eec9ce1f6e2f6051912f6570573f9162b140da Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 16:25:10 -0400 Subject: [PATCH 0282/1141] Move get_duplicate_col_defs into __msar --- db/sql/00_msar.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 4f0e27da9c..498ccd6cef 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1267,7 +1267,7 @@ END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION msar.get_duplicate_col_defs( +CREATE OR REPLACE FUNCTION __msar.get_duplicate_col_defs( tab_id oid, col_ids smallint[], new_names text[], @@ -2021,7 +2021,7 @@ DECLARE col_name text; created_col_id smallint; BEGIN - col_defs := msar.get_duplicate_col_defs( + col_defs := __msar.get_duplicate_col_defs( tab_id, ARRAY[col_id], ARRAY[copy_name], copy_data ); tab_name := __msar.get_relation_name(tab_id); From 34b15bc338da6d23332ede571ebc486355d7df0d Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 16:26:40 -0400 Subject: [PATCH 0283/1141] Move process_col_def_jsonb into __msar --- db/sql/00_msar.sql | 12 ++++++------ db/sql/test_00_msar.sql | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 498ccd6cef..bd8397e70d 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1351,7 +1351,7 @@ CREATE OR REPLACE FUNCTION msar.get_extracted_col_def_jsonb(tab_id oid, col_ids integer[]) RETURNS jsonb AS $$/* Get a JSON array of column definitions from given columns for creation of an extracted table. -See the msar.process_col_def_jsonb for a description of the JSON. +See the __msar.process_col_def_jsonb for a description of the JSON. Args: tab_id: The OID of the table containing the columns whose definitions we want. @@ -1578,7 +1578,7 @@ $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION -msar.process_col_def_jsonb( +__msar.process_col_def_jsonb( tab_id oid, col_defs jsonb, raw_default boolean, @@ -1687,14 +1687,14 @@ Add columns to a table. Args: tab_id: The OID of the table to which we'll add columns. - col_defs: a JSONB array defining columns to add. See msar.process_col_def_jsonb for details. + col_defs: a JSONB array defining columns to add. See __msar.process_col_def_jsonb for details. raw_default: Whether to treat defaults as raw SQL. DANGER! */ DECLARE col_create_defs __msar.col_def[]; fq_table_name text := __msar.get_relation_name(tab_id); BEGIN - col_create_defs := msar.process_col_def_jsonb(tab_id, col_defs, raw_default); + col_create_defs := __msar.process_col_def_jsonb(tab_id, col_defs, raw_default); PERFORM __msar.add_columns(fq_table_name, variadic col_create_defs); PERFORM @@ -1722,7 +1722,7 @@ Add columns to a table. Args: sch_name: unquoted schema name of the table to which we'll add columns. tab_name: unquoted, unqualified name of the table to which we'll add columns. - col_defs: a JSONB array defining columns to add. See msar.process_col_def_jsonb for details. + col_defs: a JSONB array defining columns to add. See __msar.process_col_def_jsonb for details. raw_default: Whether to treat defaults as raw SQL. DANGER! */ SELECT msar.add_columns(msar.get_relation_oid(sch_name, tab_name), col_defs, raw_default); @@ -2271,7 +2271,7 @@ DECLARE constraint_defs __msar.con_def[]; BEGIN fq_table_name := format('%s.%s', __msar.get_schema_name(sch_oid), quote_ident(tab_name)); - column_defs := msar.process_col_def_jsonb(0, col_defs, false, true); + column_defs := __msar.process_col_def_jsonb(0, col_defs, false, true); constraint_defs := msar.process_con_def_jsonb(0, con_defs); PERFORM __msar.add_table(fq_table_name, column_defs, constraint_defs); created_table_id := fq_table_name::regclass::oid; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index db10bb73e5..1fa87b46b7 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -169,12 +169,12 @@ END; $$ LANGUAGE plpgsql; --- msar.process_col_def_jsonb ---------------------------------------------------------------------- +-- __msar.process_col_def_jsonb ---------------------------------------------------------------------- CREATE OR REPLACE FUNCTION test_process_col_def_jsonb() RETURNS SETOF TEXT AS $f$ BEGIN RETURN NEXT is( - msar.process_col_def_jsonb(0, '[{}, {}]'::jsonb, false), + __msar.process_col_def_jsonb(0, '[{}, {}]'::jsonb, false), ARRAY[ ('"Column 1"', 'text', null, null, false, null), ('"Column 2"', 'text', null, null, false, null) @@ -182,12 +182,12 @@ BEGIN 'Empty columns should result in defaults' ); RETURN NEXT is( - msar.process_col_def_jsonb(0, '[{"name": "id"}]'::jsonb, false), + __msar.process_col_def_jsonb(0, '[{"name": "id"}]'::jsonb, false), null, 'Column definition processing should ignore "id" column' ); RETURN NEXT is( - msar.process_col_def_jsonb(0, '[{}, {}]'::jsonb, false, true), + __msar.process_col_def_jsonb(0, '[{}, {}]'::jsonb, false, true), ARRAY[ ('id', 'integer', true, null, true, 'Mathesar default ID column'), ('"Column 1"', 'text', null, null, false, null), @@ -196,7 +196,7 @@ BEGIN 'Column definition processing add "id" column' ); RETURN NEXT is( - msar.process_col_def_jsonb(0, '[{"description": "Some comment"}]'::jsonb, false), + __msar.process_col_def_jsonb(0, '[{"description": "Some comment"}]'::jsonb, false), ARRAY[ ('"Column 1"', 'text', null, null, false, '''Some comment''') ]::__msar.col_def[], From dcacde00c58c482529b63c580eafd3404342db11 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 16:27:17 -0400 Subject: [PATCH 0284/1141] Move process_con_def_jsonb into __msar --- db/sql/00_msar.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index bd8397e70d..9295fb1d80 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1831,7 +1831,7 @@ $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.process_con_def_jsonb(tab_id oid, con_create_arr jsonb) +__msar.process_con_def_jsonb(tab_id oid, con_create_arr jsonb) RETURNS __msar.con_def[] AS $$/* Create an array of __msar.con_def from a JSON array of constraint creation defining JSON. @@ -1927,12 +1927,12 @@ Add constraints to a table. Args: tab_id: The OID of the table to which we'll add constraints. - col_defs: a JSONB array defining constraints to add. See msar.process_con_def_jsonb for details. + col_defs: a JSONB array defining constraints to add. See __msar.process_con_def_jsonb for details. */ DECLARE con_create_defs __msar.con_def[]; BEGIN - con_create_defs := msar.process_con_def_jsonb(tab_id, con_defs); + con_create_defs := __msar.process_con_def_jsonb(tab_id, con_defs); PERFORM __msar.add_constraints(__msar.get_relation_name(tab_id), variadic con_create_defs); RETURN array_agg(oid) FROM pg_constraint WHERE conrelid=tab_id; END; @@ -1947,7 +1947,7 @@ Add constraints to a table. Args: sch_name: unquoted schema name of the table to which we'll add constraints. tab_name: unquoted, unqualified name of the table to which we'll add constraints. - con_defs: a JSONB array defining constraints to add. See msar.process_con_def_jsonb for details. + con_defs: a JSONB array defining constraints to add. See __msar.process_con_def_jsonb for details. */ SELECT msar.add_constraints(msar.get_relation_oid(sch_name, tab_name), con_defs); $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -2054,7 +2054,7 @@ CREATE OR REPLACE FUNCTION msar.get_extracted_con_def_jsonb(tab_id oid, col_ids integer[]) RETURNS jsonb AS $$/* Get a JSON array of constraint definitions from given columns for creation of an extracted table. -See the msar.process_con_def_jsonb for a description of the JSON. +See the __msar.process_con_def_jsonb for a description of the JSON. Args: tab_id: The OID of the table containing the constraints whose definitions we want. @@ -2272,7 +2272,7 @@ DECLARE BEGIN fq_table_name := format('%s.%s', __msar.get_schema_name(sch_oid), quote_ident(tab_name)); column_defs := __msar.process_col_def_jsonb(0, col_defs, false, true); - constraint_defs := msar.process_con_def_jsonb(0, con_defs); + constraint_defs := __msar.process_con_def_jsonb(0, con_defs); PERFORM __msar.add_table(fq_table_name, column_defs, constraint_defs); created_table_id := fq_table_name::regclass::oid; PERFORM msar.comment_on_table(created_table_id, comment_); From efe4cf2be743b8a4fe860d006f49ce8bd9431852 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 17:02:50 -0400 Subject: [PATCH 0285/1141] Unquote msar.get_column_name return value --- db/sql/00_msar.sql | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 9295fb1d80..c22a7b838a 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -205,7 +205,7 @@ $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION msar.get_column_name(rel_id oid, col_id integer) RETURNS text AS $$/* -Return the name for a given column in a given relation (e.g., table). +Return the UNQUOTED name for a given column in a given relation (e.g., table). More precisely, this function returns the name of attributes of any relation appearing in the pg_class catalog table (so you could find attributes of indices with this function). @@ -214,13 +214,13 @@ Args: rel_id: The OID of the relation. col_id: The attnum of the column in the relation. */ -SELECT quote_ident(attname::text) FROM pg_attribute WHERE attrelid=rel_id AND attnum=col_id; +SELECT attname::text FROM pg_attribute WHERE attrelid=rel_id AND attnum=col_id; $$ LANGUAGE sql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION msar.get_column_name(rel_id oid, col_name text) RETURNS text AS $$/* -Return the name for a given column in a given relation (e.g., table). +Return the UNQUOTED name for a given column in a given relation (e.g., table). More precisely, this function returns the quoted name of attributes of any relation appearing in the pg_class catalog table (so you could find attributes of indices with this function). If the given @@ -233,7 +233,7 @@ Args: rel_id: The OID of the relation. col_name: The unquoted name of the column in the relation. */ -SELECT quote_ident(attname::text) FROM pg_attribute WHERE attrelid=rel_id AND attname=col_name; +SELECT attname::text FROM pg_attribute WHERE attrelid=rel_id AND attname=col_name; $$ LANGUAGE sql RETURNS NULL ON NULL INPUT; @@ -259,8 +259,8 @@ Args: SELECT array_agg( CASE WHEN rel_id=0 THEN quote_ident(col #>> '{}') - WHEN jsonb_typeof(col)='number' THEN msar.get_column_name(rel_id, col::integer) - WHEN jsonb_typeof(col)='string' THEN msar.get_column_name(rel_id, col #>> '{}') + WHEN jsonb_typeof(col)='number' THEN quote_ident(msar.get_column_name(rel_id, col::integer)) + WHEN jsonb_typeof(col)='string' THEN quote_ident(msar.get_column_name(rel_id, col #>> '{}')) END ) FROM jsonb_array_elements(columns) AS x(col); @@ -1145,7 +1145,7 @@ DECLARE tab_name text; DECLARE col_name text; BEGIN tab_name := __msar.get_relation_name(tab_id); - col_name := msar.get_column_name(tab_id, col_id); + col_name := quote_ident(msar.get_column_name(tab_id, col_id)); RETURN __msar.update_pk_sequence_to_latest(tab_name, col_name); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -2025,7 +2025,7 @@ BEGIN tab_id, ARRAY[col_id], ARRAY[copy_name], copy_data ); tab_name := __msar.get_relation_name(tab_id); - col_name := msar.get_column_name(tab_id, col_id); + col_name := quote_ident(msar.get_column_name(tab_id, col_id)); PERFORM __msar.add_columns(tab_name, VARIADIC col_defs); created_col_id := attnum FROM pg_attribute @@ -2033,7 +2033,7 @@ BEGIN IF copy_data THEN PERFORM __msar.exec_ddl( 'UPDATE %s SET %s=%s', - tab_name, col_defs[1].name_, msar.get_column_name(tab_id, col_id) + tab_name, col_defs[1].name_, quote_ident(msar.get_column_name(tab_id, col_id)) ); END IF; IF copy_constraints THEN @@ -2326,7 +2326,7 @@ Args: BEGIN PERFORM __msar.rename_column( tab_name => __msar.get_relation_name(tab_id), - old_col_name => msar.get_column_name(tab_id, col_id), + old_col_name => quote_ident(msar.get_column_name(tab_id, col_id)), new_col_name => quote_ident(new_col_name) ); RETURN col_id; @@ -2376,7 +2376,7 @@ Args: column's default. */ SELECT CASE WHEN new_type IS NOT NULL OR jsonb_typeof(new_default)='null' THEN - 'ALTER COLUMN ' || msar.get_column_name(tab_id, col_id) || ' DROP DEFAULT' + 'ALTER COLUMN ' || quote_ident(msar.get_column_name(tab_id, col_id)) || ' DROP DEFAULT' END; $$ LANGUAGE SQL; @@ -2393,11 +2393,11 @@ Args: new_type: The target type to which we'll alter the column. */ SELECT 'ALTER COLUMN ' - || msar.get_column_name(tab_id, col_id) + || quote_ident(msar.get_column_name(tab_id, col_id)) || ' TYPE ' || new_type || ' USING ' - || __msar.build_cast_expr(msar.get_column_name(tab_id, col_id), new_type); + || __msar.build_cast_expr(quote_ident(msar.get_column_name(tab_id, col_id)), new_type); $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -2452,7 +2452,7 @@ BEGIN default_expr := format('%L', raw_default_expr); END IF; RETURN - format('ALTER COLUMN %s SET DEFAULT ', msar.get_column_name(tab_id, col_id)) || default_expr; + format('ALTER COLUMN %I SET DEFAULT ', msar.get_column_name(tab_id, col_id)) || default_expr; END; $$ LANGUAGE plpgsql; @@ -2467,7 +2467,7 @@ Args: not_null: If true, we 'SET NOT NULL'. If false, we 'DROP NOT NULL' if null, we do nothing. */ SELECT 'ALTER COLUMN ' - || msar.get_column_name(tab_id, col_id) + || quote_ident(msar.get_column_name(tab_id, col_id)) || CASE WHEN not_null THEN ' SET ' ELSE ' DROP ' END || 'NOT NULL'; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -2482,7 +2482,7 @@ Args: col_id: The attnum of the column whose nullability we'll alter. col_delete: If true, we drop the column. If false or null, we do nothing. */ -SELECT CASE WHEN col_delete THEN 'DROP COLUMN ' || msar.get_column_name(tab_id, col_id) END; +SELECT CASE WHEN col_delete THEN 'DROP COLUMN ' || quote_ident(msar.get_column_name(tab_id, col_id)) END; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -2681,7 +2681,7 @@ Args: */ SELECT __msar.comment_on_column( __msar.get_relation_name(tab_id), - msar.get_column_name(tab_id, col_id), + quote_ident(msar.get_column_name(tab_id, col_id)), comment_ ); $$ LANGUAGE SQL; From 19509ad72df1bcaf0849139a0564edcc06a37298 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 17:04:06 -0400 Subject: [PATCH 0286/1141] Move get_column_names into __msar --- db/sql/00_msar.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index c22a7b838a..a635625fb2 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -238,8 +238,8 @@ $$ LANGUAGE sql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.get_column_names(rel_id oid, columns jsonb) RETURNS text[] AS $$/* -Return the names for given columns in a given relation (e.g., table). +__msar.get_column_names(rel_id oid, columns jsonb) RETURNS text[] AS $$/* +Return the QUOTED names for given columns in a given relation (e.g., table). - If the rel_id is given as 0, the assumption is that this is a new table, so we just apply normal quoting rules to a column without validating anything further. @@ -1872,7 +1872,7 @@ SELECT array_agg( -- set the constraint type as a single char. See __msar.build_con_def_text for details. con_create_obj ->> 'type', -- Set the column names associated with the constraint. - msar.get_column_names(tab_id, con_create_obj -> 'columns'), + __msar.get_column_names(tab_id, con_create_obj -> 'columns'), -- Set whether the constraint is deferrable or not (boolean). con_create_obj ->> 'deferrable', -- Build the relation name where the constraint will be applied. Prefer numeric ID. @@ -1883,7 +1883,7 @@ SELECT array_agg( ) ), -- Build the array of foreign columns for an fkey constraint. - msar.get_column_names( + __msar.get_column_names( COALESCE( -- We validate that the given OID (if any) is correct. (con_create_obj -> 'fkey_relation_id')::integer::oid, From 06d6ebb5ea5b3db2cfa53676dce80200846a59d9 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 17:07:00 -0400 Subject: [PATCH 0287/1141] Unquote msar.get_constraint_name return value --- db/sql/00_msar.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index a635625fb2..485476b227 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -379,13 +379,13 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION msar.get_constraint_name(con_id oid) RETURNS text AS $$/* -Return the quoted constraint name of the correponding constraint oid. +Return the UNQUOTED constraint name of the corresponding constraint oid. Args: con_id: The OID of the constraint. */ BEGIN - RETURN quote_ident(conname::text) FROM pg_constraint WHERE pg_constraint.oid = con_id; + RETURN conname::text FROM pg_constraint WHERE pg_constraint.oid = con_id; END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -2211,7 +2211,8 @@ Args: */ BEGIN RETURN __msar.drop_constraint( - __msar.get_relation_name(tab_id), msar.get_constraint_name(con_id) + __msar.get_relation_name(tab_id), + quote_ident(msar.get_constraint_name(con_id)) ); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; From f2dd7bee725d1a28f3fdb77e795ea7b42c3c4960 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 17:36:05 -0400 Subject: [PATCH 0288/1141] Adjust wording of standards --- db/sql/STANDARDS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/sql/STANDARDS.md b/db/sql/STANDARDS.md index 7310f20697..a02a5b04a7 100644 --- a/db/sql/STANDARDS.md +++ b/db/sql/STANDARDS.md @@ -46,7 +46,9 @@ As of mid-2024, Mathesar is in the midst of a gradual transition from one patter **If it _can_ be unquoted, then it _should_ be unquoted.** - For example, if you're dealing with a plain table name such as `foo bar`, then leave it unquoted. If you need to qualify that table name with a schema too, then either store both values unquoted in some composite type (e.g. jsonb, tuple, custom type, etc) or store them _quoted_ in a string like `"my schema"."foo bar"`. That string represents a fragment of SQL, and because of the dot there is no way to leave the values unquoted. With fragments of SQL like this, utilize descriptive naming and helpful code comments to be extra clear about when a string represents an SQL fragment. Try not to pass around too many fragments of SQL if you can avoid it. Prefer to pass around raw, unquoted values when possible/easy. + For example, if you're dealing with a plain table name such as `foo bar`, then definitely leave it unquoted. + + To hone in on an edge case, let's say you need to qualify that table name with a schema name too. In this case try to handle and store both values (schema name and table name) separately (unquoted) as much as possible. You can use separate variables, separate arguments, or a composite type for return values. As a last resort, you can store the qualified name quoted in an SQL fragment string like `"my schema"."foo bar"`. We have some code like this already, but it's not ideal. Because of the dot in that SQL fragment, there is no way to leave the values unquoted. With fragments of SQL like this, take care to utilize descriptive naming and helpful code comments to be extra clear about when a string represents an SQL fragment. But in general try to avoid passing around SQL fragments if you can. Prefer to pass around raw unquoted values. Or better yet, pass around unique identifiers like OIDs when possible. From [OWASP](https://owasp.org/www-project-proactive-controls/v3/en/c4-encode-escape-data): From 2a6da8fbe21dc6a40428c1f84cb25a7ee462beed Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 18 Jun 2024 17:41:35 -0400 Subject: [PATCH 0289/1141] Move get_relation_name_or_null into __msar --- db/sql/00_msar.sql | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 485476b227..aad8e27e2d 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -154,7 +154,7 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.get_relation_name_or_null(rel_id oid) RETURNS text AS $$/* +__msar.get_relation_name_or_null(rel_id oid) RETURNS text AS $$/* Return the name for a given relation (e.g., table), qualified or quoted as appropriate. In cases where the relation is already included in the search path, the returned name will not be @@ -1002,7 +1002,10 @@ Args: new_tab_name: unquoted, unqualified table name */ BEGIN - RETURN __msar.rename_table(msar.get_relation_name_or_null(tab_id), quote_ident(new_tab_name)); + RETURN __msar.rename_table( + __msar.get_relation_name_or_null(tab_id), + quote_ident(new_tab_name) + ); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1050,7 +1053,7 @@ Args: tab_id: The OID of the table whose comment we will change. comment_: The new comment. */ -SELECT __msar.comment_on_table(msar.get_relation_name_or_null(tab_id), quote_literal(comment_)); +SELECT __msar.comment_on_table(__msar.get_relation_name_or_null(tab_id), quote_literal(comment_)); $$ LANGUAGE SQL; @@ -1097,7 +1100,7 @@ BEGIN PERFORM msar.rename_table(tab_id, new_tab_name); PERFORM msar.comment_on_table(tab_id, comment); PERFORM msar.alter_columns(tab_id, col_alters); - RETURN msar.get_relation_name_or_null(tab_id); + RETURN __msar.get_relation_name_or_null(tab_id); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1202,7 +1205,7 @@ BEGIN FROM pg_catalog.pg_attribute WHERE attrelid=tab_id AND NOT attisdropped AND ARRAY[attnum::integer] <@ col_ids INTO col_names; - PERFORM __msar.drop_columns(msar.get_relation_name_or_null(tab_id), variadic col_names); + PERFORM __msar.drop_columns(__msar.get_relation_name_or_null(tab_id), variadic col_names); RETURN array_length(col_names, 1); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -2128,7 +2131,7 @@ Args: */ DECLARE relation_name text; BEGIN - relation_name := msar.get_relation_name_or_null(tab_id); + relation_name := __msar.get_relation_name_or_null(tab_id); -- if_exists doesn't work while working with oids because -- the SQL query gets parameterized with tab_id instead of relation_name -- since we're unable to find the relation_name for a non existing table. From e1e17c080f9794350c97ad883adedd0897c68ccf Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 19 Jun 2024 17:40:39 +0530 Subject: [PATCH 0290/1141] fix flaky relation_name wrangling --- db/sql/00_msar.sql | 66 +++++++++++++++++++++------------ db/tables/operations/create.py | 23 ++++++++---- db/tables/operations/import_.py | 15 ++++---- db/tables/operations/select.py | 10 ----- 4 files changed, 64 insertions(+), 50 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 85965f84e1..2351a6e17e 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -222,30 +222,6 @@ END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION -msar.get_fully_qualified_relation_name(rel_id oid) RETURNS text AS $$/* -Return the fully-qualified name for a given relation (e.g., table). - -The relation *must* be in the pg_class table to use this function. This function will return NULL if -no corresponding relation can be found. - -Args: - rel_id: The OID of the relation. -*/ -DECLARE - sch_name text; - rel_name text; -BEGIN - SELECT nspname, relname INTO sch_name, rel_name - FROM pg_catalog.pg_class AS pgc - LEFT JOIN pg_catalog.pg_namespace AS pgn - ON pgc.relnamespace = pgn.oid - WHERE pgc.oid = rel_id; - RETURN msar.get_fully_qualified_object_name(sch_name, rel_name); -END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; - - CREATE OR REPLACE FUNCTION msar.get_relation_name_or_null(rel_id oid) RETURNS text AS $$/* Return the name for a given relation (e.g., table), qualified or quoted as appropriate. @@ -2374,6 +2350,48 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION +msar.prepare_table_for_import( + sch_oid oid, + tab_name text, + col_defs jsonb, + comment_ text +) RETURNS jsonb AS $$/* +Add a table, with a default id column, returning a JSON object describing the table. + +Each returned JSON object will have the form: + { + "schema_name": , + "table_name": , + "table_oid": + } + +Args: + sch_oid: The OID of the schema where the table will be created. + tab_name: The unquoted name for the new table. + col_defs: The columns for the new table, in order. + comment_ (optional): The comment for the new table. +*/ +DECLARE + sch_name text; + rel_name text; + rel_id oid; +BEGIN + rel_id := msar.add_mathesar_table(sch_oid, tab_name, col_defs, NULL, comment_); + SELECT nspname, relname INTO sch_name, rel_name + FROM pg_catalog.pg_class AS pgc + LEFT JOIN pg_catalog.pg_namespace AS pgn + ON pgc.relnamespace = pgn.oid + WHERE pgc.oid = rel_id; + RETURN jsonb_build_object( + 'schema_name', quote_ident(sch_name), + 'table_name', quote_ident(rel_name), + 'table_oid', rel_id + ); +END; +$$ LANGUAGE plpgsql; + + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -- COLUMN ALTERATION FUNCTIONS diff --git a/db/tables/operations/create.py b/db/tables/operations/create.py index d93d4a7972..9db9cd64a3 100644 --- a/db/tables/operations/create.py +++ b/db/tables/operations/create.py @@ -87,21 +87,28 @@ def prepare_table_for_import(table_name, schema_oid, column_names, conn, comment """ This method creates a Postgres table in the specified schema, with all columns being String type. + + Returns the schema_name, table_name and table_oid of the created table. """ - columns_ = [ + column_data_list = [ { "name": column_name, "type": {"name": PostgresType.TEXT.id} } for column_name in column_names ] - table_oid = create_table_on_database( - table_name=table_name, - schema_oid=schema_oid, - conn=conn, - column_data_list=columns_, - comment=comment + table_info = exec_msar_func( + conn, + 'prepare_table_for_import', + schema_oid, + table_name, + json.dumps(column_data_list), + comment + ).fetchone()[0] + return ( + table_info['schema_name'], + table_info['table_name'], + table_info['table_oid'] ) - return table_oid class CreateTableAs(DDLElement): diff --git a/db/tables/operations/import_.py b/db/tables/operations/import_.py index 52f02d2fe8..a22ec9121c 100644 --- a/db/tables/operations/import_.py +++ b/db/tables/operations/import_.py @@ -2,7 +2,6 @@ import clevercsv as csv from psycopg import sql from db.tables.operations.create import prepare_table_for_import -from db.tables.operations.select import get_fully_qualified_relation_name from db.encoding_utils import get_sql_compatible_encoding from mathesar.models.deprecated import DataFile from mathesar.imports.csv import get_file_encoding, get_sv_reader, process_column_names @@ -21,7 +20,7 @@ def import_csv(data_file_id, table_name, schema_oid, conn, comment=None): with open(file_path, 'rb') as csv_file: csv_reader = get_sv_reader(csv_file, header, dialect) column_names = process_column_names(csv_reader.fieldnames) - table_oid = prepare_table_for_import( + schema_name, table_name, table_oid = prepare_table_for_import( table_name, schema_oid, column_names, @@ -29,7 +28,8 @@ def import_csv(data_file_id, table_name, schema_oid, conn, comment=None): comment ) insert_csv_records( - table_oid, + schema_name, + table_name, conn, file_path, column_names, @@ -43,7 +43,8 @@ def import_csv(data_file_id, table_name, schema_oid, conn, comment=None): def insert_csv_records( - table_oid, + schema_name, + table_name, conn, file_path, column_names, @@ -54,13 +55,11 @@ def insert_csv_records( encoding=None ): conversion_encoding, sql_encoding = get_sql_compatible_encoding(encoding) - schema_name, table_name = get_fully_qualified_relation_name(table_oid, conn).split('.') formatted_columns = sql.SQL(",").join(sql.Identifier(column_name) for column_name in column_names) copy_sql = sql.SQL( - "COPY {schema_name}.{table_name} ({formatted_columns}) FROM STDIN CSV {header} {delimiter} {escape} {quote} {encoding}" + "COPY {relation_name} ({formatted_columns}) FROM STDIN CSV {header} {delimiter} {escape} {quote} {encoding}" ).format( - schema_name=sql.Identifier(schema_name), - table_name=sql.Identifier(table_name), + relation_name=sql.Identifier(schema_name, table_name), formatted_columns=formatted_columns, header=sql.SQL("HEADER" if header else ""), delimiter=sql.SQL(f"DELIMITER E'{delimiter}'" if delimiter else ""), diff --git a/db/tables/operations/select.py b/db/tables/operations/select.py index 9008b25030..e25ad3c976 100644 --- a/db/tables/operations/select.py +++ b/db/tables/operations/select.py @@ -59,16 +59,6 @@ def get_table_info(schema, conn): return exec_msar_func(conn, 'get_table_info', schema).fetchone()[0] -def get_fully_qualified_relation_name(table_oid, conn): - """ - Return a fully qualified table name. - - Args: - table_oid: The table oid for which we want fully qualified name. - """ - return exec_msar_func(conn, 'get_fully_qualified_relation_name', table_oid).fetchone()[0] - - def reflect_table(name, schema, engine, metadata, connection_to_use=None, keep_existing=False): extend_existing = not keep_existing autoload_with = engine if connection_to_use is None else connection_to_use From bf66a48c9a472b5381a5f42e264aae96dcbdcec8 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 19 Jun 2024 22:36:56 -0400 Subject: [PATCH 0291/1141] Improve function naming --- db/sql/00_msar.sql | 47 ++++++++++++++++++++++++---------------- db/sql/20_msar_views.sql | 2 +- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index aad8e27e2d..f2c6ee6d4e 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -136,7 +136,7 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -__msar.get_relation_name(rel_id oid) RETURNS text AS $$/* +__msar.get_qualified_relation_name(rel_id oid) RETURNS text AS $$/* Return the name for a given relation (e.g., table), qualified or quoted as appropriate. In cases where the relation is already included in the search path, the returned name will not be @@ -154,7 +154,7 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -__msar.get_relation_name_or_null(rel_id oid) RETURNS text AS $$/* +__msar.get_qualified_relation_name_or_null(rel_id oid) RETURNS text AS $$/* Return the name for a given relation (e.g., table), qualified or quoted as appropriate. In cases where the relation is already included in the search path, the returned name will not be @@ -1003,7 +1003,7 @@ Args: */ BEGIN RETURN __msar.rename_table( - __msar.get_relation_name_or_null(tab_id), + __msar.get_qualified_relation_name_or_null(tab_id), quote_ident(new_tab_name) ); END; @@ -1053,7 +1053,10 @@ Args: tab_id: The OID of the table whose comment we will change. comment_: The new comment. */ -SELECT __msar.comment_on_table(__msar.get_relation_name_or_null(tab_id), quote_literal(comment_)); +SELECT __msar.comment_on_table( + __msar.get_qualified_relation_name_or_null(tab_id), + quote_literal(comment_) +); $$ LANGUAGE SQL; @@ -1100,7 +1103,7 @@ BEGIN PERFORM msar.rename_table(tab_id, new_tab_name); PERFORM msar.comment_on_table(tab_id, comment); PERFORM msar.alter_columns(tab_id, col_alters); - RETURN __msar.get_relation_name_or_null(tab_id); + RETURN __msar.get_qualified_relation_name_or_null(tab_id); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1147,7 +1150,7 @@ Args: DECLARE tab_name text; DECLARE col_name text; BEGIN - tab_name := __msar.get_relation_name(tab_id); + tab_name := __msar.get_qualified_relation_name(tab_id); col_name := quote_ident(msar.get_column_name(tab_id, col_id)); RETURN __msar.update_pk_sequence_to_latest(tab_name, col_name); END; @@ -1205,7 +1208,10 @@ BEGIN FROM pg_catalog.pg_attribute WHERE attrelid=tab_id AND NOT attisdropped AND ARRAY[attnum::integer] <@ col_ids INTO col_names; - PERFORM __msar.drop_columns(__msar.get_relation_name_or_null(tab_id), variadic col_names); + PERFORM __msar.drop_columns( + __msar.get_qualified_relation_name_or_null(tab_id), + variadic col_names + ); RETURN array_length(col_names, 1); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1695,7 +1701,7 @@ Args: */ DECLARE col_create_defs __msar.col_def[]; - fq_table_name text := __msar.get_relation_name(tab_id); + fq_table_name text := __msar.get_qualified_relation_name(tab_id); BEGIN col_create_defs := __msar.process_col_def_jsonb(tab_id, col_defs, raw_default); PERFORM __msar.add_columns(fq_table_name, variadic col_create_defs); @@ -1880,7 +1886,7 @@ SELECT array_agg( con_create_obj ->> 'deferrable', -- Build the relation name where the constraint will be applied. Prefer numeric ID. COALESCE( - __msar.get_relation_name((con_create_obj -> 'fkey_relation_id')::integer::oid), + __msar.get_qualified_relation_name((con_create_obj -> 'fkey_relation_id')::integer::oid), msar.build_qualified_name_sql( con_create_obj ->> 'fkey_relation_schema', con_create_obj ->> 'fkey_relation_name' ) @@ -1936,7 +1942,10 @@ DECLARE con_create_defs __msar.con_def[]; BEGIN con_create_defs := __msar.process_con_def_jsonb(tab_id, con_defs); - PERFORM __msar.add_constraints(__msar.get_relation_name(tab_id), variadic con_create_defs); + PERFORM __msar.add_constraints( + __msar.get_qualified_relation_name(tab_id), + variadic con_create_defs + ); RETURN array_agg(oid) FROM pg_constraint WHERE conrelid=tab_id; END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -2027,7 +2036,7 @@ BEGIN col_defs := __msar.get_duplicate_col_defs( tab_id, ARRAY[col_id], ARRAY[copy_name], copy_data ); - tab_name := __msar.get_relation_name(tab_id); + tab_name := __msar.get_qualified_relation_name(tab_id); col_name := quote_ident(msar.get_column_name(tab_id, col_id)); PERFORM __msar.add_columns(tab_name, VARIADIC col_defs); created_col_id := attnum @@ -2131,7 +2140,7 @@ Args: */ DECLARE relation_name text; BEGIN - relation_name := __msar.get_relation_name_or_null(tab_id); + relation_name := __msar.get_qualified_relation_name_or_null(tab_id); -- if_exists doesn't work while working with oids because -- the SQL query gets parameterized with tab_id instead of relation_name -- since we're unable to find the relation_name for a non existing table. @@ -2214,7 +2223,7 @@ Args: */ BEGIN RETURN __msar.drop_constraint( - __msar.get_relation_name(tab_id), + __msar.get_qualified_relation_name(tab_id), quote_ident(msar.get_constraint_name(con_id)) ); END; @@ -2329,7 +2338,7 @@ Args: */ BEGIN PERFORM __msar.rename_column( - tab_name => __msar.get_relation_name(tab_id), + tab_name => __msar.get_qualified_relation_name(tab_id), old_col_name => quote_ident(msar.get_column_name(tab_id, col_id)), new_col_name => quote_ident(new_col_name) ); @@ -2587,7 +2596,7 @@ BEGIN IF col_alter_str IS NOT NULL THEN PERFORM __msar.exec_ddl( 'ALTER TABLE %s %s', - __msar.get_relation_name(tab_id), + __msar.get_qualified_relation_name(tab_id), msar.process_col_alter_jsonb(tab_id, col_alters) ); END IF; @@ -2684,7 +2693,7 @@ Args: comment_: The new comment. */ SELECT __msar.comment_on_column( - __msar.get_relation_name(tab_id), + __msar.get_qualified_relation_name(tab_id), quote_ident(msar.get_column_name(tab_id, col_id)), comment_ ); @@ -2845,7 +2854,7 @@ BEGIN new_tab_name, extracted_col_defs, extracted_con_defs, - format('Extracted from %s', __msar.get_relation_name(tab_id)) + format('Extracted from %s', __msar.get_qualified_relation_name(tab_id)) ); -- Create a new fkey column and foreign key linking the original table to the extracted one. fkey_attnum := msar.create_many_to_one_link(extracted_table_id, tab_id, fkey_name); @@ -2866,9 +2875,9 @@ BEGIN -- %1$s This is a comma separated string of the extracted column names string_agg(quote_ident(col_def ->> 'name'), ', '), -- %2$s This is the name of the original (remainder) table - __msar.get_relation_name(tab_id), + __msar.get_qualified_relation_name(tab_id), -- %3$s This is the new extracted table name - __msar.get_relation_name(extracted_table_id), + __msar.get_qualified_relation_name(extracted_table_id), -- %4$I This is the name of the fkey column in the remainder table. fkey_name ) FROM jsonb_array_elements(extracted_col_defs) AS col_def; diff --git a/db/sql/20_msar_views.sql b/db/sql/20_msar_views.sql index cccb08b737..48ad01ca8c 100644 --- a/db/sql/20_msar_views.sql +++ b/db/sql/20_msar_views.sql @@ -56,7 +56,7 @@ BEGIN INTO view_cols; RETURN __msar.exec_ddl( 'CREATE OR REPLACE VIEW %s AS SELECT %s FROM %s', - view_name, view_cols, __msar.get_relation_name(tab_id) + view_name, view_cols, __msar.get_qualified_relation_name(tab_id) ); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; From d3f7c82f15fd0a5e8992217ddd0a5ecfcac717ca Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 19 Jun 2024 22:53:27 -0400 Subject: [PATCH 0292/1141] Fix missing quotes in error message --- mathesar/api/exceptions/database_exceptions/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/api/exceptions/database_exceptions/exceptions.py b/mathesar/api/exceptions/database_exceptions/exceptions.py index f9e7d3c089..07e2039cf6 100644 --- a/mathesar/api/exceptions/database_exceptions/exceptions.py +++ b/mathesar/api/exceptions/database_exceptions/exceptions.py @@ -159,7 +159,7 @@ def __init__( @staticmethod def err_msg(exception): if type(exception) is InvalidTypeError and exception.column_name and exception.new_type: - return f'{exception.column_name} cannot be cast to {exception.new_type}.' + return f'"{exception.column_name}" cannot be cast to {exception.new_type}.' return 'Invalid type cast requested.' From bbfb54d95c67e3ef591bc9bca59e41ca08567adb Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 20 Jun 2024 16:20:50 +0800 Subject: [PATCH 0293/1141] add column metadata model --- .../migrations/0008_add_metadata_models.py | 46 +++++++++++++++++++ mathesar/models/base.py | 41 +++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 mathesar/migrations/0008_add_metadata_models.py diff --git a/mathesar/migrations/0008_add_metadata_models.py b/mathesar/migrations/0008_add_metadata_models.py new file mode 100644 index 0000000000..13dd921ad3 --- /dev/null +++ b/mathesar/migrations/0008_add_metadata_models.py @@ -0,0 +1,46 @@ +# Generated by Django 4.2.11 on 2024-06-20 08:02 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('mathesar', '0007_users_permissions_remodel'), + ] + + operations = [ + migrations.CreateModel( + name='ColumnMetaData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('table_oid', models.PositiveIntegerField()), + ('attnum', models.PositiveIntegerField()), + ('bool_input', models.CharField(blank=True, choices=[('dropdown', 'dropdown'), ('checkbox', 'checkbox')])), + ('bool_true', models.CharField(default='True')), + ('bool_false', models.CharField(default='False')), + ('num_min_frac_digits', models.PositiveIntegerField(blank=True)), + ('num_max_frac_digits', models.PositiveIntegerField(blank=True)), + ('num_show_as_perc', models.BooleanField(default=False)), + ('mon_currency_symbol', models.CharField(default='$')), + ('mon_currency_location', models.CharField(choices=[('after-minus', 'after-minus'), ('end-with-space', 'end-with-space')], default='after-minus')), + ('time_format', models.CharField(blank=True)), + ('date_format', models.CharField(blank=True)), + ('duration_min', models.CharField(blank=True, max_length=255)), + ('duration_max', models.CharField(blank=True, max_length=255)), + ('duration_show_units', models.BooleanField(default=True)), + ('database', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mathesar.database')), + ], + ), + migrations.AddConstraint( + model_name='columnmetadata', + constraint=models.UniqueConstraint(fields=('database', 'table_oid', 'attnum'), name='unique_column_metadata'), + ), + migrations.AddConstraint( + model_name='columnmetadata', + constraint=models.CheckConstraint(check=models.Q(('num_max_frac_digits__lte', 20), ('num_min_frac_digits__lte', 20), ('num_min_frac_digits__lte', models.F('num_max_frac_digits'))), name='frac_digits_integrity'), + ), + ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index de7454dd6d..43b9e1607e 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -80,3 +80,44 @@ def connection(self): user=self.role.name, password=self.role.password, ) + + +class ColumnMetaData(BaseModel): + database = models.ForeignKey('Database', on_delete=models.CASCADE) + table_oid = models.PositiveIntegerField() + attnum = models.PositiveIntegerField() + bool_input = models.CharField( + choices=[("dropdown", "dropdown"), ("checkbox", "checkbox")], + blank=True + ) + bool_true = models.CharField(default='True') + bool_false = models.CharField(default='False') + num_min_frac_digits = models.PositiveIntegerField(blank=True) + num_max_frac_digits = models.PositiveIntegerField(blank=True) + num_show_as_perc = models.BooleanField(default=False) + mon_currency_symbol = models.CharField(default="$") + mon_currency_location = models.CharField( + choices=[("after-minus", "after-minus"), ("end-with-space", "end-with-space")], + default="after-minus" + ) + time_format = models.CharField(blank=True) + date_format = models.CharField(blank=True) + duration_min = models.CharField(max_length=255, blank=True) + duration_max = models.CharField(max_length=255, blank=True) + duration_show_units = models.BooleanField(default=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["database", "table_oid", "attnum"], + name="unique_column_metadata" + ), + models.CheckConstraint( + check=( + models.Q(num_max_frac_digits__lte=20) + & models.Q(num_min_frac_digits__lte=20) + & models.Q(num_min_frac_digits__lte=models.F("num_max_frac_digits")) + ), + name="frac_digits_integrity" + ) + ] From 162debe0da7ed7a23ac358e6b949d19f5c9432ca Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 20 Jun 2024 16:51:36 +0800 Subject: [PATCH 0294/1141] move column rpc module to make room for metadata --- mathesar/rpc/columns/__init__.py | 1 + mathesar/rpc/{columns.py => columns/base.py} | 0 mathesar/tests/rpc/test_columns.py | 18 +++++++++--------- 3 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 mathesar/rpc/columns/__init__.py rename mathesar/rpc/{columns.py => columns/base.py} (100%) diff --git a/mathesar/rpc/columns/__init__.py b/mathesar/rpc/columns/__init__.py new file mode 100644 index 0000000000..3cd5390e65 --- /dev/null +++ b/mathesar/rpc/columns/__init__.py @@ -0,0 +1 @@ +from mathesar.rpc.columns.base import * diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns/base.py similarity index 100% rename from mathesar/rpc/columns.py rename to mathesar/rpc/columns/base.py diff --git a/mathesar/tests/rpc/test_columns.py b/mathesar/tests/rpc/test_columns.py index 1afd813a33..3069f04d13 100644 --- a/mathesar/tests/rpc/test_columns.py +++ b/mathesar/tests/rpc/test_columns.py @@ -84,9 +84,9 @@ def mock_display_options(_database_id, _table_oid, attnums, user): 'minimum_fraction_digits': 2 } ] - monkeypatch.setattr(columns, 'connect', mock_connect) - monkeypatch.setattr(columns, 'get_column_info_for_table', mock_column_info) - monkeypatch.setattr(columns, 'get_raw_display_options', mock_display_options) + monkeypatch.setattr(columns.base, 'connect', mock_connect) + monkeypatch.setattr(columns.base, 'get_column_info_for_table', mock_column_info) + monkeypatch.setattr(columns.base, 'get_raw_display_options', mock_display_options) expect_col_list = { 'column_info': ( { @@ -160,8 +160,8 @@ def mock_column_alter(_table_oid, _column_data_list, conn): raise AssertionError('incorrect parameters passed') return 1 - monkeypatch.setattr(columns, 'connect', mock_connect) - monkeypatch.setattr(columns, 'alter_columns_in_table', mock_column_alter) + monkeypatch.setattr(columns.base, 'connect', mock_connect) + monkeypatch.setattr(columns.base, 'alter_columns_in_table', mock_column_alter) actual_result = columns.patch( column_data_list=column_data_list, table_oid=table_oid, @@ -193,8 +193,8 @@ def mock_column_create(_table_oid, _column_data_list, conn): raise AssertionError('incorrect parameters passed') return [3, 4] - monkeypatch.setattr(columns, 'connect', mock_connect) - monkeypatch.setattr(columns, 'add_columns_to_table', mock_column_create) + monkeypatch.setattr(columns.base, 'connect', mock_connect) + monkeypatch.setattr(columns.base, 'add_columns_to_table', mock_column_create) actual_result = columns.add( column_data_list=column_data_list, table_oid=table_oid, @@ -226,8 +226,8 @@ def mock_column_drop(_table_oid, _column_attnums, conn): raise AssertionError('incorrect parameters passed') return 3 - monkeypatch.setattr(columns, 'connect', mock_connect) - monkeypatch.setattr(columns, 'drop_columns_from_table', mock_column_drop) + monkeypatch.setattr(columns.base, 'connect', mock_connect) + monkeypatch.setattr(columns.base, 'drop_columns_from_table', mock_column_drop) actual_result = columns.delete( column_attnums=column_attnums, table_oid=table_oid, From 910ccf226077781476696604ccc00361b06e0dd6 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 20 Jun 2024 16:55:46 +0800 Subject: [PATCH 0295/1141] wire up metadata list function --- config/settings/common_settings.py | 1 + mathesar/rpc/columns/metadata.py | 60 ++++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 5 +++ mathesar/utils/columns.py | 5 +++ 4 files changed, 71 insertions(+) create mode 100644 mathesar/rpc/columns/metadata.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index 7697ed053c..c37d6d23ca 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -67,6 +67,7 @@ def pipe_delim(pipe_string): MODERNRPC_METHODS_MODULES = [ 'mathesar.rpc.connections', 'mathesar.rpc.columns', + 'mathesar.rpc.columns.metadata', 'mathesar.rpc.schemas', 'mathesar.rpc.tables' ] diff --git a/mathesar/rpc/columns/metadata.py b/mathesar/rpc/columns/metadata.py new file mode 100644 index 0000000000..c3f3169d49 --- /dev/null +++ b/mathesar/rpc/columns/metadata.py @@ -0,0 +1,60 @@ +""" +Classes and functions exposed to the RPC endpoint for managing column metadata. +""" +from typing import Literal, Optional, TypedDict + +from modernrpc.core import rpc_method, REQUEST_KEY +from modernrpc.auth.basic import http_basic_auth_login_required + +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions +from mathesar.utils.columns import get_columns_meta_data + + +class ColumnMetaData(TypedDict): + database: int + table_oid: int + attnum: int + bool_input: Optional[Literal["dropdown", "checkbox"]] + bool_true: Optional[str] + bool_false: Optional[str] + num_min_frac_digits: Optional[int] + num_max_frac_digits: Optional[int] + num_show_as_perc: Optional[bool] + mon_currency_symbol: Optional[str] + mon_currency_location: Optional[Literal["after-minus", "end-with-space"]] + time_format: Optional[str] + date_format: Optional[str] + duration_min: Optional[str] + duration_max: Optional[str] + duration_show_units: Optional[bool] + + @classmethod + def from_db_model(cls, db_model): + return cls( + database=db_model.database.id, + table_oid=db_model.table_oid, + attnum=db_model.attnum, + bool_input=db_model.bool_input, + bool_true=db_model.bool_true, + bool_false=db_model.bool_false, + num_min_frac_digits=db_model.num_min_frac_digits, + num_max_frac_digits=db_model.num_max_frac_digits, + num_show_as_perc=db_model.num_show_as_perc, + mon_currency_symbol=db_model.mon_currency_symbol, + mon_currency_location=db_model.mon_currency_location, + time_format=db_model.time_format, + date_format=db_model.date_format, + duration_min=db_model.duration_min, + duration_max=db_model.duration_max, + duration_show_units=db_model.duration_show_units, + ) + + +@rpc_method(name="columns.metadata.list") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_(*, table_oid: int, database_id: int, **kwargs) -> list[ColumnMetaData]: + columns_meta_data = get_columns_meta_data(table_oid, database_id) + return [ + ColumnMetaData.from_db_model(db_model) for db_model in columns_meta_data + ] diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 35a7cd75e1..6f940c272c 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -34,6 +34,11 @@ "columns.add", [user_is_authenticated] ), + ( + columns.metadata.list_, + "columns.metadata.list", + [user_is_authenticated] + ), ( connections.add_from_known_connection, "connections.add_from_known_connection", diff --git a/mathesar/utils/columns.py b/mathesar/utils/columns.py index 97075a7d52..2429ad59e0 100644 --- a/mathesar/utils/columns.py +++ b/mathesar/utils/columns.py @@ -1,4 +1,5 @@ from mathesar.models.deprecated import Column +from mathesar.models.base import ColumnMetaData # This should be replaced once we have the ColumnMetadata model sorted out. @@ -14,3 +15,7 @@ def get_raw_display_options(database_id, table_oid, attnums, user): ) if c.display_options is not None ] + + +def get_columns_meta_data(table_oid, database_id): + return ColumnMetaData.filter(database__id=database_id, table_oid=table_oid) From 2fedad29d2ad846328e68ae4366080366523ef84 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 20 Jun 2024 17:31:27 +0800 Subject: [PATCH 0296/1141] set up metadata module for documentation --- mathesar/rpc/columns/__init__.py | 2 +- mathesar/rpc/columns/metadata.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mathesar/rpc/columns/__init__.py b/mathesar/rpc/columns/__init__.py index 3cd5390e65..4b40b38c84 100644 --- a/mathesar/rpc/columns/__init__.py +++ b/mathesar/rpc/columns/__init__.py @@ -1 +1 @@ -from mathesar.rpc.columns.base import * +from .base import * # noqa diff --git a/mathesar/rpc/columns/metadata.py b/mathesar/rpc/columns/metadata.py index c3f3169d49..60ac733fa7 100644 --- a/mathesar/rpc/columns/metadata.py +++ b/mathesar/rpc/columns/metadata.py @@ -3,7 +3,7 @@ """ from typing import Literal, Optional, TypedDict -from modernrpc.core import rpc_method, REQUEST_KEY +from modernrpc.core import rpc_method from modernrpc.auth.basic import http_basic_auth_login_required from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -54,6 +54,16 @@ def from_db_model(cls, db_model): @http_basic_auth_login_required @handle_rpc_exceptions def list_(*, table_oid: int, database_id: int, **kwargs) -> list[ColumnMetaData]: + """ + List metadata associated with columns for a table. Exposed as `list`. + + Args: + table_oid: Identity of the table in the user's database. + database_id: The Django id of the database containing the table. + + Returns: + A list of column meta data objects. + """ columns_meta_data = get_columns_meta_data(table_oid, database_id) return [ ColumnMetaData.from_db_model(db_model) for db_model in columns_meta_data From 0d5bee1e889e1eb76ce3e022c852282555111081 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 20 Jun 2024 17:31:56 +0800 Subject: [PATCH 0297/1141] add metadata module into documentation --- docs/docs/api/rpc.md | 15 +++++++++++---- docs/mkdocs.yml | 3 +-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index b149a9045d..8f2bfc6846 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -39,7 +39,7 @@ To use an RPC function: --- -::: mathesar.rpc.connections +::: connections options: members: - add_from_known_connection @@ -49,7 +49,7 @@ To use an RPC function: --- -::: mathesar.rpc.schemas +::: schemas options: members: - list_ @@ -59,7 +59,7 @@ To use an RPC function: --- -::: mathesar.rpc.tables +::: tables options: members: - list_ @@ -72,7 +72,7 @@ To use an RPC function: --- -::: mathesar.rpc.columns +::: columns options: members: - list_ @@ -84,6 +84,13 @@ To use an RPC function: - SettableColumnInfo - TypeOptions - ColumnDefault + +--- + +::: columns.metadata + options: + members: + - list_ ## Responses diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 58875327c4..0db1e9c1e8 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -57,12 +57,11 @@ plugins: - mkdocstrings: handlers: python: - paths: [..] + paths: [../mathesar/rpc/] options: docstring_style: google separate_signature: true show_root_heading: true - show_root_full_path: false show_source: false show_symbol_type_heading: true group_by_category: false From 03f3b07add3112cd33fd1f8a68f155e9355565bc Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 20 Jun 2024 17:54:06 +0800 Subject: [PATCH 0298/1141] improve docs formatting for RPC functions --- docs/docs/api/rpc.md | 11 ++++++----- docs/mkdocs.yml | 5 +++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 8f2bfc6846..f2f4a0b214 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -37,7 +37,7 @@ To use an RPC function: } ``` ---- +## Connections ::: connections options: @@ -47,7 +47,7 @@ To use an RPC function: - grant_access_to_user - DBModelReturn ---- +## Schemas ::: schemas options: @@ -57,7 +57,7 @@ To use an RPC function: - delete - SchemaInfo ---- +## Tables ::: tables options: @@ -70,7 +70,7 @@ To use an RPC function: - TableInfo - SettableTableInfo ---- +## Columns ::: columns options: @@ -85,12 +85,13 @@ To use an RPC function: - TypeOptions - ColumnDefault ---- +## Column Metadata ::: columns.metadata options: members: - list_ + - ColumnMetaData ## Responses diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 0db1e9c1e8..e2488e3424 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -59,11 +59,12 @@ plugins: python: paths: [../mathesar/rpc/] options: + heading_level: 3 docstring_style: google separate_signature: true - show_root_heading: true + show_root_toc_entry: false + show_root_members_full_path: true show_source: false - show_symbol_type_heading: true group_by_category: false theme: From 2a9eddc3b75b8164ac8f9c593ffe2713a31812b9 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 20 Jun 2024 17:54:39 +0800 Subject: [PATCH 0299/1141] add documentation for metadata object, fix attribute name --- mathesar/rpc/columns/metadata.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/mathesar/rpc/columns/metadata.py b/mathesar/rpc/columns/metadata.py index 60ac733fa7..e8268c7faf 100644 --- a/mathesar/rpc/columns/metadata.py +++ b/mathesar/rpc/columns/metadata.py @@ -11,7 +11,34 @@ class ColumnMetaData(TypedDict): - database: int + """ + Metadata for a column in a table. + + Only the + - database, + - table_oid, and + - attnum + keys are required. + + Attributes: + database_id: The Django id of the database containing the table. + table_oid: The OID of the table containing the column + attnum: The attnum of the column in the table. + bool_input: How the input for a boolean column should be shown. + bool_true: A string to display for `true` values. + bool_false: A string to display for `false` values. + num_min_frac_digits: Minimum digits shown after the decimal point. + num_max_frac_digits: Maximum digits shown after the decimal point. + num_show_as_perc: Whether to show a numeric value as a percentage. + mon_currency_symbol: The currency symbol shown for money value. + mon_currency_location: Where the currency symbol should be shown. + time_format: A string representing the format of time values. + date_format: A string representing the format of date values. + duration_min: Optional[str] + duration_max: Optional[str] + duration_show_units: Optional[bool] + """ + database_id: int table_oid: int attnum: int bool_input: Optional[Literal["dropdown", "checkbox"]] @@ -31,7 +58,7 @@ class ColumnMetaData(TypedDict): @classmethod def from_db_model(cls, db_model): return cls( - database=db_model.database.id, + database_id=db_model.database.id, table_oid=db_model.table_oid, attnum=db_model.attnum, bool_input=db_model.bool_input, From 1c1b55a287fdae83fc1856d9ab48201cecc848f2 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Thu, 20 Jun 2024 13:14:15 -0400 Subject: [PATCH 0300/1141] Fix typos in code comments --- db/sql/00_msar.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index f242b84797..948dacfbf2 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -869,7 +869,7 @@ CREATE OR REPLACE FUNCTION msar.set_schema_description( Set the PostgreSQL description (aka COMMENT) of a schema. Descriptions are removed by passing an empty string. Passing a NULL description will cause -this function to return NULL withou doing anything. +this function to return NULL without doing anything. Args: sch_id: The OID of the schema. @@ -902,7 +902,7 @@ CREATE OR REPLACE FUNCTION msar.patch_schema(sch_name text, patch jsonb) RETURNS Modify a schema according to the given patch. Args: - sch_id: The name of the schema. + sch_name: The name of the schema, UNQUOTED patch: A JSONB object as specified by msar.patch_schema(sch_id oid, patch jsonb) */ BEGIN From d55ce1b8ab63f58c25285110fbc047035fe09ff0 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 21 Jun 2024 16:59:25 +0800 Subject: [PATCH 0301/1141] complete documentation of column metadata object --- mathesar/rpc/columns/metadata.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/mathesar/rpc/columns/metadata.py b/mathesar/rpc/columns/metadata.py index e8268c7faf..dd867608a5 100644 --- a/mathesar/rpc/columns/metadata.py +++ b/mathesar/rpc/columns/metadata.py @@ -14,11 +14,7 @@ class ColumnMetaData(TypedDict): """ Metadata for a column in a table. - Only the - - database, - - table_oid, and - - attnum - keys are required. + Only the `database`, `table_oid`, and `attnum` keys are required. Attributes: database_id: The Django id of the database containing the table. @@ -34,9 +30,9 @@ class ColumnMetaData(TypedDict): mon_currency_location: Where the currency symbol should be shown. time_format: A string representing the format of time values. date_format: A string representing the format of date values. - duration_min: Optional[str] - duration_max: Optional[str] - duration_show_units: Optional[bool] + duration_min: The smallest unit for displaying durations. + duration_max: The largest unit for displaying durations. + duration_show_units: Whether to show the units for durations. """ database_id: int table_oid: int From e3daa059e04d857c61adcd02955ae99e88529824 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 21 Jun 2024 23:10:29 +0530 Subject: [PATCH 0302/1141] implement sql and python functions for getting import preview --- db/sql/00_msar.sql | 50 +++++++++++++++++++++++++++- db/tables/operations/import_.py | 10 ++++++ mathesar/rpc/columns.py | 6 ++++ mathesar/rpc/tables.py | 20 +++++++++-- mathesar/tests/rpc/test_endpoints.py | 5 +++ 5 files changed, 88 insertions(+), 3 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 2351a6e17e..2ccf6d68ee 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -105,6 +105,18 @@ __msar.build_text_tuple(text[]) RETURNS text AS $$ SELECT '(' || string_agg(col, ', ') || ')' FROM unnest($1) x(col); $$ LANGUAGE sql RETURNS NULL ON NULL INPUT; + +CREATE OR REPLACE FUNCTION +__msar.exec_dql(command text) RETURNS jsonb AS $$ +DECLARE + records jsonb; +BEGIN + EXECUTE 'WITH cte AS (' || command || ') + SELECT jsonb_agg(row_to_json(cte.*)) FROM cte' INTO records; + RETURN records; +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -- INFO FUNCTIONS @@ -1301,7 +1313,7 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; DROP TYPE IF EXISTS __msar.col_def CASCADE; CREATE TYPE __msar.col_def AS ( name_ text, -- The name of the column to create, quoted. - type_ text, -- The type of the column to create, fully specced with arguments. + type_ jsonb, -- The type of the column to create, fully specced with arguments. not_null boolean, -- A boolean to describe whether the column is nullable or not. default_ text, -- Text SQL giving the default value for the column. identity_ boolean, -- A boolean giving whether the column is an identity pkey column. @@ -2392,6 +2404,42 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION +msar.get_preview( + tab_id oid, + col_cast_def jsonb, + rec_limit integer +) RETURNS jsonb AS $$ +DECLARE + tab_name text; + sel_query text; + result jsonb; +BEGIN + tab_name := __msar.get_relation_name(tab_id); + WITH preview_cte AS ( + SELECT string_agg( + 'CAST(' || + __msar.build_cast_expr(msar.get_column_name(tab_id, (col_cast ->> 'attnum')::integer), col_cast -> 'type' ->> 'name') || + ' AS ' || + msar.build_type_text(col_cast -> 'type') || + ')'|| ' AS ' || msar.get_column_name(tab_id, (col_cast ->> 'attnum')::integer), + ', ' + ) AS cast_expr + FROM jsonb_array_elements(col_cast_def) AS col_cast + WHERE NOT msar.is_mathesar_id_column(tab_id, (col_cast ->> 'attnum')::integer) + ) + SELECT + CASE WHEN rec_limit IS NOT NULL THEN + format('SELECT id, %s FROM %s LIMIT %s', cast_expr, tab_name, rec_limit) + ELSE + format('SELECT id, %s FROM %s', cast_expr, tab_name) + END + INTO sel_query FROM preview_cte; + RETURN __msar.exec_dql(sel_query); +END; +$$ LANGUAGE plpgsql; + + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -- COLUMN ALTERATION FUNCTIONS diff --git a/db/tables/operations/import_.py b/db/tables/operations/import_.py index a22ec9121c..39ec1a2d77 100644 --- a/db/tables/operations/import_.py +++ b/db/tables/operations/import_.py @@ -1,6 +1,11 @@ +import json import tempfile + import clevercsv as csv from psycopg import sql + +from db.connection import exec_msar_func +from db.columns.operations.alter import _transform_column_alter_dict from db.tables.operations.create import prepare_table_for_import from db.encoding_utils import get_sql_compatible_encoding from mathesar.models.deprecated import DataFile @@ -89,3 +94,8 @@ def insert_csv_records( with cursor.copy(copy_sql) as copy: while data := temp_file.read(): copy.write(data) + + +def get_preview(table_oid, column_list, conn, limit=20): + transformed_column_data = [_transform_column_alter_dict(col) for col in column_list] + return exec_msar_func(conn, 'get_preview', table_oid, json.dumps(transformed_column_data), limit).fetchone()[0] diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns.py index 6f198d8553..84890af53c 100644 --- a/mathesar/rpc/columns.py +++ b/mathesar/rpc/columns.py @@ -131,6 +131,12 @@ class SettableColumnInfo(TypedDict): description: Optional[str] +class PreviewableColumnInfo(TypedDict): + id: int + type: Optional[str] + type_options: Optional[TypeOptions] + + class ColumnInfo(TypedDict): """ Information about a column. Extends the settable fields. diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 1516e12ad6..ae692b8c56 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -7,8 +7,8 @@ from db.tables.operations.drop import drop_table_from_database from db.tables.operations.create import create_table_on_database from db.tables.operations.alter import alter_table_on_database -from db.tables.operations.import_ import import_csv -from mathesar.rpc.columns import CreatableColumnInfo, SettableColumnInfo +from db.tables.operations.import_ import import_csv, get_preview +from mathesar.rpc.columns import CreatableColumnInfo, SettableColumnInfo, PreviewableColumnInfo from mathesar.rpc.constraints import CreatableConstraintInfo from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -200,3 +200,19 @@ def import_( user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: return import_csv(data_file_id, table_name, schema_oid, conn, comment) + + +@rpc_method(name="tables.get_import_preview") +@http_basic_auth_login_required +@handle_rpc_exceptions +def get_import_preview( + *, + table_oid: int, + columns: PreviewableColumnInfo, + database_id: int, + limit: int = 20, + **kwargs +): + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + return get_preview(table_oid, columns, conn, limit) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index ae18825f41..288aa6257c 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -93,6 +93,11 @@ tables.import_, "tables.import", [user_is_authenticated] + ), + ( + tables.get_import_preview, + "tables.get_import_preview", + [user_is_authenticated] ) ] From 21ea0dd69b3d813e2da1dc0bc668382e5738edd0 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 21 Jun 2024 23:23:24 +0530 Subject: [PATCH 0303/1141] revert col_def --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 2ccf6d68ee..ca4799a570 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1313,7 +1313,7 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; DROP TYPE IF EXISTS __msar.col_def CASCADE; CREATE TYPE __msar.col_def AS ( name_ text, -- The name of the column to create, quoted. - type_ jsonb, -- The type of the column to create, fully specced with arguments. + type_ text, -- The type of the column to create, fully specced with arguments. not_null boolean, -- A boolean to describe whether the column is nullable or not. default_ text, -- Text SQL giving the default value for the column. identity_ boolean, -- A boolean giving whether the column is an identity pkey column. From d9371db00047c414b3b5263562b70bc177fb4608 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 21 Jun 2024 23:30:09 +0530 Subject: [PATCH 0304/1141] update docs --- docs/docs/api/rpc.md | 5 ++++- mathesar/rpc/tables.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index d9a727120f..356cde63bc 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -68,6 +68,7 @@ To use an RPC function: - delete - patch - import_ + - get_import_preview - TableInfo - SettableTableInfo @@ -80,8 +81,10 @@ To use an RPC function: - add - patch - delete - - ColumnListReturn - ColumnInfo + - ColumnListReturn + - CreatableColumnInfo + - PreviewableColumnInfo - SettableColumnInfo - TypeOptions - ColumnDefault diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index ae692b8c56..40837bfded 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -212,7 +212,7 @@ def get_import_preview( database_id: int, limit: int = 20, **kwargs -): +) -> list[dict]: user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: return get_preview(table_oid, columns, conn, limit) From 9bd73959ad2cee0776b7ebaf94e570b39ba15682 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Sat, 22 Jun 2024 00:46:23 +0530 Subject: [PATCH 0305/1141] add generate a table name if not given --- db/sql/00_msar.sql | 15 ++++++++++++--- mathesar/rpc/tables.py | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 948dacfbf2..6db0bac0aa 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2300,23 +2300,32 @@ Add a table, with a default id column, returning the OID of the created table. Args: sch_oid: The OID of the schema where the table will be created. - tab_name: The unquoted name for the new table. + tab_name (optional): The unquoted name for the new table. col_defs (optional): The columns for the new table, in order. con_defs (optional): The constraints for the new table. comment_ (optional): The comment for the new table. -Note that even if col_defs is null, we will still create a table with a default 'id' column. Also, +Note that if tab_name is null, the table will be created with a name in the format 'Table '. +If col_defs is null, the table will still be created with a default 'id' column. Also, if an 'id' column is given in the input, it will be replaced with our default 'id' column. This is the behavior of the current python functions, so we're keeping it for now. In any case, the created table will always have our default 'id' column as its first column. */ DECLARE + schema_name text; + table_count integer; fq_table_name text; created_table_id oid; column_defs __msar.col_def[]; constraint_defs __msar.con_def[]; BEGIN - fq_table_name := format('%I.%I', msar.get_schema_name(sch_oid), tab_name); + schema_name := __msar.get_schema_name(sch_oid); + IF NULLIF(tab_name, '') IS NOT NULL THEN + fq_table_name := format('%s.%s', schema_name, quote_ident(tab_name)); + ELSE + SELECT COUNT(*) INTO table_count FROM pg_catalog.pg_class WHERE relkind = 'r' AND relnamespace = sch_oid; + fq_table_name := format('%s.%s', schema_name, quote_ident('Table ' || (table_count + 1))); + END IF; column_defs := msar.process_col_def_jsonb(0, col_defs, false, true); constraint_defs := msar.process_con_def_jsonb(0, con_defs); PERFORM __msar.add_table(fq_table_name, column_defs, constraint_defs); diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 0af725241d..3ba21c3b85 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -97,9 +97,9 @@ def get(*, table_oid: int, database_id: int, **kwargs) -> TableInfo: @handle_rpc_exceptions def add( *, - table_name: str, schema_oid: int, database_id: int, + table_name: str = None, column_data_list: list[CreatableColumnInfo] = [], constraint_data_list: list[CreatableConstraintInfo] = [], comment: str = None, @@ -109,9 +109,9 @@ def add( Add a table with a default id column. Args: - table_name: Name of the table to be created. schema_oid: Identity of the schema in the user's database. database_id: The Django id of the database containing the table. + table_name: Name of the table to be created. column_data_list: A list describing columns to be created for the new table, in order. constraint_data_list: A list describing constraints to be created for the new table. comment: The comment for the new table. From cb1ab23fbbfd3cf7c5363ba150c80c39ac903c74 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Sat, 22 Jun 2024 01:13:34 +0530 Subject: [PATCH 0306/1141] fix non existent function name --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 6db0bac0aa..b5ebbd9f4b 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2319,7 +2319,7 @@ DECLARE column_defs __msar.col_def[]; constraint_defs __msar.con_def[]; BEGIN - schema_name := __msar.get_schema_name(sch_oid); + schema_name := msar.get_schema_name(sch_oid); IF NULLIF(tab_name, '') IS NOT NULL THEN fq_table_name := format('%s.%s', schema_name, quote_ident(tab_name)); ELSE From 6dfad5d8da87f603eebbf9eeb7246b3f0734fc31 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Sat, 22 Jun 2024 01:20:34 +0530 Subject: [PATCH 0307/1141] add sql test --- db/sql/test_00_msar.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 9bce00d3cf..c7716225a8 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -1451,6 +1451,19 @@ END; $f$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION test_add_mathesar_table_noname() RETURNS SETOF TEXT AS $f$ +DECLARE + generated_name text := 'Table 1'; +BEGIN + PERFORM __setup_create_table(); + PERFORM msar.add_mathesar_table( + 'tab_create_schema'::regnamespace::oid, null, null, null, null + ); + RETURN NEXT has_table('tab_create_schema'::name, generated_name::name); +END; +$f$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION test_add_mathesar_table_columns() RETURNS SETOF TEXT AS $f$ DECLARE col_defs jsonb := $j$[ From 7560423b5332f68f69aad0abf565a07b6cf37ed4 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Sat, 22 Jun 2024 01:27:34 +0530 Subject: [PATCH 0308/1141] sch_oid -> sch_id --- db/sql/00_msar.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index b5ebbd9f4b..8126248805 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2294,12 +2294,12 @@ $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION -msar.add_mathesar_table(sch_oid oid, tab_name text, col_defs jsonb, con_defs jsonb, comment_ text) +msar.add_mathesar_table(sch_id oid, tab_name text, col_defs jsonb, con_defs jsonb, comment_ text) RETURNS oid AS $$/* Add a table, with a default id column, returning the OID of the created table. Args: - sch_oid: The OID of the schema where the table will be created. + sch_id: The OID of the schema where the table will be created. tab_name (optional): The unquoted name for the new table. col_defs (optional): The columns for the new table, in order. con_defs (optional): The constraints for the new table. @@ -2319,11 +2319,11 @@ DECLARE column_defs __msar.col_def[]; constraint_defs __msar.con_def[]; BEGIN - schema_name := msar.get_schema_name(sch_oid); + schema_name := msar.get_schema_name(sch_id); IF NULLIF(tab_name, '') IS NOT NULL THEN fq_table_name := format('%s.%s', schema_name, quote_ident(tab_name)); ELSE - SELECT COUNT(*) INTO table_count FROM pg_catalog.pg_class WHERE relkind = 'r' AND relnamespace = sch_oid; + SELECT COUNT(*) INTO table_count FROM pg_catalog.pg_class WHERE relkind = 'r' AND relnamespace = sch_id; fq_table_name := format('%s.%s', schema_name, quote_ident('Table ' || (table_count + 1))); END IF; column_defs := msar.process_col_def_jsonb(0, col_defs, false, true); From e614f677972be5074324c10a302836012fa29e35 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Sat, 22 Jun 2024 01:31:28 +0530 Subject: [PATCH 0309/1141] sch_oid -> sch_id --- db/sql/00_msar.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 2351a6e17e..74e372b827 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2352,7 +2352,7 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION msar.prepare_table_for_import( - sch_oid oid, + sch_id oid, tab_name text, col_defs jsonb, comment_ text @@ -2367,7 +2367,7 @@ Each returned JSON object will have the form: } Args: - sch_oid: The OID of the schema where the table will be created. + sch_id: The OID of the schema where the table will be created. tab_name: The unquoted name for the new table. col_defs: The columns for the new table, in order. comment_ (optional): The comment for the new table. @@ -2377,7 +2377,7 @@ DECLARE rel_name text; rel_id oid; BEGIN - rel_id := msar.add_mathesar_table(sch_oid, tab_name, col_defs, NULL, comment_); + rel_id := msar.add_mathesar_table(sch_id, tab_name, col_defs, NULL, comment_); SELECT nspname, relname INTO sch_name, rel_name FROM pg_catalog.pg_class AS pgc LEFT JOIN pg_catalog.pg_namespace AS pgn From 69fb14f2c077e403e724b699b78f76997003d08a Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Sat, 22 Jun 2024 01:45:40 +0530 Subject: [PATCH 0310/1141] use %I instead of %s --- db/sql/00_msar.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 8126248805..e853955f66 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2321,10 +2321,10 @@ DECLARE BEGIN schema_name := msar.get_schema_name(sch_id); IF NULLIF(tab_name, '') IS NOT NULL THEN - fq_table_name := format('%s.%s', schema_name, quote_ident(tab_name)); + fq_table_name := format('%I.%I', schema_name, quote_ident(tab_name)); ELSE SELECT COUNT(*) INTO table_count FROM pg_catalog.pg_class WHERE relkind = 'r' AND relnamespace = sch_id; - fq_table_name := format('%s.%s', schema_name, quote_ident('Table ' || (table_count + 1))); + fq_table_name := format('%I.%I', schema_name, quote_ident('Table ' || (table_count + 1))); END IF; column_defs := msar.process_col_def_jsonb(0, col_defs, false, true); constraint_defs := msar.process_con_def_jsonb(0, con_defs); From c839ef956f2e9968529b2d527561a04b9b7c8f9e Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Sat, 22 Jun 2024 01:47:05 +0530 Subject: [PATCH 0311/1141] quote_ident no longer required --- db/sql/00_msar.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index e853955f66..fa204738aa 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2321,10 +2321,10 @@ DECLARE BEGIN schema_name := msar.get_schema_name(sch_id); IF NULLIF(tab_name, '') IS NOT NULL THEN - fq_table_name := format('%I.%I', schema_name, quote_ident(tab_name)); + fq_table_name := format('%I.%I', schema_name, tab_name); ELSE SELECT COUNT(*) INTO table_count FROM pg_catalog.pg_class WHERE relkind = 'r' AND relnamespace = sch_id; - fq_table_name := format('%I.%I', schema_name, quote_ident('Table ' || (table_count + 1))); + fq_table_name := format('%I.%I', schema_name, 'Table ' || (table_count + 1)); END IF; column_defs := msar.process_col_def_jsonb(0, col_defs, false, true); constraint_defs := msar.process_con_def_jsonb(0, con_defs); From 38f5f763fea48197680f1e7216582f8f4c34c1d2 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Sat, 22 Jun 2024 02:20:44 +0530 Subject: [PATCH 0312/1141] improve formatting --- db/sql/00_msar.sql | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index ca4799a570..8e3e2d782f 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2419,7 +2419,9 @@ BEGIN WITH preview_cte AS ( SELECT string_agg( 'CAST(' || - __msar.build_cast_expr(msar.get_column_name(tab_id, (col_cast ->> 'attnum')::integer), col_cast -> 'type' ->> 'name') || + __msar.build_cast_expr( + msar.get_column_name(tab_id, (col_cast ->> 'attnum')::integer), col_cast -> 'type' ->> 'name' + ) || ' AS ' || msar.build_type_text(col_cast -> 'type') || ')'|| ' AS ' || msar.get_column_name(tab_id, (col_cast ->> 'attnum')::integer), @@ -2428,12 +2430,8 @@ BEGIN FROM jsonb_array_elements(col_cast_def) AS col_cast WHERE NOT msar.is_mathesar_id_column(tab_id, (col_cast ->> 'attnum')::integer) ) - SELECT - CASE WHEN rec_limit IS NOT NULL THEN - format('SELECT id, %s FROM %s LIMIT %s', cast_expr, tab_name, rec_limit) - ELSE - format('SELECT id, %s FROM %s', cast_expr, tab_name) - END + SELECT + format('SELECT id, %s FROM %s LIMIT %L', cast_expr, tab_name, rec_limit) INTO sel_query FROM preview_cte; RETURN __msar.exec_dql(sel_query); END; From b64752e75dfd6e72d711dbf56a75de19700d061d Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 24 Jun 2024 14:35:31 +0800 Subject: [PATCH 0313/1141] remove display options from column list return --- docs/docs/api/rpc.md | 1 - mathesar/rpc/columns/base.py | 32 +-------- mathesar/tests/rpc/test_columns.py | 101 ++++++++++------------------- 3 files changed, 38 insertions(+), 96 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index f2f4a0b214..c0460a0a9a 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -79,7 +79,6 @@ To use an RPC function: - add - patch - delete - - ColumnListReturn - ColumnInfo - SettableColumnInfo - TypeOptions diff --git a/mathesar/rpc/columns/base.py b/mathesar/rpc/columns/base.py index 6f198d8553..f7e53704c5 100644 --- a/mathesar/rpc/columns/base.py +++ b/mathesar/rpc/columns/base.py @@ -12,7 +12,6 @@ from db.columns.operations.select import get_column_info_for_table from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect -from mathesar.utils.columns import get_raw_display_options class TypeOptions(TypedDict, total=False): @@ -171,49 +170,24 @@ def from_dict(cls, col_info): ) -class ColumnListReturn(TypedDict): - """ - Information about the columns of a table. - - Attributes: - column_info: Column information from the user's database. - display_options: Display metadata managed by Mathesar. - """ - column_info: list[ColumnInfo] - display_options: list[dict] - - @rpc_method(name="columns.list") @http_basic_auth_login_required @handle_rpc_exceptions -def list_(*, table_oid: int, database_id: int, **kwargs) -> ColumnListReturn: +def list_(*, table_oid: int, database_id: int, **kwargs) -> list[ColumnInfo]: """ List information about columns for a table. Exposed as `list`. - Also return display options for each column, if they're defined. - Args: table_oid: Identity of the table in the user's database. database_id: The Django id of the database containing the table. Returns: - A list of column details, and a separate list of display options. + A list of column details. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: raw_column_info = get_column_info_for_table(table_oid, conn) - column_info, attnums = tuple( - zip( - *[(ColumnInfo.from_dict(col), col['id']) for col in raw_column_info] - ) - ) - display_options = get_raw_display_options( - database_id, table_oid, attnums, user - ) - return ColumnListReturn( - column_info=column_info, - display_options=display_options, - ) + return [ColumnInfo.from_dict(col) for col in raw_column_info] @rpc_method(name="columns.add") diff --git a/mathesar/tests/rpc/test_columns.py b/mathesar/tests/rpc/test_columns.py index 3069f04d13..6c54a9df39 100644 --- a/mathesar/tests/rpc/test_columns.py +++ b/mathesar/tests/rpc/test_columns.py @@ -66,74 +66,43 @@ def mock_column_info(_table_oid, conn): }, ] - def mock_display_options(_database_id, _table_oid, attnums, user): - if ( - database_id != 2 - or table_oid != 23457 - or attnums != (1, 2, 4, 8, 10) - or user.username != 'alice' - ): - raise AssertionError("incorrect parameters passed") - return [ - { - 'id': 4, - 'use_grouping': 'true', - 'number_format': 'english', - 'show_as_percentage': False, - 'maximum_fraction_digits': 2, - 'minimum_fraction_digits': 2 - } - ] monkeypatch.setattr(columns.base, 'connect', mock_connect) monkeypatch.setattr(columns.base, 'get_column_info_for_table', mock_column_info) - monkeypatch.setattr(columns.base, 'get_raw_display_options', mock_display_options) - expect_col_list = { - 'column_info': ( - { - 'id': 1, 'name': 'id', 'type': 'integer', - 'default': {'value': 'identity', 'is_dynamic': True}, - 'nullable': False, 'description': None, 'primary_key': True, - 'type_options': None, - 'has_dependents': True - }, { - 'id': 2, 'name': 'numcol', 'type': 'numeric', - 'default': {'value': "'8'::numeric", 'is_dynamic': False}, - 'nullable': True, - 'description': 'My super numeric column', - 'primary_key': False, - 'type_options': None, - 'has_dependents': False - }, { - 'id': 4, 'name': 'numcolmod', 'type': 'numeric', - 'default': None, - 'nullable': True, 'description': None, 'primary_key': False, - 'type_options': {'scale': 3, 'precision': 5}, - 'has_dependents': False - }, { - 'id': 8, 'name': 'ivlcolmod', 'type': 'interval', - 'default': None, - 'nullable': True, 'description': None, 'primary_key': False, - 'type_options': {'fields': 'day to second'}, - 'has_dependents': False - }, { - 'id': 10, 'name': 'arrcol', 'type': '_array', - 'default': None, - 'nullable': True, 'description': None, 'primary_key': False, - 'type_options': {'item_type': 'character varying', 'length': 3}, - 'has_dependents': False - } - ), - 'display_options': [ - { - 'id': 4, - 'use_grouping': 'true', - 'number_format': 'english', - 'show_as_percentage': False, - 'maximum_fraction_digits': 2, - 'minimum_fraction_digits': 2 - } - ] - } + expect_col_list = [ + { + 'id': 1, 'name': 'id', 'type': 'integer', + 'default': {'value': 'identity', 'is_dynamic': True}, + 'nullable': False, 'description': None, 'primary_key': True, + 'type_options': None, + 'has_dependents': True + }, { + 'id': 2, 'name': 'numcol', 'type': 'numeric', + 'default': {'value': "'8'::numeric", 'is_dynamic': False}, + 'nullable': True, + 'description': 'My super numeric column', + 'primary_key': False, + 'type_options': None, + 'has_dependents': False + }, { + 'id': 4, 'name': 'numcolmod', 'type': 'numeric', + 'default': None, + 'nullable': True, 'description': None, 'primary_key': False, + 'type_options': {'scale': 3, 'precision': 5}, + 'has_dependents': False + }, { + 'id': 8, 'name': 'ivlcolmod', 'type': 'interval', + 'default': None, + 'nullable': True, 'description': None, 'primary_key': False, + 'type_options': {'fields': 'day to second'}, + 'has_dependents': False + }, { + 'id': 10, 'name': 'arrcol', 'type': '_array', + 'default': None, + 'nullable': True, 'description': None, 'primary_key': False, + 'type_options': {'item_type': 'character varying', 'length': 3}, + 'has_dependents': False + } + ] actual_col_list = columns.list_(table_oid=23457, database_id=2, request=request) assert actual_col_list == expect_col_list From 183dfe4b0987e7eefb87c9925e6a33f0514c0ba4 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 24 Jun 2024 23:24:32 +0530 Subject: [PATCH 0314/1141] add docstrings --- db/sql/00_msar.sql | 99 +++++++++++++++++++++++++++++++-- db/tables/operations/import_.py | 11 ++++ mathesar/rpc/columns.py | 8 +++ mathesar/rpc/tables.py | 14 ++++- 4 files changed, 125 insertions(+), 7 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 8e3e2d782f..50b64ea1f1 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -106,8 +106,33 @@ SELECT '(' || string_agg(col, ', ') || ')' FROM unnest($1) x(col); $$ LANGUAGE sql RETURNS NULL ON NULL INPUT; +---------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- +-- GENERAL DQL FUNCTIONS +-- +-- Functions in this section are quite general, and are the basis of the others. +---------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- + + CREATE OR REPLACE FUNCTION -__msar.exec_dql(command text) RETURNS jsonb AS $$ +__msar.exec_dql(command text) RETURNS jsonb AS $$/* +Execute the given command, returning a JSON object describing the records in the following form: +[ + {"id": 1, "col1_name": "value1", "col2_name": "value2"}, + {"id": 2, "col1_name": "value1", "col2_name": "value2"}, + {"id": 3, "col1_name": "value1", "col2_name": "value2"}, + ... +] + +Useful for SELECTing from tables. Most useful when you're performing DQL. + +Note that you always have to include the primary key column(`id` in case of a Mathesar table) in the +command_template for the returned records to be uniquely identifiable. + +Args: + command: Raw string that will be executed as a command. +*/ DECLARE records jsonb; BEGIN @@ -117,6 +142,35 @@ BEGIN END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + +CREATE OR REPLACE FUNCTION +__msar.exec_dql(command_template text, arguments variadic anynonarray) RETURNS jsonb AS $$/* +Execute a templated command, returning a JSON object describing the records in the following form: +[ + {"id": 1, "col1_name": "value1", "col2_name": "value2"}, + {"id": 2, "col1_name": "value1", "col2_name": "value2"}, + {"id": 3, "col1_name": "value1", "col2_name": "value2"}, + ... +] + +The template is given in the first argument, and all further arguments are used to fill in the +template. Useful for SELECTing from tables. Most useful when you're performing DQL. + +Note that you always have to include the primary key column(`id` in case of a Mathesar table) in the +command_template for the returned records to be uniquely identifiable. + +Args: + command_template: Raw string that will be executed as a command. + arguments: arguments that will be used to fill in the template. +*/ +DECLARE formatted_command TEXT; +BEGIN + formatted_command := format(command_template, VARIADIC arguments); + RETURN __msar.exec_dql(formatted_command); +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -- INFO FUNCTIONS @@ -2409,13 +2463,46 @@ msar.get_preview( tab_id oid, col_cast_def jsonb, rec_limit integer -) RETURNS jsonb AS $$ +) RETURNS jsonb AS $$/* +Preview a table, applying different type casts and options to the underlying columns before import, +returning a JSON object describing the records of the table. + +Note that these casts are temporary and do not alter the data in the underlying table, +if you wish to alter these settings permanantly for the columns see msar.alter_columns. + +Args: + tab_id: The OID of the table to preview. + col_cast_def: A JSON object describing the column settings to apply. + rec_limit (optional): The upper limit for the number of records to return. + +The col_cast_def JSONB should have the form: +[ + { + "attnum": , + "type": { + "name": , + "options": { + "length": , + "precision": , + "scale": + "fields": , + "array": + } + }, + }, + { + ... + }, + ... +] +*/ DECLARE tab_name text; sel_query text; - result jsonb; + records jsonb; BEGIN tab_name := __msar.get_relation_name(tab_id); + sel_query := 'SELECT id, %s FROM %s LIMIT %L'; WITH preview_cte AS ( SELECT string_agg( 'CAST(' || @@ -2431,9 +2518,9 @@ BEGIN WHERE NOT msar.is_mathesar_id_column(tab_id, (col_cast ->> 'attnum')::integer) ) SELECT - format('SELECT id, %s FROM %s LIMIT %L', cast_expr, tab_name, rec_limit) - INTO sel_query FROM preview_cte; - RETURN __msar.exec_dql(sel_query); + __msar.exec_dql(sel_query, cast_expr, tab_name, rec_limit::text) + INTO records FROM preview_cte; + RETURN records; END; $$ LANGUAGE plpgsql; diff --git a/db/tables/operations/import_.py b/db/tables/operations/import_.py index 39ec1a2d77..70029587e7 100644 --- a/db/tables/operations/import_.py +++ b/db/tables/operations/import_.py @@ -97,5 +97,16 @@ def insert_csv_records( def get_preview(table_oid, column_list, conn, limit=20): + """ + Preview an imported table. Returning the records from the specified columns of the table. + + Args: + table_oid: Identity of the imported table in the user's database. + column_list: List of settings describing the casts to be applied to the columns. + limit: The upper limit for the number of records to return. + + Note that these casts are temporary and do not alter the data in the underlying table, + if you wish to alter these settings permanantly for the columns see tables/alter.py. + """ transformed_column_data = [_transform_column_alter_dict(col) for col in column_list] return exec_msar_func(conn, 'get_preview', table_oid, json.dumps(transformed_column_data), limit).fetchone()[0] diff --git a/mathesar/rpc/columns.py b/mathesar/rpc/columns.py index 84890af53c..b9833d7c8a 100644 --- a/mathesar/rpc/columns.py +++ b/mathesar/rpc/columns.py @@ -132,6 +132,14 @@ class SettableColumnInfo(TypedDict): class PreviewableColumnInfo(TypedDict): + """ + Information needed to preview a column. + + Attributes: + id: The `attnum` of the column in the table. + type: The new type to be applied to a column. + type_options: The options to be applied to the column type. + """ id: int type: Optional[str] type_options: Optional[TypeOptions] diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables.py index 40837bfded..63f93dafe6 100644 --- a/mathesar/rpc/tables.py +++ b/mathesar/rpc/tables.py @@ -208,11 +208,23 @@ def import_( def get_import_preview( *, table_oid: int, - columns: PreviewableColumnInfo, + columns: list[PreviewableColumnInfo], database_id: int, limit: int = 20, **kwargs ) -> list[dict]: + """ + Preview an imported table. + + Args: + table_oid: Identity of the imported table in the user's database. + columns: List of settings describing the casts to be applied to the columns. + database_id: The Django id of the database containing the table. + limit: The upper limit for the number of records to return. + + Returns: + The records from the specified columns of the table. + """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: return get_preview(table_oid, columns, conn, limit) From 7f09220662bda25727dee32e3a9f643c3c54002c Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 24 Jun 2024 15:14:32 -0400 Subject: [PATCH 0315/1141] Move build_qualified_name_sql to __msar --- db/sql/00_msar.sql | 22 +++++++++++----------- db/sql/20_msar_views.sql | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 6971e80334..56b5d35d36 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -130,7 +130,7 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION -msar.build_qualified_name_sql(sch_name text, obj_name text) RETURNS text AS $$/* +__msar.build_qualified_name_sql(sch_name text, obj_name text) RETURNS text AS $$/* Return the fully-qualified, properly quoted, name for a given database object (e.g., table). Args: @@ -193,7 +193,7 @@ Args: rel_name: The name of the relation, unqualified and unquoted. */ BEGIN - RETURN msar.build_qualified_name_sql(sch_name, rel_name)::regclass::oid; + RETURN __msar.build_qualified_name_sql(sch_name, rel_name)::regclass::oid; END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -998,7 +998,7 @@ Args: */ DECLARE fullname text; BEGIN - fullname := msar.build_qualified_name_sql(sch_name, old_tab_name); + fullname := __msar.build_qualified_name_sql(sch_name, old_tab_name); RETURN __msar.rename_table(fullname, quote_ident(new_tab_name)); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1047,7 +1047,7 @@ Args: comment_: The new comment. */ SELECT __msar.comment_on_table( - msar.build_qualified_name_sql(sch_name, tab_name), + __msar.build_qualified_name_sql(sch_name, tab_name), quote_literal(comment_) ); $$ LANGUAGE SQL; @@ -1145,7 +1145,7 @@ Args: */ DECLARE qualified_tab_name text; BEGIN - qualified_tab_name := msar.build_qualified_name_sql(sch_name, tab_name); + qualified_tab_name := __msar.build_qualified_name_sql(sch_name, tab_name); RETURN __msar.update_pk_sequence_to_latest(qualified_tab_name, quote_ident(col_name)); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1207,7 +1207,7 @@ DECLARE prepared_col_names text[]; DECLARE fully_qualified_tab_name text; BEGIN SELECT array_agg(quote_ident(col)) FROM unnest(col_names) AS col INTO prepared_col_names; - fully_qualified_tab_name := msar.build_qualified_name_sql(sch_name, tab_name); + fully_qualified_tab_name := __msar.build_qualified_name_sql(sch_name, tab_name); RETURN __msar.drop_columns(fully_qualified_tab_name, variadic prepared_col_names); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1503,7 +1503,7 @@ SELECT COALESCE( -- Second choice is the type specified by string IDs. __msar.get_formatted_base_type( COALESCE( - msar.build_qualified_name_sql(typ_jsonb ->> 'schema', typ_jsonb ->> 'name'), + __msar.build_qualified_name_sql(typ_jsonb ->> 'schema', typ_jsonb ->> 'name'), typ_jsonb ->> 'name', 'text' -- We fall back to 'text' when input is null or empty. ), @@ -1864,7 +1864,7 @@ SELECT array_agg( -- Build the relation name where the constraint will be applied. Prefer numeric ID. COALESCE( __msar.get_qualified_relation_name((con_create_obj -> 'fkey_relation_id')::integer::oid), - msar.build_qualified_name_sql( + __msar.build_qualified_name_sql( con_create_obj ->> 'fkey_relation_schema', con_create_obj ->> 'fkey_relation_name' ) ), @@ -2140,7 +2140,7 @@ Args: */ DECLARE qualified_tab_name text; BEGIN - qualified_tab_name := msar.build_qualified_name_sql(sch_name, tab_name); + qualified_tab_name := __msar.build_qualified_name_sql(sch_name, tab_name); RETURN __msar.drop_table(qualified_tab_name, cascade_, if_exists); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -2184,7 +2184,7 @@ Args: */ BEGIN RETURN __msar.drop_constraint( - msar.build_qualified_name_sql(sch_name, tab_name), quote_ident(con_name) + __msar.build_qualified_name_sql(sch_name, tab_name), quote_ident(con_name) ); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -2649,7 +2649,7 @@ Args: comment_: The new comment. */ SELECT __msar.comment_on_column( - msar.build_qualified_name_sql(sch_name, tab_name), + __msar.build_qualified_name_sql(sch_name, tab_name), quote_ident(col_name), quote_literal(comment_) ); diff --git a/db/sql/20_msar_views.sql b/db/sql/20_msar_views.sql index 48ad01ca8c..6e4e4fb2cd 100644 --- a/db/sql/20_msar_views.sql +++ b/db/sql/20_msar_views.sql @@ -32,7 +32,7 @@ Args: tab_id: The OID of the table whose associated view we want to name. */ BEGIN - RETURN msar.build_qualified_name_sql('msar_views', format('mv%s', tab_id)); + RETURN __msar.build_qualified_name_sql('msar_views', format('mv%s', tab_id)); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; From 0450fbef8169d905df752f669f34aa5a6c78c67f Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 25 Jun 2024 00:50:45 +0530 Subject: [PATCH 0316/1141] handle name collisions when previously generate tables get deleted --- db/sql/00_msar.sql | 16 ++++++++++++++-- db/sql/test_00_msar.sql | 29 +++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index fa204738aa..c3ea7b88c5 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2314,6 +2314,7 @@ table will always have our default 'id' column as its first column. DECLARE schema_name text; table_count integer; + uq_table_name text; fq_table_name text; created_table_id oid; column_defs __msar.col_def[]; @@ -2323,8 +2324,19 @@ BEGIN IF NULLIF(tab_name, '') IS NOT NULL THEN fq_table_name := format('%I.%I', schema_name, tab_name); ELSE - SELECT COUNT(*) INTO table_count FROM pg_catalog.pg_class WHERE relkind = 'r' AND relnamespace = sch_id; - fq_table_name := format('%I.%I', schema_name, 'Table ' || (table_count + 1)); + -- generate a table name if one doesn't exist + SELECT COUNT(*) + 1 INTO table_count + FROM pg_catalog.pg_class + WHERE relkind = 'r' AND relnamespace = sch_id; + uq_table_name := 'Table ' || table_count; + -- avoid name collisions + WHILE EXISTS ( + SELECT oid FROM pg_catalog.pg_class WHERE relname = uq_table_name AND relnamespace = sch_id + ) LOOP + table_count := table_count + 1; + uq_table_name := 'Table ' || table_count; + END LOOP; + fq_table_name := format('%I.%I', schema_name, uq_table_name); END IF; column_defs := msar.process_col_def_jsonb(0, col_defs, false, true); constraint_defs := msar.process_con_def_jsonb(0, con_defs); diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index c7716225a8..2d3332c89c 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -1464,6 +1464,35 @@ END; $f$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION test_add_mathesar_table_noname_avoid_collision() +RETURNS SETOF TEXT AS $f$ +DECLARE + generated_name text := 'Table 3'; +BEGIN + PERFORM __setup_create_table(); + PERFORM msar.add_mathesar_table( + 'tab_create_schema'::regnamespace::oid, null, null, null, null + ); + PERFORM msar.add_mathesar_table( + 'tab_create_schema'::regnamespace::oid, null, null, null, null + ); + RETURN NEXT has_table('tab_create_schema'::name, 'Table 1'::name); + RETURN NEXT has_table('tab_create_schema'::name, 'Table 2'::name); + PERFORM msar.drop_table( + sch_name => 'tab_create_schema', + tab_name => 'Table 1', + cascade_ => false, + if_exists => false + ); + RETURN NEXT hasnt_table('tab_create_schema'::name, 'Table 1'::name); + PERFORM msar.add_mathesar_table( + 'tab_create_schema'::regnamespace::oid, null, null, null, null + ); + RETURN NEXT has_table('tab_create_schema'::name, generated_name::name); +END; +$f$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION test_add_mathesar_table_columns() RETURNS SETOF TEXT AS $f$ DECLARE col_defs jsonb := $j$[ From 7ae93e60dfc210c6ef1794a13820f5601c6f5c91 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 25 Jun 2024 01:00:35 +0530 Subject: [PATCH 0317/1141] fix exec_dql --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 50b64ea1f1..2382a1b014 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -144,7 +144,7 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -__msar.exec_dql(command_template text, arguments variadic anynonarray) RETURNS jsonb AS $$/* +__msar.exec_dql(command_template text, arguments variadic anyarray) RETURNS jsonb AS $$/* Execute a templated command, returning a JSON object describing the records in the following form: [ {"id": 1, "col1_name": "value1", "col2_name": "value2"}, From 55d4a3ba8d7e072c4b304989f61b179f9092bb34 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 25 Jun 2024 01:39:51 +0530 Subject: [PATCH 0318/1141] add pytest --- mathesar/tests/rpc/test_tables.py | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/test_tables.py index d0761730e5..1d25765168 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/test_tables.py @@ -5,6 +5,7 @@ rf(pytest-django): Provides mocked `Request` objects. monkeypatch(pytest): Lets you monkeypatch an object for testing. """ +from decimal import Decimal from contextlib import contextmanager from mathesar.rpc import tables @@ -225,3 +226,44 @@ def mock_table_import(_data_file_id, table_name, _schema_oid, conn, comment): request=request ) assert imported_table_oid == 1964474 + + +def test_tables_preview(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + table_oid = 1964474 + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_table_preview(_table_oid, columns, conn, limit): + if _table_oid != table_oid: + raise AssertionError('incorrect parameters passed') + return [ + {'id': 1, 'length': Decimal('2.0')}, + {'id': 2, 'length': Decimal('3.0')}, + {'id': 3, 'length': Decimal('4.0')}, + {'id': 4, 'length': Decimal('5.22')} + ] + monkeypatch.setattr(tables, 'connect', mock_connect) + monkeypatch.setattr(tables, 'get_preview', mock_table_preview) + records = tables.get_import_preview( + table_oid=1964474, + columns=[{'attnum': 2, 'type': {'name': 'numeric', 'options': {'precision': 3, 'scale': 2}}}], + database_id=11, + request=request + ) + assert records == [ + {'id': 1, 'length': Decimal('2.0')}, + {'id': 2, 'length': Decimal('3.0')}, + {'id': 3, 'length': Decimal('4.0')}, + {'id': 4, 'length': Decimal('5.22')} + ] From 146949775f3f613b7415d2526601c65fb16f2353 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 25 Jun 2024 02:16:36 +0530 Subject: [PATCH 0319/1141] add sql test --- db/sql/test_00_msar.sql | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index db10bb73e5..46fef76ece 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -1486,6 +1486,36 @@ END; $f$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION test_get_preview() RETURNS SETOF TEXT AS $f$ +DECLARE + col_cast_def jsonb := $j$[ + { + "attnum":2, + "type": {"name": "numeric", "options": {"precision":5, "scale":2}} + } + ]$j$; + want_records jsonb := $j$[ + {"id": 1, "length": 2.00}, + {"id": 2, "length": 3.00}, + {"id": 3, "length": 4.00}, + {"id": 4, "length": 5.22} + ] + $j$; + have_records jsonb; +BEGIN + PERFORM __setup_create_table(); + CREATE TABLE tab_create_schema.foo(id INTEGER GENERATED BY DEFAULT AS IDENTITY, length FLOAT8); + INSERT INTO tab_create_schema.foo(length) VALUES (2), (3), (4), (5.2225); + have_records := msar.get_preview( + tab_id => 'tab_create_schema.foo'::regclass::oid, + col_cast_def => col_cast_def, + rec_limit => NULL + ); + RETURN NEXT is(have_records, want_records); +END; +$f$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION test_add_mathesar_table_comment() RETURNS SETOF TEXT AS $f$ DECLARE comment_ text := $c$my "Super;";'; DROP SCHEMA tab_create_schema;'$c$; From a4f0a9b85a6ee760b0401e7cd7acdc8e7d050e41 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 25 Jun 2024 18:36:03 +0530 Subject: [PATCH 0320/1141] move sql creation logic from python to sql --- db/sql/00_msar.sql | 39 +++++++++++++++++++++---- db/tables/operations/create.py | 27 +++++++++++++---- db/tables/operations/import_.py | 52 ++++++++++----------------------- 3 files changed, 70 insertions(+), 48 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 74e372b827..29b36485cf 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2355,14 +2355,19 @@ msar.prepare_table_for_import( sch_id oid, tab_name text, col_defs jsonb, + header boolean, + delimiter text, + escapechar text, + quotechar text, + encoding_ text, comment_ text ) RETURNS jsonb AS $$/* -Add a table, with a default id column, returning a JSON object describing the table. +Add a table, with a default id column, returning a JSON object containing +a properly formatted SQL statement to carry out `COPY FROM` and also contains table_oid of the created table. Each returned JSON object will have the form: { - "schema_name": , - "table_name": , + "copy_sql": , "table_oid": } @@ -2370,22 +2375,46 @@ Args: sch_id: The OID of the schema where the table will be created. tab_name: The unquoted name for the new table. col_defs: The columns for the new table, in order. + header: Whether or not the file contains a header line with the names of each column in the file. + delimiter: The character that separates columns within each row (line) of the file. + escapechar: The character that should appear before a data character that matches the `quotechar` value. + quotechar: The quoting character to be used when a data value is quoted. + encoding_: The encoding in which the file is encoded. comment_ (optional): The comment for the new table. */ DECLARE sch_name text; rel_name text; rel_id oid; + col_names_sql text; + options_sql text; + copy_sql text; BEGIN + -- Create string table rel_id := msar.add_mathesar_table(sch_id, tab_name, col_defs, NULL, comment_); + -- Get unquoted schema and table name for the created table SELECT nspname, relname INTO sch_name, rel_name FROM pg_catalog.pg_class AS pgc LEFT JOIN pg_catalog.pg_namespace AS pgn ON pgc.relnamespace = pgn.oid WHERE pgc.oid = rel_id; + -- Aggregate TEXT type column names of the created table + SELECT string_agg(quote_ident(attname), ', ') INTO col_names_sql + FROM pg_catalog.pg_attribute + WHERE attrelid = rel_id AND atttypid = 'TEXT'::regtype::oid; + -- Form a substring for COPY related options + options_sql := concat_ws( + ' ', + CASE WHEN header THEN 'HEADER' END, + CASE WHEN delimiter IS NOT NULL THEN 'DELIMITER ' || quote_literal(delimiter) END, + CASE WHEN NULLIF(escapechar, '') IS NOT NULL THEN 'ESCAPE ' || quote_literal(escapechar) END, + CASE WHEN quotechar IS NOT NULL THEN 'QUOTE ' || quote_literal(quotechar) END, + CASE WHEN encoding_ IS NOT NULL THEN 'ENCODING '|| quote_literal(encoding_) END + ); + -- Create a properly formatted COPY SQL string + copy_sql := format('COPY %I.%I (%s) FROM STDIN CSV %s', sch_name, rel_name, col_names_sql, options_sql); RETURN jsonb_build_object( - 'schema_name', quote_ident(sch_name), - 'table_name', quote_ident(rel_name), + 'copy_sql', copy_sql, 'table_oid', rel_id ); END; diff --git a/db/tables/operations/create.py b/db/tables/operations/create.py index 9db9cd64a3..d2c486cbe5 100644 --- a/db/tables/operations/create.py +++ b/db/tables/operations/create.py @@ -83,12 +83,23 @@ def create_string_column_table(name, schema_oid, column_names, engine, comment=N return table -def prepare_table_for_import(table_name, schema_oid, column_names, conn, comment=None): +def prepare_table_for_import( + table_name, + schema_oid, + column_names, + header, + conn, + delimiter=None, + escapechar=None, + quotechar=None, + encoding=None, + comment=None +): """ This method creates a Postgres table in the specified schema, with all columns being String type. - Returns the schema_name, table_name and table_oid of the created table. + Returns the copy_sql and table_oid for carrying out import into the created table. """ column_data_list = [ { @@ -96,18 +107,22 @@ def prepare_table_for_import(table_name, schema_oid, column_names, conn, comment "type": {"name": PostgresType.TEXT.id} } for column_name in column_names ] - table_info = exec_msar_func( + import_info = exec_msar_func( conn, 'prepare_table_for_import', schema_oid, table_name, json.dumps(column_data_list), + header, + delimiter, + escapechar, + quotechar, + encoding, comment ).fetchone()[0] return ( - table_info['schema_name'], - table_info['table_name'], - table_info['table_oid'] + import_info['copy_sql'], + import_info['table_oid'] ) diff --git a/db/tables/operations/import_.py b/db/tables/operations/import_.py index a22ec9121c..9385afa67a 100644 --- a/db/tables/operations/import_.py +++ b/db/tables/operations/import_.py @@ -1,6 +1,5 @@ import tempfile import clevercsv as csv -from psycopg import sql from db.tables.operations.create import prepare_table_for_import from db.encoding_utils import get_sql_compatible_encoding from mathesar.models.deprecated import DataFile @@ -17,60 +16,39 @@ def import_csv(data_file_id, table_name, schema_oid, conn, comment=None): data_file.escapechar ) encoding = get_file_encoding(data_file.file) + conversion_encoding, sql_encoding = get_sql_compatible_encoding(encoding) with open(file_path, 'rb') as csv_file: csv_reader = get_sv_reader(csv_file, header, dialect) column_names = process_column_names(csv_reader.fieldnames) - schema_name, table_name, table_oid = prepare_table_for_import( + copy_sql, table_oid = prepare_table_for_import( table_name, schema_oid, column_names, + header, conn, + dialect.delimiter, + dialect.escapechar, + dialect.quotechar, + sql_encoding, comment ) insert_csv_records( - schema_name, - table_name, - conn, + copy_sql, file_path, - column_names, - header, - dialect.delimiter, - dialect.escapechar, - dialect.quotechar, - encoding + encoding, + conversion_encoding, + conn ) return table_oid def insert_csv_records( - schema_name, - table_name, - conn, + copy_sql, file_path, - column_names, - header, - delimiter=None, - escape=None, - quote=None, - encoding=None + encoding, + conversion_encoding, + conn ): - conversion_encoding, sql_encoding = get_sql_compatible_encoding(encoding) - formatted_columns = sql.SQL(",").join(sql.Identifier(column_name) for column_name in column_names) - copy_sql = sql.SQL( - "COPY {relation_name} ({formatted_columns}) FROM STDIN CSV {header} {delimiter} {escape} {quote} {encoding}" - ).format( - relation_name=sql.Identifier(schema_name, table_name), - formatted_columns=formatted_columns, - header=sql.SQL("HEADER" if header else ""), - delimiter=sql.SQL(f"DELIMITER E'{delimiter}'" if delimiter else ""), - escape=sql.SQL(f"ESCAPE '{escape}'" if escape else ""), - quote=sql.SQL( - ("QUOTE ''''" if quote == "'" else f"QUOTE '{quote}'") - if quote - else "" - ), - encoding=sql.SQL(f"ENCODING '{sql_encoding}'" if sql_encoding else ""), - ) cursor = conn.cursor() with open(file_path, 'r', encoding=encoding) as csv_file: if conversion_encoding == encoding: From 39defa03f68061d368f77e549b5049462cb6a06a Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 25 Jun 2024 19:01:25 +0530 Subject: [PATCH 0321/1141] add NULLIFs --- db/sql/00_msar.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 29b36485cf..5f93479513 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2406,10 +2406,10 @@ BEGIN options_sql := concat_ws( ' ', CASE WHEN header THEN 'HEADER' END, - CASE WHEN delimiter IS NOT NULL THEN 'DELIMITER ' || quote_literal(delimiter) END, + CASE WHEN NULLIF(delimiter, '') IS NOT NULL THEN 'DELIMITER ' || quote_literal(delimiter) END, CASE WHEN NULLIF(escapechar, '') IS NOT NULL THEN 'ESCAPE ' || quote_literal(escapechar) END, - CASE WHEN quotechar IS NOT NULL THEN 'QUOTE ' || quote_literal(quotechar) END, - CASE WHEN encoding_ IS NOT NULL THEN 'ENCODING '|| quote_literal(encoding_) END + CASE WHEN NULLIF(quotechar, '') IS NOT NULL THEN 'QUOTE ' || quote_literal(quotechar) END, + CASE WHEN NULLIF(encoding_, '') IS NOT NULL THEN 'ENCODING '|| quote_literal(encoding_) END ); -- Create a properly formatted COPY SQL string copy_sql := format('COPY %I.%I (%s) FROM STDIN CSV %s', sch_name, rel_name, col_names_sql, options_sql); From af82a18ffff300c03068aa810e712ef11c22b32b Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 25 Jun 2024 21:50:42 +0800 Subject: [PATCH 0322/1141] remove deprecated function and other cruft --- mathesar/tests/rpc/test_columns.py | 4 ++-- mathesar/utils/columns.py | 16 ---------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/mathesar/tests/rpc/test_columns.py b/mathesar/tests/rpc/test_columns.py index 6c54a9df39..70c21a8f2f 100644 --- a/mathesar/tests/rpc/test_columns.py +++ b/mathesar/tests/rpc/test_columns.py @@ -19,7 +19,7 @@ def test_columns_list(rf, monkeypatch): @contextmanager def mock_connect(_database_id, user): - if _database_id == 2 and user.username == 'alice': + if _database_id == database_id and user.username == 'alice': try: yield True finally: @@ -103,7 +103,7 @@ def mock_column_info(_table_oid, conn): 'has_dependents': False } ] - actual_col_list = columns.list_(table_oid=23457, database_id=2, request=request) + actual_col_list = columns.list_(table_oid=23457, database_id=database_id, request=request) assert actual_col_list == expect_col_list diff --git a/mathesar/utils/columns.py b/mathesar/utils/columns.py index 2429ad59e0..7c48cd914e 100644 --- a/mathesar/utils/columns.py +++ b/mathesar/utils/columns.py @@ -1,21 +1,5 @@ -from mathesar.models.deprecated import Column from mathesar.models.base import ColumnMetaData -# This should be replaced once we have the ColumnMetadata model sorted out. -def get_raw_display_options(database_id, table_oid, attnums, user): - """Get display options for the columns from Django.""" - if user.metadata_privileges(database_id) is not None: - return [ - {"id": c.attnum} | c.display_options - for c in Column.current_objects.filter( - table__schema__database__id=database_id, - table__oid=table_oid, - attnum__in=attnums - ) - if c.display_options is not None - ] - - def get_columns_meta_data(table_oid, database_id): return ColumnMetaData.filter(database__id=database_id, table_oid=table_oid) From 022834ab1f3f70dba99aecb1995b4f8bb31a9715 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 25 Jun 2024 21:56:33 +0800 Subject: [PATCH 0323/1141] Move RPC columns tests to new namespace --- mathesar/tests/rpc/{test_columns.py => columns/test_base.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mathesar/tests/rpc/{test_columns.py => columns/test_base.py} (100%) diff --git a/mathesar/tests/rpc/test_columns.py b/mathesar/tests/rpc/columns/test_base.py similarity index 100% rename from mathesar/tests/rpc/test_columns.py rename to mathesar/tests/rpc/columns/test_base.py From 56fabcf4a63d8edad764c1ac01b8d8408e9ab84b Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 25 Jun 2024 22:47:03 +0800 Subject: [PATCH 0324/1141] add basic test for metadata RPC function --- mathesar/tests/rpc/columns/test_base.py | 2 +- mathesar/tests/rpc/columns/test_metadata.py | 59 +++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 mathesar/tests/rpc/columns/test_metadata.py diff --git a/mathesar/tests/rpc/columns/test_base.py b/mathesar/tests/rpc/columns/test_base.py index 70c21a8f2f..657145f514 100644 --- a/mathesar/tests/rpc/columns/test_base.py +++ b/mathesar/tests/rpc/columns/test_base.py @@ -1,5 +1,5 @@ """ -This file tests the column listing function. +This file tests the column RPC functions. Fixtures: rf(pytest-django): Provides mocked `Request` objects. diff --git a/mathesar/tests/rpc/columns/test_metadata.py b/mathesar/tests/rpc/columns/test_metadata.py new file mode 100644 index 0000000000..f9b6e087c0 --- /dev/null +++ b/mathesar/tests/rpc/columns/test_metadata.py @@ -0,0 +1,59 @@ +""" +This file tests the column metadata RPC functions. + +Fixtures: + monkeypatch(pytest): Lets you monkeypatch an object for testing. +""" +from mathesar.models.base import ColumnMetaData, Database, Server +from mathesar.rpc.columns import metadata + + +# TODO consider mocking out ColumnMetaData queryset for this test +def test_columns_meta_data_list(monkeypatch): + database_id = 2 + table_oid = 123456 + + def mock_get_columns_meta_data(_table_oid, _database_id): + server_model = Server(id=2, host='example.com', port=5432) + db_model = Database(id=_database_id, name='mymathesardb', server=server_model) + return [ + ColumnMetaData( + database=db_model, table_oid=_table_oid, attnum=2, + bool_input="dropdown", bool_true="TRUE", bool_false="FALSE", + num_min_frac_digits=5, num_max_frac_digits=10, num_show_as_perc=False, + mon_currency_symbol="EUR", mon_currency_location="end-with-space", + time_format=None, date_format=None, + duration_min=None, duration_max=None, duration_show_units=True, + ), + ColumnMetaData( + database=db_model, table_oid=_table_oid, attnum=8, + bool_input="checkbox", bool_true="true", bool_false="false", + num_min_frac_digits=2, num_max_frac_digits=8, num_show_as_perc=True, + mon_currency_symbol="$", mon_currency_location="after-minus", + time_format=None, date_format=None, + duration_min=None, duration_max=None, duration_show_units=True, + ) + ] + + monkeypatch.setattr(metadata, "get_columns_meta_data", mock_get_columns_meta_data) + + expect_metadata_list = [ + metadata.ColumnMetaData( + database_id=database_id, table_oid=table_oid, attnum=2, + bool_input="dropdown", bool_true="TRUE", bool_false="FALSE", + num_min_frac_digits=5, num_max_frac_digits=10, num_show_as_perc=False, + mon_currency_symbol="EUR", mon_currency_location="end-with-space", + time_format=None, date_format=None, + duration_min=None, duration_max=None, duration_show_units=True, + ), + metadata.ColumnMetaData( + database_id=database_id, table_oid=table_oid, attnum=8, + bool_input="checkbox", bool_true="true", bool_false="false", + num_min_frac_digits=2, num_max_frac_digits=8, num_show_as_perc=True, + mon_currency_symbol="$", mon_currency_location="after-minus", + time_format=None, date_format=None, + duration_min=None, duration_max=None, duration_show_units=True, + ), + ] + actual_metadata_list = metadata.list_(table_oid=table_oid, database_id=database_id) + assert actual_metadata_list == expect_metadata_list From ebc8b15ac349f8cbaf1b4ab2744d000ec1a1781c Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 25 Jun 2024 22:48:00 +0800 Subject: [PATCH 0325/1141] fix improperly copied variable name --- mathesar/rpc/columns/metadata.py | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mathesar/rpc/columns/metadata.py b/mathesar/rpc/columns/metadata.py index dd867608a5..561355e04e 100644 --- a/mathesar/rpc/columns/metadata.py +++ b/mathesar/rpc/columns/metadata.py @@ -52,24 +52,24 @@ class ColumnMetaData(TypedDict): duration_show_units: Optional[bool] @classmethod - def from_db_model(cls, db_model): + def from_model(cls, model): return cls( - database_id=db_model.database.id, - table_oid=db_model.table_oid, - attnum=db_model.attnum, - bool_input=db_model.bool_input, - bool_true=db_model.bool_true, - bool_false=db_model.bool_false, - num_min_frac_digits=db_model.num_min_frac_digits, - num_max_frac_digits=db_model.num_max_frac_digits, - num_show_as_perc=db_model.num_show_as_perc, - mon_currency_symbol=db_model.mon_currency_symbol, - mon_currency_location=db_model.mon_currency_location, - time_format=db_model.time_format, - date_format=db_model.date_format, - duration_min=db_model.duration_min, - duration_max=db_model.duration_max, - duration_show_units=db_model.duration_show_units, + database_id=model.database.id, + table_oid=model.table_oid, + attnum=model.attnum, + bool_input=model.bool_input, + bool_true=model.bool_true, + bool_false=model.bool_false, + num_min_frac_digits=model.num_min_frac_digits, + num_max_frac_digits=model.num_max_frac_digits, + num_show_as_perc=model.num_show_as_perc, + mon_currency_symbol=model.mon_currency_symbol, + mon_currency_location=model.mon_currency_location, + time_format=model.time_format, + date_format=model.date_format, + duration_min=model.duration_min, + duration_max=model.duration_max, + duration_show_units=model.duration_show_units, ) @@ -89,5 +89,5 @@ def list_(*, table_oid: int, database_id: int, **kwargs) -> list[ColumnMetaData] """ columns_meta_data = get_columns_meta_data(table_oid, database_id) return [ - ColumnMetaData.from_db_model(db_model) for db_model in columns_meta_data + ColumnMetaData.from_model(model) for model in columns_meta_data ] From a15f6fefbf7d269f5039b16bfa9c1fbb85a63f4e Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 25 Jun 2024 13:35:38 -0400 Subject: [PATCH 0326/1141] Fix typo in function doc --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 56b5d35d36..950c403c24 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -230,7 +230,7 @@ CREATE OR REPLACE FUNCTION msar.get_column_name(rel_id oid, col_name text) RETURNS text AS $$/* Return the UNQUOTED name for a given column in a given relation (e.g., table). -More precisely, this function returns the quoted name of attributes of any relation appearing in the +More precisely, this function returns the unquoted name of attributes of any relation appearing in the pg_class catalog table (so you could find attributes of indices with this function). If the given col_name is not in the relation, we return null. From b67da61df9337a7317815693d3a453ec8309d52a Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 25 Jun 2024 21:04:54 -0400 Subject: [PATCH 0327/1141] Add SQL code standard for casting OIDs to bigint --- db/sql/STANDARDS.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/db/sql/STANDARDS.md b/db/sql/STANDARDS.md index a02a5b04a7..81ead6d690 100644 --- a/db/sql/STANDARDS.md +++ b/db/sql/STANDARDS.md @@ -58,4 +58,27 @@ From [OWASP](https://owasp.org/www-project-proactive-controls/v3/en/c4-encode-es Always qualify system catalog tables by prefixing them with `pg_catalog.`. If you don't, then user-defined tables can shadow the system catalog tables, breaking core functionality. +## Casting OIDs to JSON + +Always cast OID values to `bigint` before putting them in JSON (or jsonb). + +_Don't_ cast OID values to `integer`. + +This is because the [`oid` type](https://www.postgresql.org/docs/current/datatype-oid.html) is an _unsigned_ 32-bit integer whereas the `integer` type is a _signed_ 32-bit integer. That means it's possible for a database to have OID values which don't fit into the `integer` type. + +For example, putting a large OID value into JSON by casting it to an integer will cause overflow: + +```SQL +SELECT jsonb_build_object('foo', 3333333333::oid::integer); -- ❌ Bad +``` + +> `{"foo": -961633963}` + +Instead, cast it to `bigint` + +```SQL +SELECT jsonb_build_object('foo', 3333333333::oid::bigint); -- ✅ Good +``` + +> `{"foo": 3333333333}` From 084a4ee130db64d542bcdf94854eb42e54dd8167 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 26 Jun 2024 15:59:04 +0530 Subject: [PATCH 0328/1141] quote column names --- db/sql/00_msar.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 2382a1b014..931ec6afb2 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2507,11 +2507,11 @@ BEGIN SELECT string_agg( 'CAST(' || __msar.build_cast_expr( - msar.get_column_name(tab_id, (col_cast ->> 'attnum')::integer), col_cast -> 'type' ->> 'name' + quote_ident(msar.get_column_name(tab_id, (col_cast ->> 'attnum')::integer)), col_cast -> 'type' ->> 'name' ) || ' AS ' || msar.build_type_text(col_cast -> 'type') || - ')'|| ' AS ' || msar.get_column_name(tab_id, (col_cast ->> 'attnum')::integer), + ')'|| ' AS ' || quote_ident(msar.get_column_name(tab_id, (col_cast ->> 'attnum')::integer)), ', ' ) AS cast_expr FROM jsonb_array_elements(col_cast_def) AS col_cast From ce0d8d2245a07c9fbc252d7935755164a9fc7530 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 26 Jun 2024 16:06:42 +0530 Subject: [PATCH 0329/1141] fix merge conflict --- db/tables/operations/import_.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/db/tables/operations/import_.py b/db/tables/operations/import_.py index d9fde5d6dc..7929bff2e3 100644 --- a/db/tables/operations/import_.py +++ b/db/tables/operations/import_.py @@ -2,13 +2,9 @@ import tempfile import clevercsv as csv -<<<<<<< import_preview -from psycopg import sql from db.connection import exec_msar_func from db.columns.operations.alter import _transform_column_alter_dict -======= ->>>>>>> develop from db.tables.operations.create import prepare_table_for_import from db.encoding_utils import get_sql_compatible_encoding from mathesar.models.deprecated import DataFile From 136b252a1ee59e2086b038d5223a276e2cd1d760 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 26 Jun 2024 16:10:23 +0530 Subject: [PATCH 0330/1141] fix function name --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index cc0fd6ec4c..260c37d5e1 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2451,7 +2451,7 @@ DECLARE sel_query text; records jsonb; BEGIN - tab_name := __msar.get_relation_name(tab_id); + tab_name := __msar.get_qualified_relation_name(tab_id); sel_query := 'SELECT id, %s FROM %s LIMIT %L'; WITH preview_cte AS ( SELECT string_agg( From a20e663cba091f27a20f4e7b4c1caa3e4aaac1a2 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 26 Jun 2024 16:18:42 +0530 Subject: [PATCH 0331/1141] remove id requirement --- db/sql/00_msar.sql | 3 +-- db/sql/test_00_msar.sql | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 260c37d5e1..be7c43f42f 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2452,7 +2452,7 @@ DECLARE records jsonb; BEGIN tab_name := __msar.get_qualified_relation_name(tab_id); - sel_query := 'SELECT id, %s FROM %s LIMIT %L'; + sel_query := 'SELECT %s FROM %s LIMIT %L'; WITH preview_cte AS ( SELECT string_agg( 'CAST(' || @@ -2465,7 +2465,6 @@ BEGIN ', ' ) AS cast_expr FROM jsonb_array_elements(col_cast_def) AS col_cast - WHERE NOT msar.is_mathesar_id_column(tab_id, (col_cast ->> 'attnum')::integer) ) SELECT __msar.exec_dql(sel_query, cast_expr, tab_name, rec_limit::text) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 86b59081e3..986e3a89e3 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -1482,6 +1482,10 @@ $f$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION test_get_preview() RETURNS SETOF TEXT AS $f$ DECLARE col_cast_def jsonb := $j$[ + { + "attnum": 1, + "type": {"name": "integer"} + }, { "attnum":2, "type": {"name": "numeric", "options": {"precision":5, "scale":2}} From cb78fe9f57d57e13d1282f7041fe83432a05dca0 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 26 Jun 2024 16:28:40 +0530 Subject: [PATCH 0332/1141] clear up documentation about id --- db/sql/00_msar.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index be7c43f42f..b175c85681 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -72,8 +72,8 @@ Execute the given command, returning a JSON object describing the records in the Useful for SELECTing from tables. Most useful when you're performing DQL. -Note that you always have to include the primary key column(`id` in case of a Mathesar table) in the -command_template for the returned records to be uniquely identifiable. +Note that you must include the primary key column(`id` in case of a Mathesar table) in the +command_template if you want the returned records to be uniquely identifiable. Args: command: Raw string that will be executed as a command. @@ -101,8 +101,8 @@ Execute a templated command, returning a JSON object describing the records in t The template is given in the first argument, and all further arguments are used to fill in the template. Useful for SELECTing from tables. Most useful when you're performing DQL. -Note that you always have to include the primary key column(`id` in case of a Mathesar table) in the -command_template for the returned records to be uniquely identifiable. +Note that you must include the primary key column(`id` in case of a Mathesar table) in the +command_template if you want the returned records to be uniquely identifiable. Args: command_template: Raw string that will be executed as a command. From 7c31e76cf4c9d41d5d5355f15c6fec255f01028f Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 28 Jun 2024 01:32:29 +0530 Subject: [PATCH 0333/1141] add table metadata model and rpc endpoint --- config/settings/common_settings.py | 3 +- ...09_alter_columnmetadata_attnum_and_more.py | 43 ++++++++++++ mathesar/models/base.py | 22 ++++++- mathesar/rpc/columns/metadata.py | 2 +- mathesar/rpc/tables/__init__.py | 1 + mathesar/rpc/{tables.py => tables/base.py} | 0 mathesar/rpc/tables/metadata.py | 66 +++++++++++++++++++ .../{test_tables.py => tables/test_base.py} | 28 ++++---- mathesar/tests/rpc/tables/test_metadata.py | 0 mathesar/tests/rpc/test_endpoints.py | 5 ++ mathesar/utils/tables.py | 5 ++ 11 files changed, 157 insertions(+), 18 deletions(-) create mode 100644 mathesar/migrations/0009_alter_columnmetadata_attnum_and_more.py create mode 100644 mathesar/rpc/tables/__init__.py rename mathesar/rpc/{tables.py => tables/base.py} (100%) create mode 100644 mathesar/rpc/tables/metadata.py rename mathesar/tests/rpc/{test_tables.py => tables/test_base.py} (89%) create mode 100644 mathesar/tests/rpc/tables/test_metadata.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index c37d6d23ca..c574b418d5 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -69,7 +69,8 @@ def pipe_delim(pipe_string): 'mathesar.rpc.columns', 'mathesar.rpc.columns.metadata', 'mathesar.rpc.schemas', - 'mathesar.rpc.tables' + 'mathesar.rpc.tables', + 'mathesar.rpc.tables.metadata' ] TEMPLATES = [ diff --git a/mathesar/migrations/0009_alter_columnmetadata_attnum_and_more.py b/mathesar/migrations/0009_alter_columnmetadata_attnum_and_more.py new file mode 100644 index 0000000000..35481c0d02 --- /dev/null +++ b/mathesar/migrations/0009_alter_columnmetadata_attnum_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.11 on 2024-06-27 20:00 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('mathesar', '0008_add_metadata_models'), + ] + + operations = [ + migrations.AlterField( + model_name='columnmetadata', + name='attnum', + field=models.SmallIntegerField(), + ), + migrations.AlterField( + model_name='columnmetadata', + name='table_oid', + field=models.PositiveBigIntegerField(), + ), + migrations.CreateModel( + name='TableMetaData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('schema_oid', models.PositiveBigIntegerField()), + ('table_oid', models.PositiveBigIntegerField()), + ('import_verified', models.BooleanField(default=False)), + ('column_order', models.JSONField(default=list)), + ('preview_customized', models.BooleanField(default=False)), + ('preview_template', models.CharField(blank=True, max_length=255)), + ('database', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mathesar.database')), + ], + ), + migrations.AddConstraint( + model_name='tablemetadata', + constraint=models.UniqueConstraint(fields=('database', 'schema_oid', 'table_oid'), name='unique_table_metadata'), + ), + ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 43b9e1607e..c900eb176a 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -84,8 +84,8 @@ def connection(self): class ColumnMetaData(BaseModel): database = models.ForeignKey('Database', on_delete=models.CASCADE) - table_oid = models.PositiveIntegerField() - attnum = models.PositiveIntegerField() + table_oid = models.PositiveBigIntegerField() + attnum = models.SmallIntegerField() bool_input = models.CharField( choices=[("dropdown", "dropdown"), ("checkbox", "checkbox")], blank=True @@ -121,3 +121,21 @@ class Meta: name="frac_digits_integrity" ) ] + + +class TableMetaData(BaseModel): + database = models.ForeignKey('Database', on_delete=models.CASCADE) + schema_oid = models.PositiveBigIntegerField() + table_oid = models.PositiveBigIntegerField() + import_verified = models.BooleanField(default=False) + column_order = models.JSONField(default=list) + preview_customized = models.BooleanField(default=False) + preview_template = models.CharField(max_length=255, blank=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["database", "schema_oid", "table_oid"], + name="unique_table_metadata" + ) + ] diff --git a/mathesar/rpc/columns/metadata.py b/mathesar/rpc/columns/metadata.py index 561355e04e..f99118b6b5 100644 --- a/mathesar/rpc/columns/metadata.py +++ b/mathesar/rpc/columns/metadata.py @@ -18,7 +18,7 @@ class ColumnMetaData(TypedDict): Attributes: database_id: The Django id of the database containing the table. - table_oid: The OID of the table containing the column + table_oid: The OID of the table containing the column. attnum: The attnum of the column in the table. bool_input: How the input for a boolean column should be shown. bool_true: A string to display for `true` values. diff --git a/mathesar/rpc/tables/__init__.py b/mathesar/rpc/tables/__init__.py new file mode 100644 index 0000000000..79c539308a --- /dev/null +++ b/mathesar/rpc/tables/__init__.py @@ -0,0 +1 @@ +from .base import * # noqa \ No newline at end of file diff --git a/mathesar/rpc/tables.py b/mathesar/rpc/tables/base.py similarity index 100% rename from mathesar/rpc/tables.py rename to mathesar/rpc/tables/base.py diff --git a/mathesar/rpc/tables/metadata.py b/mathesar/rpc/tables/metadata.py new file mode 100644 index 0000000000..ac72a55f25 --- /dev/null +++ b/mathesar/rpc/tables/metadata.py @@ -0,0 +1,66 @@ +""" +Classes and functions exposed to the RPC endpoint for managing table metadata. +""" +from typing import Optional, TypedDict + +from modernrpc.core import rpc_method +from modernrpc.auth.basic import http_basic_auth_login_required + +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions +from mathesar.utils.tables import get_table_meta_data + + +class TableMetaData(TypedDict): + """ + Metadata for a table in a database. + + Only the `database`, `schema_oid`, and `table_oid` keys are required. + + Attributes: + database_id: The Django id of the database containing the table. + schema_oid: The OID of the schema containing the table. + table_oid: The OID of the table in the database. + import_verified: Specifies whether a file has been successfully imported into a table. + column_order: The order in which columns of a table are displayed. + preview_customized: Specifies whether the preview has been customized. + preview_template: Preview template for a referent column. + """ + database_id: int + schema_oid: int + table_oid: int + import_verified: Optional[bool] + column_order: Optional[list[int]] + preview_customized: Optional[bool] + preview_template: Optional[str] + + @classmethod + def from_model(cls, model): + return cls( + database_id=model.database.id, + schema_oid=model.schema_oid, + table_oid=model.table_oid, + import_verified=model.import_verified, + column_order=model.column_order, + preview_customized=model.preview_customized, + preview_template=model.preview_template, + ) + + +@rpc_method(name="tables.metadata.get") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list(*, schema_oid: int, database_id: int, **kwargs) -> list[TableMetaData]: + """ + List metadata associated with tables for a schema. + + Args: + schema_oid: Identity of the schema in the user's database. + database_id: The Django id of the database containing the table. + + Returns: + Metadata object for a given table oid. + """ + table_meta_data = get_table_meta_data(schema_oid, database_id) + return [ + TableMetaData.from_model(model) for model in table_meta_data + ] diff --git a/mathesar/tests/rpc/test_tables.py b/mathesar/tests/rpc/tables/test_base.py similarity index 89% rename from mathesar/tests/rpc/test_tables.py rename to mathesar/tests/rpc/tables/test_base.py index 1d25765168..693f9b901d 100644 --- a/mathesar/tests/rpc/test_tables.py +++ b/mathesar/tests/rpc/tables/test_base.py @@ -45,8 +45,8 @@ def mock_table_info(_schema_oid, conn): 'description': None } ] - monkeypatch.setattr(tables, 'connect', mock_connect) - monkeypatch.setattr(tables, 'get_table_info', mock_table_info) + monkeypatch.setattr(tables.base, 'connect', mock_connect) + monkeypatch.setattr(tables.base, 'get_table_info', mock_table_info) expect_table_list = [ { 'oid': 17408, @@ -90,8 +90,8 @@ def mock_table_get(_table_oid, conn): 'schema': 2200, 'description': 'a description on the authors table.' } - monkeypatch.setattr(tables, 'connect', mock_connect) - monkeypatch.setattr(tables, 'get_table', mock_table_get) + monkeypatch.setattr(tables.base, 'connect', mock_connect) + monkeypatch.setattr(tables.base, 'get_table', mock_table_get) expect_table_list = { 'oid': table_oid, 'name': 'Authors', @@ -123,8 +123,8 @@ def mock_drop_table(_table_oid, conn, cascade): raise AssertionError('incorrect parameters passed') return 'public."Table 0"' - monkeypatch.setattr(tables, 'connect', mock_connect) - monkeypatch.setattr(tables, 'drop_table_from_database', mock_drop_table) + monkeypatch.setattr(tables.base, 'connect', mock_connect) + monkeypatch.setattr(tables.base, 'drop_table_from_database', mock_drop_table) deleted_table = tables.delete(table_oid=1964474, database_id=11, request=request) assert deleted_table == 'public."Table 0"' @@ -149,8 +149,8 @@ def mock_table_add(table_name, _schema_oid, conn, column_data_list, constraint_d if _schema_oid != schema_oid: raise AssertionError('incorrect parameters passed') return 1964474 - monkeypatch.setattr(tables, 'connect', mock_connect) - monkeypatch.setattr(tables, 'create_table_on_database', mock_table_add) + monkeypatch.setattr(tables.base, 'connect', mock_connect) + monkeypatch.setattr(tables.base, 'create_table_on_database', mock_table_add) actual_table_oid = tables.add(table_name='newtable', schema_oid=2200, database_id=11, request=request) assert actual_table_oid == 1964474 @@ -180,8 +180,8 @@ def mock_table_patch(_table_oid, _table_data_dict, conn): if _table_oid != table_oid and _table_data_dict != table_data_dict: raise AssertionError('incorrect parameters passed') return 'newtabname' - monkeypatch.setattr(tables, 'connect', mock_connect) - monkeypatch.setattr(tables, 'alter_table_on_database', mock_table_patch) + monkeypatch.setattr(tables.base, 'connect', mock_connect) + monkeypatch.setattr(tables.base, 'alter_table_on_database', mock_table_patch) altered_table_name = tables.patch( table_oid=1964474, table_data_dict={ @@ -216,8 +216,8 @@ def mock_table_import(_data_file_id, table_name, _schema_oid, conn, comment): if _schema_oid != schema_oid and _data_file_id != data_file_id: raise AssertionError('incorrect parameters passed') return 1964474 - monkeypatch.setattr(tables, 'connect', mock_connect) - monkeypatch.setattr(tables, 'import_csv', mock_table_import) + monkeypatch.setattr(tables.base, 'connect', mock_connect) + monkeypatch.setattr(tables.base, 'import_csv', mock_table_import) imported_table_oid = tables.import_( data_file_id=10, table_name='imported_table', @@ -253,8 +253,8 @@ def mock_table_preview(_table_oid, columns, conn, limit): {'id': 3, 'length': Decimal('4.0')}, {'id': 4, 'length': Decimal('5.22')} ] - monkeypatch.setattr(tables, 'connect', mock_connect) - monkeypatch.setattr(tables, 'get_preview', mock_table_preview) + monkeypatch.setattr(tables.base, 'connect', mock_connect) + monkeypatch.setattr(tables.base, 'get_preview', mock_table_preview) records = tables.get_import_preview( table_oid=1964474, columns=[{'attnum': 2, 'type': {'name': 'numeric', 'options': {'precision': 3, 'scale': 2}}}], diff --git a/mathesar/tests/rpc/tables/test_metadata.py b/mathesar/tests/rpc/tables/test_metadata.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index a3c63ab75a..9190647883 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -108,6 +108,11 @@ tables.get_import_preview, "tables.get_import_preview", [user_is_authenticated] + ), + ( + tables.metadata.list_, + "tables.metadata.list", + [user_is_authenticated] ) ] diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index 1772d653d7..15b1e2ef9f 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -5,6 +5,7 @@ from mathesar.database.base import create_mathesar_engine from mathesar.imports.base import create_table_from_data_file from mathesar.models.deprecated import Table +from mathesar.models.base import TableMetaData from mathesar.state.django import reflect_columns_from_tables from mathesar.state import get_cached_metadata @@ -83,3 +84,7 @@ def create_empty_table(name, schema, comment=None): table, _ = Table.current_objects.get_or_create(oid=db_table_oid, schema=schema) reflect_columns_from_tables([table], metadata=get_cached_metadata()) return table + + +def get_table_meta_data(schema_oid, database_id): + return TableMetaData.objects.filter(database__id=database_id, schema_oid=schema_oid) From 6adf4e8230fb599b0c9af045b3b98f9d1a0f868b Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 28 Jun 2024 02:18:24 +0530 Subject: [PATCH 0334/1141] add test --- ...e.py => 0009_add_column_metadata_model.py} | 0 mathesar/rpc/tables/__init__.py | 2 +- mathesar/rpc/tables/metadata.py | 8 ++-- .../columns/{test_base.py => test_c_base.py} | 0 .../{test_metadata.py => test_c_metadata.py} | 0 mathesar/tests/rpc/tables/test_metadata.py | 0 .../tables/{test_base.py => test_t_base.py} | 0 mathesar/tests/rpc/tables/test_t_metadata.py | 45 +++++++++++++++++++ mathesar/utils/tables.py | 2 +- 9 files changed, 51 insertions(+), 6 deletions(-) rename mathesar/migrations/{0009_alter_columnmetadata_attnum_and_more.py => 0009_add_column_metadata_model.py} (100%) rename mathesar/tests/rpc/columns/{test_base.py => test_c_base.py} (100%) rename mathesar/tests/rpc/columns/{test_metadata.py => test_c_metadata.py} (100%) delete mode 100644 mathesar/tests/rpc/tables/test_metadata.py rename mathesar/tests/rpc/tables/{test_base.py => test_t_base.py} (100%) create mode 100644 mathesar/tests/rpc/tables/test_t_metadata.py diff --git a/mathesar/migrations/0009_alter_columnmetadata_attnum_and_more.py b/mathesar/migrations/0009_add_column_metadata_model.py similarity index 100% rename from mathesar/migrations/0009_alter_columnmetadata_attnum_and_more.py rename to mathesar/migrations/0009_add_column_metadata_model.py diff --git a/mathesar/rpc/tables/__init__.py b/mathesar/rpc/tables/__init__.py index 79c539308a..44c8b96840 100644 --- a/mathesar/rpc/tables/__init__.py +++ b/mathesar/rpc/tables/__init__.py @@ -1 +1 @@ -from .base import * # noqa \ No newline at end of file +from .base import * # noqa diff --git a/mathesar/rpc/tables/metadata.py b/mathesar/rpc/tables/metadata.py index ac72a55f25..64b2b85891 100644 --- a/mathesar/rpc/tables/metadata.py +++ b/mathesar/rpc/tables/metadata.py @@ -7,7 +7,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -from mathesar.utils.tables import get_table_meta_data +from mathesar.utils.tables import get_tables_meta_data class TableMetaData(TypedDict): @@ -46,10 +46,10 @@ def from_model(cls, model): ) -@rpc_method(name="tables.metadata.get") +@rpc_method(name="tables.metadata.list") @http_basic_auth_login_required @handle_rpc_exceptions -def list(*, schema_oid: int, database_id: int, **kwargs) -> list[TableMetaData]: +def list_(*, schema_oid: int, database_id: int, **kwargs) -> list[TableMetaData]: """ List metadata associated with tables for a schema. @@ -60,7 +60,7 @@ def list(*, schema_oid: int, database_id: int, **kwargs) -> list[TableMetaData]: Returns: Metadata object for a given table oid. """ - table_meta_data = get_table_meta_data(schema_oid, database_id) + table_meta_data = get_tables_meta_data(schema_oid, database_id) return [ TableMetaData.from_model(model) for model in table_meta_data ] diff --git a/mathesar/tests/rpc/columns/test_base.py b/mathesar/tests/rpc/columns/test_c_base.py similarity index 100% rename from mathesar/tests/rpc/columns/test_base.py rename to mathesar/tests/rpc/columns/test_c_base.py diff --git a/mathesar/tests/rpc/columns/test_metadata.py b/mathesar/tests/rpc/columns/test_c_metadata.py similarity index 100% rename from mathesar/tests/rpc/columns/test_metadata.py rename to mathesar/tests/rpc/columns/test_c_metadata.py diff --git a/mathesar/tests/rpc/tables/test_metadata.py b/mathesar/tests/rpc/tables/test_metadata.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mathesar/tests/rpc/tables/test_base.py b/mathesar/tests/rpc/tables/test_t_base.py similarity index 100% rename from mathesar/tests/rpc/tables/test_base.py rename to mathesar/tests/rpc/tables/test_t_base.py diff --git a/mathesar/tests/rpc/tables/test_t_metadata.py b/mathesar/tests/rpc/tables/test_t_metadata.py new file mode 100644 index 0000000000..ad12b09a61 --- /dev/null +++ b/mathesar/tests/rpc/tables/test_t_metadata.py @@ -0,0 +1,45 @@ +""" +This file tests the table metadata RPC functions. + +Fixtures: + monkeypatch(pytest): Lets you monkeypatch an object for testing. +""" +from mathesar.models.base import TableMetaData, Database, Server +from mathesar.rpc.tables import metadata + + +def test_columns_meta_data_list(monkeypatch): + database_id = 2 + schema_oid = 123456 + + def mock_get_tables_meta_data(_schema_oid, _database_id): + server_model = Server(id=2, host='example.com', port=5432) + db_model = Database(id=_database_id, name='mymathesardb', server=server_model) + return [ + TableMetaData( + database=db_model, schema_oid=_schema_oid, table_oid=1234, + import_verified=True, column_order=[8, 9, 10], preview_customized=False, + preview_template="{5555}" + ), + TableMetaData( + database=db_model, schema_oid=_schema_oid, table_oid=4567, + import_verified=False, column_order=[], preview_customized=True, + preview_template="{5512} {1223}" + ) + ] + monkeypatch.setattr(metadata, "get_tables_meta_data", mock_get_tables_meta_data) + + expect_metadata_list = [ + metadata.TableMetaData( + database_id=database_id, schema_oid=schema_oid, table_oid=1234, + import_verified=True, column_order=[8, 9, 10], preview_customized=False, + preview_template="{5555}" + ), + metadata.TableMetaData( + database_id=database_id, schema_oid=schema_oid, table_oid=4567, + import_verified=False, column_order=[], preview_customized=True, + preview_template="{5512} {1223}" + ) + ] + actual_metadata_list = metadata.list_(schema_oid=schema_oid, database_id=database_id) + assert actual_metadata_list == expect_metadata_list diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index 15b1e2ef9f..b6d1db135d 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -86,5 +86,5 @@ def create_empty_table(name, schema, comment=None): return table -def get_table_meta_data(schema_oid, database_id): +def get_tables_meta_data(schema_oid, database_id): return TableMetaData.objects.filter(database__id=database_id, schema_oid=schema_oid) From 42ef077056bbe0e4b63c387a03737552c553ce22 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 28 Jun 2024 02:25:37 +0530 Subject: [PATCH 0335/1141] fix test name --- mathesar/tests/rpc/tables/test_t_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/tests/rpc/tables/test_t_metadata.py b/mathesar/tests/rpc/tables/test_t_metadata.py index ad12b09a61..fdaf7af638 100644 --- a/mathesar/tests/rpc/tables/test_t_metadata.py +++ b/mathesar/tests/rpc/tables/test_t_metadata.py @@ -8,7 +8,7 @@ from mathesar.rpc.tables import metadata -def test_columns_meta_data_list(monkeypatch): +def test_tables_meta_data_list(monkeypatch): database_id = 2 schema_oid = 123456 From ba96a91b66e8be49e17d1ee700354c682e6c6ce7 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 28 Jun 2024 14:08:58 +0530 Subject: [PATCH 0336/1141] add defaults for min and max frac digits --- .../migrations/0009_add_column_metadata_model.py | 12 +++++++++++- mathesar/models/base.py | 4 ++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mathesar/migrations/0009_add_column_metadata_model.py b/mathesar/migrations/0009_add_column_metadata_model.py index 35481c0d02..9d511c5922 100644 --- a/mathesar/migrations/0009_add_column_metadata_model.py +++ b/mathesar/migrations/0009_add_column_metadata_model.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.11 on 2024-06-27 20:00 +# Generated by Django 4.2.11 on 2024-06-28 08:35 from django.db import migrations, models import django.db.models.deletion @@ -16,6 +16,16 @@ class Migration(migrations.Migration): name='attnum', field=models.SmallIntegerField(), ), + migrations.AlterField( + model_name='columnmetadata', + name='num_max_frac_digits', + field=models.PositiveIntegerField(default=20), + ), + migrations.AlterField( + model_name='columnmetadata', + name='num_min_frac_digits', + field=models.PositiveIntegerField(default=0), + ), migrations.AlterField( model_name='columnmetadata', name='table_oid', diff --git a/mathesar/models/base.py b/mathesar/models/base.py index c900eb176a..1291516a79 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -92,8 +92,8 @@ class ColumnMetaData(BaseModel): ) bool_true = models.CharField(default='True') bool_false = models.CharField(default='False') - num_min_frac_digits = models.PositiveIntegerField(blank=True) - num_max_frac_digits = models.PositiveIntegerField(blank=True) + num_min_frac_digits = models.PositiveIntegerField(default=0) + num_max_frac_digits = models.PositiveIntegerField(default=20) num_show_as_perc = models.BooleanField(default=False) mon_currency_symbol = models.CharField(default="$") mon_currency_location = models.CharField( From 5b8bcbf2d7f08d431b578619286176d80f819e70 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 28 Jun 2024 17:38:18 +0530 Subject: [PATCH 0337/1141] add tables.metadata.patch endpoint --- mathesar/rpc/tables/metadata.py | 41 +++++++++++++++++++- mathesar/tests/rpc/tables/test_t_metadata.py | 32 +++++++++++++-- mathesar/utils/tables.py | 10 +++++ 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/mathesar/rpc/tables/metadata.py b/mathesar/rpc/tables/metadata.py index 64b2b85891..b177314b6c 100644 --- a/mathesar/rpc/tables/metadata.py +++ b/mathesar/rpc/tables/metadata.py @@ -7,7 +7,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -from mathesar.utils.tables import get_tables_meta_data +from mathesar.utils.tables import get_tables_meta_data, patch_table_meta_data class TableMetaData(TypedDict): @@ -17,6 +17,7 @@ class TableMetaData(TypedDict): Only the `database`, `schema_oid`, and `table_oid` keys are required. Attributes: + id: The Django id of the TableMetaData object. database_id: The Django id of the database containing the table. schema_oid: The OID of the schema containing the table. table_oid: The OID of the table in the database. @@ -25,6 +26,7 @@ class TableMetaData(TypedDict): preview_customized: Specifies whether the preview has been customized. preview_template: Preview template for a referent column. """ + id: int database_id: int schema_oid: int table_oid: int @@ -36,6 +38,7 @@ class TableMetaData(TypedDict): @classmethod def from_model(cls, model): return cls( + id=model.id, database_id=model.database.id, schema_oid=model.schema_oid, table_oid=model.table_oid, @@ -46,6 +49,22 @@ def from_model(cls, model): ) +class SettableTableMetaData(TypedDict): + """ + Settable metadata fields for a table in a database. + + Attributes: + import_verified: Specifies whether a file has been successfully imported into a table. + column_order: The order in which columns of a table are displayed. + preview_customized: Specifies whether the preview has been customized. + preview_template: Preview template for a referent column. + """ + import_verified: Optional[bool] + column_order: Optional[list[int]] + preview_customized: Optional[bool] + preview_template: Optional[str] + + @rpc_method(name="tables.metadata.list") @http_basic_auth_login_required @handle_rpc_exceptions @@ -64,3 +83,23 @@ def list_(*, schema_oid: int, database_id: int, **kwargs) -> list[TableMetaData] return [ TableMetaData.from_model(model) for model in table_meta_data ] + + +@rpc_method(name="tables.metadata.patch") +@http_basic_auth_login_required +@handle_rpc_exceptions +def patch( + *, metadata_id: int, metadata_dict: SettableTableMetaData, **kwargs +) -> TableMetaData: + """ + Alter metadata settings associated with a table for a schema. + + Args: + metadata_id: Identity of the table metadata in the user's database. + metadata_dict: The dict describing desired table metadata alterations. + + Returns: + Altered metadata object. + """ + table_meta_data = patch_table_meta_data(metadata_id, metadata_dict) + return TableMetaData.from_model(table_meta_data) diff --git a/mathesar/tests/rpc/tables/test_t_metadata.py b/mathesar/tests/rpc/tables/test_t_metadata.py index fdaf7af638..012edf6f85 100644 --- a/mathesar/tests/rpc/tables/test_t_metadata.py +++ b/mathesar/tests/rpc/tables/test_t_metadata.py @@ -17,12 +17,12 @@ def mock_get_tables_meta_data(_schema_oid, _database_id): db_model = Database(id=_database_id, name='mymathesardb', server=server_model) return [ TableMetaData( - database=db_model, schema_oid=_schema_oid, table_oid=1234, + id=1, database=db_model, schema_oid=_schema_oid, table_oid=1234, import_verified=True, column_order=[8, 9, 10], preview_customized=False, preview_template="{5555}" ), TableMetaData( - database=db_model, schema_oid=_schema_oid, table_oid=4567, + id=2, database=db_model, schema_oid=_schema_oid, table_oid=4567, import_verified=False, column_order=[], preview_customized=True, preview_template="{5512} {1223}" ) @@ -31,15 +31,39 @@ def mock_get_tables_meta_data(_schema_oid, _database_id): expect_metadata_list = [ metadata.TableMetaData( - database_id=database_id, schema_oid=schema_oid, table_oid=1234, + id=1, database_id=database_id, schema_oid=schema_oid, table_oid=1234, import_verified=True, column_order=[8, 9, 10], preview_customized=False, preview_template="{5555}" ), metadata.TableMetaData( - database_id=database_id, schema_oid=schema_oid, table_oid=4567, + id=2, database_id=database_id, schema_oid=schema_oid, table_oid=4567, import_verified=False, column_order=[], preview_customized=True, preview_template="{5512} {1223}" ) ] actual_metadata_list = metadata.list_(schema_oid=schema_oid, database_id=database_id) assert actual_metadata_list == expect_metadata_list + + +def test_tables_meta_data_patch(monkeypatch): + database_id = 2 + schema_oid = 123456 + metadata_dict = {'import_verified': True, 'column_order': [1, 4, 12]} + + def mock_patch_tables_meta_data(metadata_id, metadata_dict): + server_model = Server(id=2, host='example.com', port=5432) + db_model = Database(id=database_id, name='mymathesardb', server=server_model) + return TableMetaData( + id=1, database=db_model, schema_oid=schema_oid, table_oid=1234, + import_verified=True, column_order=[1, 4, 12], preview_customized=False, + preview_template="{5555}" + ) + monkeypatch.setattr(metadata, "patch_table_meta_data", mock_patch_tables_meta_data) + + expect_metadata_object = metadata.TableMetaData( + id=1, database_id=database_id, schema_oid=schema_oid, table_oid=1234, + import_verified=True, column_order=[1, 4, 12], preview_customized=False, + preview_template="{5555}" + ) + actual_metadata_object = metadata.patch(metadata_id=1, metadata_dict=metadata_dict) + assert actual_metadata_object == expect_metadata_object diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index b6d1db135d..1951f8bc19 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -88,3 +88,13 @@ def create_empty_table(name, schema, comment=None): def get_tables_meta_data(schema_oid, database_id): return TableMetaData.objects.filter(database__id=database_id, schema_oid=schema_oid) + + +def patch_table_meta_data(metadata_id, metadata_dict): + metadata_model = TableMetaData.objects.get(id=metadata_id) + metadata_model.import_verified = metadata_dict.get('import_verified', metadata_model.import_verified) + metadata_model.column_order = metadata_dict.get('column_order', metadata_model.column_order) + metadata_model.preview_customized = metadata_dict.get('preview_customized', metadata_model.preview_customized) + metadata_model.preview_template = metadata_dict.get('preview_template', metadata_model.preview_template) + metadata_model.save() + return metadata_model From c7a98eb525ad3a547a43e6a2904eadcac0055f55 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 28 Jun 2024 17:40:49 +0530 Subject: [PATCH 0338/1141] add test for patch endpoint --- mathesar/tests/rpc/test_endpoints.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 9190647883..b786e99027 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -113,6 +113,11 @@ tables.metadata.list_, "tables.metadata.list", [user_is_authenticated] + ), + ( + tables.metadata.patch, + "tables.metadata.patch", + [user_is_authenticated] ) ] From b6a899edb2f4435c645526fb1a555823b0d8d1af Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 28 Jun 2024 17:46:31 +0530 Subject: [PATCH 0339/1141] add docs --- docs/docs/api/rpc.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index aa3cd5e7f7..f3fd42eb25 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -74,6 +74,16 @@ To use an RPC function: - TableInfo - SettableTableInfo +## Table Metadata + +::: tables.metadata + options: + members: + - list_ + - patch + - TableMetaData + - SettableTableMetaData + ## Columns ::: columns From 6d171717641720b0edc361e3ff9cf7ada699661c Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 1 Jul 2024 23:41:33 +0530 Subject: [PATCH 0340/1141] rm schema_oid, rename preview_* to record_summary_*, refactor patch to use table_oid instead of metadata_id --- .../0009_add_column_metadata_model.py | 9 ++-- mathesar/models/base.py | 7 ++- mathesar/rpc/tables/metadata.py | 41 ++++++++-------- mathesar/tests/rpc/tables/test_t_metadata.py | 48 +++++++++---------- mathesar/utils/tables.py | 15 +++--- 5 files changed, 56 insertions(+), 64 deletions(-) diff --git a/mathesar/migrations/0009_add_column_metadata_model.py b/mathesar/migrations/0009_add_column_metadata_model.py index 9d511c5922..6b53e15f4f 100644 --- a/mathesar/migrations/0009_add_column_metadata_model.py +++ b/mathesar/migrations/0009_add_column_metadata_model.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.11 on 2024-06-28 08:35 +# Generated by Django 4.2.11 on 2024-07-01 18:07 from django.db import migrations, models import django.db.models.deletion @@ -37,17 +37,16 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), - ('schema_oid', models.PositiveBigIntegerField()), ('table_oid', models.PositiveBigIntegerField()), ('import_verified', models.BooleanField(default=False)), ('column_order', models.JSONField(default=list)), - ('preview_customized', models.BooleanField(default=False)), - ('preview_template', models.CharField(blank=True, max_length=255)), + ('record_summary_customized', models.BooleanField(default=False)), + ('record_summary_template', models.CharField(blank=True, max_length=255)), ('database', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mathesar.database')), ], ), migrations.AddConstraint( model_name='tablemetadata', - constraint=models.UniqueConstraint(fields=('database', 'schema_oid', 'table_oid'), name='unique_table_metadata'), + constraint=models.UniqueConstraint(fields=('database', 'table_oid'), name='unique_table_metadata'), ), ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 1291516a79..1ea3b0be70 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -125,17 +125,16 @@ class Meta: class TableMetaData(BaseModel): database = models.ForeignKey('Database', on_delete=models.CASCADE) - schema_oid = models.PositiveBigIntegerField() table_oid = models.PositiveBigIntegerField() import_verified = models.BooleanField(default=False) column_order = models.JSONField(default=list) - preview_customized = models.BooleanField(default=False) - preview_template = models.CharField(max_length=255, blank=True) + record_summary_customized = models.BooleanField(default=False) + record_summary_template = models.CharField(max_length=255, blank=True) class Meta: constraints = [ models.UniqueConstraint( - fields=["database", "schema_oid", "table_oid"], + fields=["database", "table_oid"], name="unique_table_metadata" ) ] diff --git a/mathesar/rpc/tables/metadata.py b/mathesar/rpc/tables/metadata.py index b177314b6c..2bc28382c8 100644 --- a/mathesar/rpc/tables/metadata.py +++ b/mathesar/rpc/tables/metadata.py @@ -14,38 +14,35 @@ class TableMetaData(TypedDict): """ Metadata for a table in a database. - Only the `database`, `schema_oid`, and `table_oid` keys are required. + Only the `database` and `table_oid` keys are required. Attributes: id: The Django id of the TableMetaData object. database_id: The Django id of the database containing the table. - schema_oid: The OID of the schema containing the table. table_oid: The OID of the table in the database. import_verified: Specifies whether a file has been successfully imported into a table. column_order: The order in which columns of a table are displayed. - preview_customized: Specifies whether the preview has been customized. - preview_template: Preview template for a referent column. + record_summary_customized: Specifies whether the record summary has been customized. + record_summary_template: Record summary template for a referent column. """ id: int database_id: int - schema_oid: int table_oid: int import_verified: Optional[bool] column_order: Optional[list[int]] - preview_customized: Optional[bool] - preview_template: Optional[str] + record_summary_customized: Optional[bool] + record_summary_template: Optional[str] @classmethod def from_model(cls, model): return cls( id=model.id, database_id=model.database.id, - schema_oid=model.schema_oid, table_oid=model.table_oid, import_verified=model.import_verified, column_order=model.column_order, - preview_customized=model.preview_customized, - preview_template=model.preview_template, + record_summary_customized=model.record_summary_customized, + record_summary_template=model.record_summary_template, ) @@ -56,30 +53,29 @@ class SettableTableMetaData(TypedDict): Attributes: import_verified: Specifies whether a file has been successfully imported into a table. column_order: The order in which columns of a table are displayed. - preview_customized: Specifies whether the preview has been customized. - preview_template: Preview template for a referent column. + record_summary_customized: Specifies whether the record summary has been customized. + record_summary_template: Record summary template for a referent column. """ import_verified: Optional[bool] column_order: Optional[list[int]] - preview_customized: Optional[bool] - preview_template: Optional[str] + record_summary_customized: Optional[bool] + record_summary_template: Optional[str] @rpc_method(name="tables.metadata.list") @http_basic_auth_login_required @handle_rpc_exceptions -def list_(*, schema_oid: int, database_id: int, **kwargs) -> list[TableMetaData]: +def list_(*, database_id: int, **kwargs) -> list[TableMetaData]: """ - List metadata associated with tables for a schema. + List metadata associated with tables for a database. Args: - schema_oid: Identity of the schema in the user's database. database_id: The Django id of the database containing the table. Returns: Metadata object for a given table oid. """ - table_meta_data = get_tables_meta_data(schema_oid, database_id) + table_meta_data = get_tables_meta_data(database_id) return [ TableMetaData.from_model(model) for model in table_meta_data ] @@ -89,17 +85,18 @@ def list_(*, schema_oid: int, database_id: int, **kwargs) -> list[TableMetaData] @http_basic_auth_login_required @handle_rpc_exceptions def patch( - *, metadata_id: int, metadata_dict: SettableTableMetaData, **kwargs + *, table_oid: int, metadata_dict: SettableTableMetaData, database_id: int, **kwargs ) -> TableMetaData: """ - Alter metadata settings associated with a table for a schema. + Alter metadata settings associated with a table for a database. Args: - metadata_id: Identity of the table metadata in the user's database. + table_oid: Identity of the table whose metadata we'll modify. metadata_dict: The dict describing desired table metadata alterations. + database_id: The Django id of the database containing the table. Returns: Altered metadata object. """ - table_meta_data = patch_table_meta_data(metadata_id, metadata_dict) + table_meta_data = patch_table_meta_data(table_oid, metadata_dict, database_id) return TableMetaData.from_model(table_meta_data) diff --git a/mathesar/tests/rpc/tables/test_t_metadata.py b/mathesar/tests/rpc/tables/test_t_metadata.py index 012edf6f85..faad49b703 100644 --- a/mathesar/tests/rpc/tables/test_t_metadata.py +++ b/mathesar/tests/rpc/tables/test_t_metadata.py @@ -10,60 +10,58 @@ def test_tables_meta_data_list(monkeypatch): database_id = 2 - schema_oid = 123456 - def mock_get_tables_meta_data(_schema_oid, _database_id): + def mock_get_tables_meta_data(_database_id): server_model = Server(id=2, host='example.com', port=5432) db_model = Database(id=_database_id, name='mymathesardb', server=server_model) return [ TableMetaData( - id=1, database=db_model, schema_oid=_schema_oid, table_oid=1234, - import_verified=True, column_order=[8, 9, 10], preview_customized=False, - preview_template="{5555}" + id=1, database=db_model, table_oid=1234, + import_verified=True, column_order=[8, 9, 10], record_summary_customized=False, + record_summary_template="{5555}" ), TableMetaData( - id=2, database=db_model, schema_oid=_schema_oid, table_oid=4567, - import_verified=False, column_order=[], preview_customized=True, - preview_template="{5512} {1223}" + id=2, database=db_model, table_oid=4567, + import_verified=False, column_order=[], record_summary_customized=True, + record_summary_template="{5512} {1223}" ) ] monkeypatch.setattr(metadata, "get_tables_meta_data", mock_get_tables_meta_data) expect_metadata_list = [ metadata.TableMetaData( - id=1, database_id=database_id, schema_oid=schema_oid, table_oid=1234, - import_verified=True, column_order=[8, 9, 10], preview_customized=False, - preview_template="{5555}" + id=1, database_id=database_id, table_oid=1234, + import_verified=True, column_order=[8, 9, 10], record_summary_customized=False, + record_summary_template="{5555}" ), metadata.TableMetaData( - id=2, database_id=database_id, schema_oid=schema_oid, table_oid=4567, - import_verified=False, column_order=[], preview_customized=True, - preview_template="{5512} {1223}" + id=2, database_id=database_id, table_oid=4567, + import_verified=False, column_order=[], record_summary_customized=True, + record_summary_template="{5512} {1223}" ) ] - actual_metadata_list = metadata.list_(schema_oid=schema_oid, database_id=database_id) + actual_metadata_list = metadata.list_(database_id=database_id) assert actual_metadata_list == expect_metadata_list def test_tables_meta_data_patch(monkeypatch): database_id = 2 - schema_oid = 123456 metadata_dict = {'import_verified': True, 'column_order': [1, 4, 12]} - def mock_patch_tables_meta_data(metadata_id, metadata_dict): + def mock_patch_tables_meta_data(table_oid, metadata_dict, _database_id): server_model = Server(id=2, host='example.com', port=5432) - db_model = Database(id=database_id, name='mymathesardb', server=server_model) + db_model = Database(id=_database_id, name='mymathesardb', server=server_model) return TableMetaData( - id=1, database=db_model, schema_oid=schema_oid, table_oid=1234, - import_verified=True, column_order=[1, 4, 12], preview_customized=False, - preview_template="{5555}" + id=1, database=db_model, table_oid=1234, + import_verified=True, column_order=[1, 4, 12], record_summary_customized=False, + record_summary_template="{5555}" ) monkeypatch.setattr(metadata, "patch_table_meta_data", mock_patch_tables_meta_data) expect_metadata_object = metadata.TableMetaData( - id=1, database_id=database_id, schema_oid=schema_oid, table_oid=1234, - import_verified=True, column_order=[1, 4, 12], preview_customized=False, - preview_template="{5555}" + id=1, database_id=database_id, table_oid=1234, + import_verified=True, column_order=[1, 4, 12], record_summary_customized=False, + record_summary_template="{5555}" ) - actual_metadata_object = metadata.patch(metadata_id=1, metadata_dict=metadata_dict) + actual_metadata_object = metadata.patch(table_oid=1234, metadata_dict=metadata_dict, database_id=2) assert actual_metadata_object == expect_metadata_object diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index 1951f8bc19..8b7875ea20 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -86,15 +86,14 @@ def create_empty_table(name, schema, comment=None): return table -def get_tables_meta_data(schema_oid, database_id): - return TableMetaData.objects.filter(database__id=database_id, schema_oid=schema_oid) +def get_tables_meta_data(database_id): + return TableMetaData.objects.filter(database__id=database_id) -def patch_table_meta_data(metadata_id, metadata_dict): - metadata_model = TableMetaData.objects.get(id=metadata_id) - metadata_model.import_verified = metadata_dict.get('import_verified', metadata_model.import_verified) - metadata_model.column_order = metadata_dict.get('column_order', metadata_model.column_order) - metadata_model.preview_customized = metadata_dict.get('preview_customized', metadata_model.preview_customized) - metadata_model.preview_template = metadata_dict.get('preview_template', metadata_model.preview_template) +def patch_table_meta_data(table_oid, metadata_dict, database_id): + metadata_model = TableMetaData.objects.get(database__id=database_id, table_oid=table_oid) + for field, value in metadata_dict.items(): + if hasattr(metadata_model, field): + setattr(metadata_model, field, value) metadata_model.save() return metadata_model From 55cc2a9185607732c5a45d8e59d325b1265c9838 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 1 Jul 2024 15:11:55 -0400 Subject: [PATCH 0341/1141] Refactor schemas APIs from REST to RPC --- mathesar_ui/src/AppTypes.ts | 12 -- mathesar_ui/src/api/rest/schemas.ts | 50 ------- mathesar_ui/src/api/rest/types/queries.ts | 4 +- mathesar_ui/src/api/rest/users.ts | 7 +- mathesar_ui/src/api/rpc/index.ts | 2 + mathesar_ui/src/api/rpc/schemas.ts | 47 +++++++ mathesar_ui/src/components/AppHeader.svelte | 6 +- mathesar_ui/src/components/SchemaName.svelte | 4 +- .../breadcrumb/BreadcrumbItem.svelte | 8 +- .../breadcrumb/EntitySelector.svelte | 11 +- .../breadcrumb/SchemaSelector.svelte | 13 +- .../components/breadcrumb/breadcrumbTypes.ts | 11 +- .../data-explorer/DataExplorerPage.svelte | 9 +- .../pages/database/AddEditSchemaModal.svelte | 5 +- .../src/pages/database/DatabaseDetails.svelte | 19 +-- .../database/SchemaConstituentCounts.svelte | 20 +-- .../src/pages/database/SchemaRow.svelte | 7 +- .../pages/exploration/ExplorationPage.svelte | 7 +- .../src/pages/exploration/Header.svelte | 7 +- .../preview/ImportPreviewContent.svelte | 11 +- .../import/preview/ImportPreviewPage.svelte | 7 +- .../import/preview/importPreviewPageUtils.ts | 7 +- .../import/upload/ImportUploadPage.svelte | 7 +- .../schema/CreateEmptyTableButton.svelte | 7 +- .../CreateNewExplorationTutorial.svelte | 7 +- .../pages/schema/CreateNewTableButton.svelte | 9 +- .../schema/CreateNewTableTutorial.svelte | 7 +- .../src/pages/schema/ExplorationItem.svelte | 7 +- .../src/pages/schema/ExplorationsList.svelte | 5 +- .../schema/SchemaAccessControlModal.svelte | 5 +- .../pages/schema/SchemaExplorations.svelte | 7 +- .../src/pages/schema/SchemaOverview.svelte | 13 +- .../src/pages/schema/SchemaPage.svelte | 13 +- .../src/pages/schema/SchemaTables.svelte | 5 +- mathesar_ui/src/pages/schema/TableCard.svelte | 11 +- .../src/pages/schema/TablesList.svelte | 5 +- .../src/routes/DataExplorerRoute.svelte | 13 +- .../src/routes/ExplorationRoute.svelte | 5 +- mathesar_ui/src/routes/ImportRoute.svelte | 7 +- mathesar_ui/src/routes/RecordPageRoute.svelte | 5 +- mathesar_ui/src/routes/TableRoute.svelte | 5 +- mathesar_ui/src/stores/queries.ts | 18 ++- mathesar_ui/src/stores/schemas.ts | 122 ++++++++---------- mathesar_ui/src/stores/storeBasedUrls.ts | 4 +- mathesar_ui/src/stores/tables.ts | 27 ++-- mathesar_ui/src/stores/users.ts | 35 +++-- .../result-pane/QueryRunErrors.svelte | 2 +- .../table-inspector/table/TableActions.svelte | 6 +- mathesar_ui/src/utils/preloadData.ts | 5 +- 49 files changed, 313 insertions(+), 323 deletions(-) delete mode 100644 mathesar_ui/src/api/rest/schemas.ts create mode 100644 mathesar_ui/src/api/rpc/schemas.ts diff --git a/mathesar_ui/src/AppTypes.ts b/mathesar_ui/src/AppTypes.ts index 168d51b58d..a779ca063e 100644 --- a/mathesar_ui/src/AppTypes.ts +++ b/mathesar_ui/src/AppTypes.ts @@ -1,5 +1,3 @@ -import type { TreeItem } from '@mathesar-component-library/types'; - /** @deprecated in favor of Connection */ export interface Database { id: number; @@ -16,16 +14,6 @@ export interface DBObjectEntry { description: string | null; } -export interface SchemaEntry extends DBObjectEntry { - has_dependencies: boolean; - num_tables: number; - num_queries: number; -} - -export interface SchemaResponse extends SchemaEntry, TreeItem { - tables: DBObjectEntry[]; -} - export type DbType = string; export interface FilterConfiguration { diff --git a/mathesar_ui/src/api/rest/schemas.ts b/mathesar_ui/src/api/rest/schemas.ts deleted file mode 100644 index 9940035c0b..0000000000 --- a/mathesar_ui/src/api/rest/schemas.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { SchemaEntry, SchemaResponse } from '@mathesar/AppTypes'; - -import type { Connection } from './connections'; -import { - type PaginatedResponse, - deleteAPI, - getAPI, - patchAPI, - postAPI, -} from './utils/requestUtils'; - -function list(connectionId: Connection['id']) { - return getAPI>( - `/api/db/v0/schemas/?connection_id=${connectionId}&limit=500`, - ); -} - -function add(props: { - name: SchemaEntry['name']; - description: SchemaEntry['description']; - connectionId: Connection['id']; -}) { - return postAPI('/api/db/v0/schemas/', { - name: props.name, - description: props.description, - connection_id: props.connectionId, - }); -} - -function update(schema: { - id: SchemaEntry['id']; - name?: SchemaEntry['name']; - description?: SchemaEntry['description']; -}) { - return patchAPI(`/api/db/v0/schemas/${schema.id}/`, { - name: schema.name, - description: schema.description, - }); -} - -function deleteSchema(schemaId: SchemaEntry['id']) { - return deleteAPI(`/api/db/v0/schemas/${schemaId}/`); -} - -export default { - list, - add, - update, - delete: deleteSchema, -}; diff --git a/mathesar_ui/src/api/rest/types/queries.ts b/mathesar_ui/src/api/rest/types/queries.ts index 713f32c96a..6517eef56a 100644 --- a/mathesar_ui/src/api/rest/types/queries.ts +++ b/mathesar_ui/src/api/rest/types/queries.ts @@ -1,7 +1,7 @@ import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { JpPath } from '@mathesar/api/rest/types/tables/joinable_tables'; import type { PaginatedResponse } from '@mathesar/api/rest/utils/requestUtils'; -import type { SchemaEntry } from '@mathesar/AppTypes'; +import type { Schema } from '@mathesar/api/rpc/schemas'; export type QueryColumnAlias = string; @@ -182,7 +182,7 @@ export interface QueryResultsResponse { export interface QueryRunResponse extends QueryResultsResponse { query: { - schema: SchemaEntry['id']; + schema: Schema['oid']; base_table: QueryInstance['base_table']; initial_columns: QueryInstanceInitialColumn[]; transformations?: QueryInstanceTransformation[]; diff --git a/mathesar_ui/src/api/rest/users.ts b/mathesar_ui/src/api/rest/users.ts index 1020ccc448..477d881d68 100644 --- a/mathesar_ui/src/api/rest/users.ts +++ b/mathesar_ui/src/api/rest/users.ts @@ -1,4 +1,5 @@ -import type { Database, SchemaEntry } from '@mathesar/AppTypes'; +import type { Schema } from '@mathesar/api/rpc/schemas'; +import type { Database } from '@mathesar/AppTypes'; import type { Language } from '@mathesar/i18n/languages/utils'; import { @@ -27,7 +28,7 @@ export interface DatabaseRole { export interface SchemaRole { id: number; - schema: SchemaEntry['id']; + schema: Schema['oid']; role: UserRole; } @@ -92,7 +93,7 @@ function deleteDatabaseRole(roleId: DatabaseRole['id']) { function addSchemaRole( userId: User['id'], - schemaId: SchemaEntry['id'], + schemaId: Schema['oid'], role: UserRole, ) { return postAPI('/api/ui/v0/schema_roles/', { diff --git a/mathesar_ui/src/api/rpc/index.ts b/mathesar_ui/src/api/rpc/index.ts index 0ed7c027c2..e9a3b3c531 100644 --- a/mathesar_ui/src/api/rpc/index.ts +++ b/mathesar_ui/src/api/rpc/index.ts @@ -3,6 +3,7 @@ import Cookies from 'js-cookie'; import { buildRpcApi } from '@mathesar/packages/json-rpc-client-builder'; import { connections } from './connections'; +import { schemas } from './schemas'; /** Mathesar's JSON-RPC API */ export const api = buildRpcApi({ @@ -10,5 +11,6 @@ export const api = buildRpcApi({ getHeaders: () => ({ 'X-CSRFToken': Cookies.get('csrftoken') }), methodTree: { connections, + schemas, }, }); diff --git a/mathesar_ui/src/api/rpc/schemas.ts b/mathesar_ui/src/api/rpc/schemas.ts new file mode 100644 index 0000000000..486a7d4728 --- /dev/null +++ b/mathesar_ui/src/api/rpc/schemas.ts @@ -0,0 +1,47 @@ +import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; + +export interface Schema { + oid: number; + name: string; + description: string; + table_count: number; +} + +export const schemas = { + list: rpcMethodTypeContainer< + { + database_id: number; + }, + Schema[] + >(), + + /** Returns the OID of the newly-created schema */ + add: rpcMethodTypeContainer< + { + database_id: number; + name: string; + description?: string; + }, + number + >(), + + patch: rpcMethodTypeContainer< + { + database_id: number; + schema_oid: number; + patch: { + name?: string; + description?: string; + }; + }, + void + >(), + + delete: rpcMethodTypeContainer< + { + database_id: number; + schema_oid: number; + }, + void + >(), +}; diff --git a/mathesar_ui/src/components/AppHeader.svelte b/mathesar_ui/src/components/AppHeader.svelte index 0fa38e1483..3bc094ebca 100644 --- a/mathesar_ui/src/components/AppHeader.svelte +++ b/mathesar_ui/src/components/AppHeader.svelte @@ -62,7 +62,7 @@ isCreatingNewEmptyTable = true; const tableInfo = await createTable(database, schema, {}); isCreatingNewEmptyTable = false; - router.goto(getTablePageUrl(database.id, schema.id, tableInfo.id), false); + router.goto(getTablePageUrl(database.id, schema.oid, tableInfo.id), false); } @@ -90,14 +90,14 @@ {$_('new_table_from_data_import')} {/if} {$_('open_data_explorer')} diff --git a/mathesar_ui/src/components/SchemaName.svelte b/mathesar_ui/src/components/SchemaName.svelte index e6fa7cf638..d3a0199027 100644 --- a/mathesar_ui/src/components/SchemaName.svelte +++ b/mathesar_ui/src/components/SchemaName.svelte @@ -1,10 +1,10 @@ diff --git a/mathesar_ui/src/pages/database/AddEditSchemaModal.svelte b/mathesar_ui/src/pages/database/AddEditSchemaModal.svelte index 734923fb38..477ec9ee99 100644 --- a/mathesar_ui/src/pages/database/AddEditSchemaModal.svelte +++ b/mathesar_ui/src/pages/database/AddEditSchemaModal.svelte @@ -2,7 +2,8 @@
-

- {$_('count_tables', { values: { count: schema.num_tables } })} -

- -

- {$_('count_explorations', { values: { count: schema.num_queries } })} +

+ {$_('count_tables', { values: { count: schema.table_count } })}

diff --git a/mathesar_ui/src/pages/database/SchemaRow.svelte b/mathesar_ui/src/pages/database/SchemaRow.svelte index 6eb6c24f72..534cec6534 100644 --- a/mathesar_ui/src/pages/database/SchemaRow.svelte +++ b/mathesar_ui/src/pages/database/SchemaRow.svelte @@ -2,7 +2,8 @@ import { createEventDispatcher } from 'svelte'; import { _ } from 'svelte-i18n'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import DropdownMenu from '@mathesar/component-library/dropdown-menu/DropdownMenu.svelte'; import MenuDivider from '@mathesar/component-library/menu/MenuDivider.svelte'; import InfoBox from '@mathesar/components/message-boxes/InfoBox.svelte'; @@ -21,13 +22,13 @@ const dispatch = createEventDispatcher(); export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let canExecuteDDL = true; let isHovered = false; let isFocused = false; - $: href = getSchemaPageUrl(database.id, schema.id); + $: href = getSchemaPageUrl(database.id, schema.oid); $: isDefault = schema.name === 'public'; $: isLocked = schema.name === 'public'; diff --git a/mathesar_ui/src/pages/exploration/ExplorationPage.svelte b/mathesar_ui/src/pages/exploration/ExplorationPage.svelte index c5372c4f36..be676788a6 100644 --- a/mathesar_ui/src/pages/exploration/ExplorationPage.svelte +++ b/mathesar_ui/src/pages/exploration/ExplorationPage.svelte @@ -3,7 +3,8 @@ import { router } from 'tinro'; import type { QueryInstance } from '@mathesar/api/rest/types/queries'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import LayoutWithHeader from '@mathesar/layouts/LayoutWithHeader.svelte'; import { getSchemaPageUrl } from '@mathesar/routes/urls'; import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; @@ -22,7 +23,7 @@ const userProfile = getUserProfileStoreFromContext(); export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let query: QueryInstance; export let shareConsumer: ShareConsumer | undefined = undefined; @@ -51,7 +52,7 @@ $: createQueryRunner(query, $currentDbAbstractTypes.data); function gotoSchemaPage() { - router.goto(getSchemaPageUrl(database.id, schema.id)); + router.goto(getSchemaPageUrl(database.id, schema.oid)); } diff --git a/mathesar_ui/src/pages/exploration/Header.svelte b/mathesar_ui/src/pages/exploration/Header.svelte index 5a96c39484..d3ec06a273 100644 --- a/mathesar_ui/src/pages/exploration/Header.svelte +++ b/mathesar_ui/src/pages/exploration/Header.svelte @@ -2,7 +2,8 @@ import { _ } from 'svelte-i18n'; import type { QueryInstance } from '@mathesar/api/rest/types/queries'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import EntityPageHeader from '@mathesar/components/EntityPageHeader.svelte'; import { iconExploration, iconInspector } from '@mathesar/icons'; import { getExplorationEditorPageUrl } from '@mathesar/routes/urls'; @@ -11,7 +12,7 @@ import ShareExplorationDropdown from './ShareExplorationDropdown.svelte'; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let query: QueryInstance; export let isInspectorOpen = true; export let canEditMetadata: boolean; @@ -30,7 +31,7 @@ {#if canEditMetadata} {$_('edit_in_data_explorer')} diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte index b6359e8e2c..838161b297 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte @@ -6,7 +6,8 @@ import type { DataFile } from '@mathesar/api/rest/types/dataFiles'; import type { TableEntry } from '@mathesar/api/rest/types/tables'; import type { Column } from '@mathesar/api/rest/types/tables/columns'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import { Field, FieldLayout, @@ -60,7 +61,7 @@ const cancelationRequest = makeDeleteTableRequest(); export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let table: TableEntry; export let dataFile: DataFile; export let useColumnTypeInference = false; @@ -108,7 +109,7 @@ }) { const tableId = props.table?.id ?? table.id; router.goto( - getImportPreviewPageUrl(database.id, schema.id, tableId, { + getImportPreviewPageUrl(database.id, schema.oid, tableId, { useColumnTypeInference: props.useColumnTypeInference ?? useColumnTypeInference, }), @@ -146,7 +147,7 @@ async function cancel() { const response = await cancelationRequest.run({ database, schema, table }); if (response.isOk) { - router.goto(getSchemaPageUrl(database.id, schema.id), true); + router.goto(getSchemaPageUrl(database.id, schema.oid), true); } else { toast.fromError(response.error); } @@ -159,7 +160,7 @@ import_verified: true, columns: finalizeColumns(columns, columnPropertiesMap), }); - router.goto(getTablePageUrl(database.id, schema.id, table.id), true); + router.goto(getTablePageUrl(database.id, schema.oid, table.id), true); } catch (err) { toast.fromError(err); } diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewPage.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewPage.svelte index d67efa353b..775b6d0712 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewPage.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewPage.svelte @@ -3,7 +3,8 @@ import { router } from 'tinro'; import { dataFilesApi } from '@mathesar/api/rest/dataFiles'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import ErrorBox from '@mathesar/components/message-boxes/ErrorBox.svelte'; import { makeSimplePageTitle } from '@mathesar/pages/pageTitleUtils'; import { getTablePageUrl } from '@mathesar/routes/urls'; @@ -19,12 +20,12 @@ const dataFileFetch = new AsyncStore(dataFilesApi.get); export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let tableId: number; export let useColumnTypeInference = false; function redirectToTablePage() { - router.goto(getTablePageUrl(database.id, schema.id, tableId)); + router.goto(getTablePageUrl(database.id, schema.oid, tableId)); } $: void (async () => { diff --git a/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts b/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts index 262ae59480..6d7474ca8c 100644 --- a/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts +++ b/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts @@ -5,7 +5,8 @@ import type { TableEntry, } from '@mathesar/api/rest/types/tables'; import type { Column } from '@mathesar/api/rest/types/tables/columns'; -import type { Database, SchemaEntry } from '@mathesar/AppTypes'; +import type { Schema } from '@mathesar/api/rpc/schemas'; +import type { Database } from '@mathesar/AppTypes'; import { getCellCap } from '@mathesar/components/cell-fabric/utils'; import { getAbstractTypeForDbType } from '@mathesar/stores/abstract-types'; import type { @@ -51,7 +52,7 @@ export function processColumns( export function makeHeaderUpdateRequest() { interface Props { database: Database; - schema: SchemaEntry; + schema: Schema; table: Pick; dataFile: Pick; firstRowIsHeader: boolean; @@ -73,7 +74,7 @@ export function makeHeaderUpdateRequest() { export function makeDeleteTableRequest() { interface Props { database: Database; - schema: SchemaEntry; + schema: Schema; table: Pick; } return new AsyncStore((props: Props) => diff --git a/mathesar_ui/src/pages/import/upload/ImportUploadPage.svelte b/mathesar_ui/src/pages/import/upload/ImportUploadPage.svelte index 35704c6d47..64c670d4d1 100644 --- a/mathesar_ui/src/pages/import/upload/ImportUploadPage.svelte +++ b/mathesar_ui/src/pages/import/upload/ImportUploadPage.svelte @@ -4,7 +4,8 @@ import { dataFilesApi } from '@mathesar/api/rest/dataFiles'; import type { RequestStatus } from '@mathesar/api/rest/utils/requestUtils'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import Spinner from '@mathesar/component-library/spinner/Spinner.svelte'; import DocsLink from '@mathesar/components/DocsLink.svelte'; import { @@ -37,7 +38,7 @@ import DataFileInput from './DataFileInput.svelte'; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; interface UploadMethod { key: 'file' | 'url' | 'clipboard'; @@ -117,7 +118,7 @@ }); const previewPage = getImportPreviewPageUrl( database.id, - schema.id, + schema.oid, table.id, { useColumnTypeInference: $useColumnTypeInference }, ); diff --git a/mathesar_ui/src/pages/schema/CreateEmptyTableButton.svelte b/mathesar_ui/src/pages/schema/CreateEmptyTableButton.svelte index 1d95dcea92..6e288c5312 100644 --- a/mathesar_ui/src/pages/schema/CreateEmptyTableButton.svelte +++ b/mathesar_ui/src/pages/schema/CreateEmptyTableButton.svelte @@ -1,13 +1,14 @@ diff --git a/mathesar_ui/src/pages/schema/CreateNewExplorationTutorial.svelte b/mathesar_ui/src/pages/schema/CreateNewExplorationTutorial.svelte index 9592e2677a..ea2eebf2f6 100644 --- a/mathesar_ui/src/pages/schema/CreateNewExplorationTutorial.svelte +++ b/mathesar_ui/src/pages/schema/CreateNewExplorationTutorial.svelte @@ -1,12 +1,13 @@ @@ -18,7 +19,7 @@ {$_('open_data_explorer')} diff --git a/mathesar_ui/src/pages/schema/CreateNewTableButton.svelte b/mathesar_ui/src/pages/schema/CreateNewTableButton.svelte index 2bb296e493..9a22b3733e 100644 --- a/mathesar_ui/src/pages/schema/CreateNewTableButton.svelte +++ b/mathesar_ui/src/pages/schema/CreateNewTableButton.svelte @@ -2,7 +2,8 @@ import { _ } from 'svelte-i18n'; import { router } from 'tinro'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import Icon from '@mathesar/component-library/icon/Icon.svelte'; import LinkMenuItem from '@mathesar/component-library/menu/LinkMenuItem.svelte'; import { iconAddNew } from '@mathesar/icons'; @@ -15,7 +16,7 @@ } from '@mathesar-component-library'; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; let isCreatingNewTable = false; @@ -23,7 +24,7 @@ isCreatingNewTable = true; const tableInfo = await createTable(database, schema, {}); isCreatingNewTable = false; - router.goto(getTablePageUrl(database.id, schema.id, tableInfo.id), false); + router.goto(getTablePageUrl(database.id, schema.oid, tableInfo.id), false); } @@ -44,7 +45,7 @@ {$_('from_scratch')} - + {$_('from_data_import')} diff --git a/mathesar_ui/src/pages/schema/CreateNewTableTutorial.svelte b/mathesar_ui/src/pages/schema/CreateNewTableTutorial.svelte index 230dcfd739..241bb17bb9 100644 --- a/mathesar_ui/src/pages/schema/CreateNewTableTutorial.svelte +++ b/mathesar_ui/src/pages/schema/CreateNewTableTutorial.svelte @@ -1,14 +1,15 @@ @@ -22,7 +23,7 @@ {$_('from_scratch')} - + {$_('import_from_file')}
diff --git a/mathesar_ui/src/pages/schema/ExplorationItem.svelte b/mathesar_ui/src/pages/schema/ExplorationItem.svelte index ade361c335..dc2a346478 100644 --- a/mathesar_ui/src/pages/schema/ExplorationItem.svelte +++ b/mathesar_ui/src/pages/schema/ExplorationItem.svelte @@ -2,7 +2,8 @@ import { _ } from 'svelte-i18n'; import type { QueryInstance } from '@mathesar/api/rest/types/queries'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import TableName from '@mathesar/components/TableName.svelte'; import { iconExploration } from '@mathesar/icons'; import { getExplorationPageUrl } from '@mathesar/routes/urls'; @@ -11,14 +12,14 @@ export let exploration: QueryInstance; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; $: baseTable = $tablesStore.data.get(exploration.base_table);
diff --git a/mathesar_ui/src/pages/schema/ExplorationsList.svelte b/mathesar_ui/src/pages/schema/ExplorationsList.svelte index 6dd223f709..8fa27682a5 100644 --- a/mathesar_ui/src/pages/schema/ExplorationsList.svelte +++ b/mathesar_ui/src/pages/schema/ExplorationsList.svelte @@ -2,7 +2,8 @@ import { _ } from 'svelte-i18n'; import type { QueryInstance } from '@mathesar/api/rest/types/queries'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import { iconExploration } from '@mathesar/icons'; import EmptyEntity from './EmptyEntity.svelte'; @@ -10,7 +11,7 @@ export let explorations: QueryInstance[]; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let bordered = true; diff --git a/mathesar_ui/src/pages/schema/SchemaAccessControlModal.svelte b/mathesar_ui/src/pages/schema/SchemaAccessControlModal.svelte index cb22a2a557..b4dc77cf9c 100644 --- a/mathesar_ui/src/pages/schema/SchemaAccessControlModal.svelte +++ b/mathesar_ui/src/pages/schema/SchemaAccessControlModal.svelte @@ -2,7 +2,8 @@ import { _ } from 'svelte-i18n'; import type { UserRole } from '@mathesar/api/rest/users'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import Identifier from '@mathesar/components/Identifier.svelte'; import ErrorBox from '@mathesar/components/message-boxes/ErrorBox.svelte'; import { RichText } from '@mathesar/components/rich-text'; @@ -19,7 +20,7 @@ export let controller: ModalController; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; const usersStore = setUsersStoreInContext(); const { requestStatus } = usersStore; diff --git a/mathesar_ui/src/pages/schema/SchemaExplorations.svelte b/mathesar_ui/src/pages/schema/SchemaExplorations.svelte index 4cfee93bce..bf947784f2 100644 --- a/mathesar_ui/src/pages/schema/SchemaExplorations.svelte +++ b/mathesar_ui/src/pages/schema/SchemaExplorations.svelte @@ -2,7 +2,8 @@ import { _ } from 'svelte-i18n'; import type { QueryInstance } from '@mathesar/api/rest/types/queries'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import EntityContainerWithFilterBar from '@mathesar/components/EntityContainerWithFilterBar.svelte'; import { RichText } from '@mathesar/components/rich-text'; import { getDataExplorerPageUrl } from '@mathesar/routes/urls'; @@ -12,7 +13,7 @@ import ExplorationsList from './ExplorationsList.svelte'; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let explorationsMap: Map; export let hasTablesToExplore: boolean; export let canEditMetadata: boolean; @@ -47,7 +48,7 @@ on:clear={clearQuery} > - + {$_('open_data_explorer')} diff --git a/mathesar_ui/src/pages/schema/SchemaOverview.svelte b/mathesar_ui/src/pages/schema/SchemaOverview.svelte index 4385fc2de7..ad9d8cd6ca 100644 --- a/mathesar_ui/src/pages/schema/SchemaOverview.svelte +++ b/mathesar_ui/src/pages/schema/SchemaOverview.svelte @@ -4,7 +4,8 @@ import type { QueryInstance } from '@mathesar/api/rest/types/queries'; import type { TableEntry } from '@mathesar/api/rest/types/tables'; import type { RequestStatus } from '@mathesar/api/rest/utils/requestUtils'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import SpinnerButton from '@mathesar/component-library/spinner-button/SpinnerButton.svelte'; import ErrorBox from '@mathesar/components/message-boxes/ErrorBox.svelte'; import { iconRefresh } from '@mathesar/icons'; @@ -31,7 +32,7 @@ export let canEditMetadata: boolean; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; $: hasTables = tablesMap.size > 0; $: hasExplorations = explorationsMap.size > 0; @@ -53,14 +54,14 @@ {#if tablesRequestStatus.state === 'processing'} - + {:else if tablesRequestStatus.state === 'failure'}

{tablesRequestStatus.errors[0]}

{ - await refetchTablesForSchema(schema.id); + await refetchTablesForSchema(schema.oid); }} label={$_('retry')} icon={iconRefresh} @@ -94,7 +95,7 @@
{ - await refetchQueriesForSchema(schema.id); + await refetchQueriesForSchema(schema.oid); }} label={$_('retry')} icon={iconRefresh} @@ -125,7 +126,7 @@ {$_('what_is_an_exploration_mini')}
- + {$_('open_data_explorer')}
diff --git a/mathesar_ui/src/pages/schema/SchemaPage.svelte b/mathesar_ui/src/pages/schema/SchemaPage.svelte index fab522012b..e7bc1d4149 100644 --- a/mathesar_ui/src/pages/schema/SchemaPage.svelte +++ b/mathesar_ui/src/pages/schema/SchemaPage.svelte @@ -1,7 +1,8 @@ diff --git a/mathesar_ui/src/routes/DataExplorerRoute.svelte b/mathesar_ui/src/routes/DataExplorerRoute.svelte index 0bbd7b7f67..6e792991fd 100644 --- a/mathesar_ui/src/routes/DataExplorerRoute.svelte +++ b/mathesar_ui/src/routes/DataExplorerRoute.svelte @@ -4,7 +4,8 @@ import { router } from 'tinro'; import type { QueryInstance } from '@mathesar/api/rest/types/queries'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import type { CancellablePromise } from '@mathesar/component-library'; import AppendBreadcrumb from '@mathesar/components/breadcrumb/AppendBreadcrumb.svelte'; import { iconEdit, iconExploration } from '@mathesar/icons'; @@ -24,7 +25,7 @@ } from '@mathesar/systems/data-explorer'; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let queryId: number | undefined; let is404 = false; @@ -42,7 +43,7 @@ try { const url = getExplorationEditorPageUrl( database.id, - schema.id, + schema.oid, instance.id, ); router.goto(url, true); @@ -134,7 +135,7 @@ import { _ } from 'svelte-i18n'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import AppendBreadcrumb from '@mathesar/components/breadcrumb/AppendBreadcrumb.svelte'; import ErrorPage from '@mathesar/pages/ErrorPage.svelte'; import ExplorationPage from '@mathesar/pages/exploration/ExplorationPage.svelte'; import { queries } from '@mathesar/stores/queries'; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let queryId: number; $: query = $queries.data.get(queryId); diff --git a/mathesar_ui/src/routes/ImportRoute.svelte b/mathesar_ui/src/routes/ImportRoute.svelte index 3690eaa72e..7d92797891 100644 --- a/mathesar_ui/src/routes/ImportRoute.svelte +++ b/mathesar_ui/src/routes/ImportRoute.svelte @@ -2,7 +2,8 @@ import { _ } from 'svelte-i18n'; import { Route } from 'tinro'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import AppendBreadcrumb from '@mathesar/components/breadcrumb/AppendBreadcrumb.svelte'; import { iconImportData } from '@mathesar/icons'; import ImportPreviewPage from '@mathesar/pages/import/preview/ImportPreviewPage.svelte'; @@ -11,13 +12,13 @@ import { getImportPageUrl, getImportPreviewPageQueryParams } from './urls'; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; import type { TableEntry } from '@mathesar/api/rest/types/tables'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import AppendBreadcrumb from '@mathesar/components/breadcrumb/AppendBreadcrumb.svelte'; import RecordPage from '@mathesar/pages/record/RecordPage.svelte'; import RecordStore from '@mathesar/pages/record/RecordStore'; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let table: TableEntry; export let recordPk: string; diff --git a/mathesar_ui/src/routes/TableRoute.svelte b/mathesar_ui/src/routes/TableRoute.svelte index 21359c159b..f75cc14cb3 100644 --- a/mathesar_ui/src/routes/TableRoute.svelte +++ b/mathesar_ui/src/routes/TableRoute.svelte @@ -3,7 +3,8 @@ import { _ } from 'svelte-i18n'; import { Route } from 'tinro'; - import type { Database, SchemaEntry } from '@mathesar/AppTypes'; + import type { Schema } from '@mathesar/api/rpc/schemas'; + import type { Database } from '@mathesar/AppTypes'; import AppendBreadcrumb from '@mathesar/components/breadcrumb/AppendBreadcrumb.svelte'; import ErrorPage from '@mathesar/pages/ErrorPage.svelte'; import TablePage from '@mathesar/pages/table/TablePage.svelte'; @@ -12,7 +13,7 @@ import RecordPageRoute from './RecordPageRoute.svelte'; export let database: Database; - export let schema: SchemaEntry; + export let schema: Schema; export let tableId: number; $: $currentTableId = tableId; diff --git a/mathesar_ui/src/stores/queries.ts b/mathesar_ui/src/stores/queries.ts index 1aa0ed73d5..5dc21f936c 100644 --- a/mathesar_ui/src/stores/queries.ts +++ b/mathesar_ui/src/stores/queries.ts @@ -53,32 +53,32 @@ import { postAPI, putAPI, } from '@mathesar/api/rest/utils/requestUtils'; -import type { SchemaEntry } from '@mathesar/AppTypes'; +import type { Schema } from '@mathesar/api/rpc/schemas'; import CacheManager from '@mathesar/utils/CacheManager'; import { preloadCommonData } from '@mathesar/utils/preloadData'; import { SHARED_LINK_UUID_QUERY_PARAM } from '@mathesar/utils/shares'; import { CancellablePromise } from '@mathesar-component-library'; -import { addCountToSchemaNumExplorations, currentSchemaId } from './schemas'; +import { currentSchemaId } from './schemas'; const commonData = preloadCommonData(); export type UnsavedQueryInstance = Partial; export interface QueriesStoreSubstance { - schemaId: SchemaEntry['id']; + schemaId: Schema['oid']; requestStatus: RequestStatus; data: Map; } // Cache the query list of the last 3 opened schemas const schemasCacheManager = new CacheManager< - SchemaEntry['id'], + Schema['oid'], Writable >(3); const requestMap: Map< - SchemaEntry['id'], + Schema['oid'], CancellablePromise> > = new Map(); @@ -87,7 +87,7 @@ function sortedQueryEntries(queryEntries: QueryInstance[]): QueryInstance[] { } function setSchemaQueriesStore( - schemaId: SchemaEntry['id'], + schemaId: Schema['oid'], queryEntries?: QueryInstance[], ): Writable { const queries: QueriesStoreSubstance['data'] = new Map(); @@ -120,7 +120,7 @@ function findSchemaStoreForQuery(id: QueryInstance['id']) { } export async function refetchQueriesForSchema( - schemaId: SchemaEntry['id'], + schemaId: Schema['oid'], ): Promise { const store = schemasCacheManager.get(schemaId); if (!store) { @@ -164,7 +164,7 @@ export async function refetchQueriesForSchema( let preload = true; export function getQueriesStoreForSchema( - schemaId: SchemaEntry['id'], + schemaId: Schema['oid'], ): Writable { let store = schemasCacheManager.get(schemaId); if (!store) { @@ -212,7 +212,6 @@ export function createQuery( ): CancellablePromise { const promise = postAPI('/api/db/v0/queries/', newQuery); void promise.then((instance) => { - addCountToSchemaNumExplorations(instance.schema, 1); void refetchQueriesForSchema(instance.schema); return instance; }); @@ -306,7 +305,6 @@ export function deleteQuery(queryId: number): CancellablePromise { storeData.data.delete(queryId); return { ...storeData, data: new Map(storeData.data) }; }); - addCountToSchemaNumExplorations(get(store).schemaId, -1); } return undefined; }); diff --git a/mathesar_ui/src/stores/schemas.ts b/mathesar_ui/src/stores/schemas.ts index 774cd840f1..2a5d31374a 100644 --- a/mathesar_ui/src/stores/schemas.ts +++ b/mathesar_ui/src/stores/schemas.ts @@ -2,12 +2,9 @@ import type { Readable, Unsubscriber, Writable } from 'svelte/store'; import { derived, get, writable } from 'svelte/store'; import type { Connection } from '@mathesar/api/rest/connections'; -import schemasApi from '@mathesar/api/rest/schemas'; -import type { - PaginatedResponse, - RequestStatus, -} from '@mathesar/api/rest/utils/requestUtils'; -import type { SchemaEntry, SchemaResponse } from '@mathesar/AppTypes'; +import type { RequestStatus } from '@mathesar/api/rest/utils/requestUtils'; +import { api } from '@mathesar/api/rpc'; +import type { Schema } from '@mathesar/api/rpc/schemas'; import { preloadCommonData } from '@mathesar/utils/preloadData'; import type { CancellablePromise } from '@mathesar-component-library'; @@ -15,12 +12,13 @@ import { connectionsStore } from './databases'; const commonData = preloadCommonData(); -export const currentSchemaId: Writable = - writable(commonData.current_schema ?? undefined); +export const currentSchemaId: Writable = writable( + commonData.current_schema ?? undefined, +); export interface DBSchemaStoreData { requestStatus: RequestStatus; - data: Map; + data: Map; } const dbSchemaStoreMap: Map< @@ -29,22 +27,16 @@ const dbSchemaStoreMap: Map< > = new Map(); const dbSchemasRequestMap: Map< Connection['id'], - CancellablePromise | undefined> + CancellablePromise > = new Map(); -function findStoreBySchemaId(id: SchemaEntry['id']) { - return [...dbSchemaStoreMap.values()].find((entry) => - get(entry).data.has(id), - ); -} - function setDBSchemaStore( connectionId: Connection['id'], - schemas: SchemaResponse[], + schemas: Schema[], ): Writable { const schemaMap: DBSchemaStoreData['data'] = new Map(); schemas.forEach((schema) => { - schemaMap.set(schema.id, schema); + schemaMap.set(schema.oid, schema); }); const storeValue: DBSchemaStoreData = { requestStatus: { state: 'success' }, @@ -63,12 +55,12 @@ function setDBSchemaStore( function updateSchemaInDBSchemaStore( connectionId: Connection['id'], - schema: SchemaResponse, + schema: Schema, ) { const store = dbSchemaStoreMap.get(connectionId); if (store) { store.update((value) => { - value.data?.set(schema.id, schema); + value.data?.set(schema.oid, schema); return { ...value, data: new Map(value.data), @@ -79,7 +71,7 @@ function updateSchemaInDBSchemaStore( function removeSchemaInDBSchemaStore( connectionId: Connection['id'], - schemaId: SchemaEntry['id'], + schemaId: Schema['oid'], ) { const store = dbSchemaStoreMap.get(connectionId); if (store) { @@ -95,36 +87,16 @@ function removeSchemaInDBSchemaStore( export function addCountToSchemaNumTables( connection: Connection, - schema: SchemaEntry, + schema: Schema, count: number, ) { const store = dbSchemaStoreMap.get(connection.id); if (store) { store.update((value) => { - const schemaToModify = value.data.get(schema.id); - if (schemaToModify) { - schemaToModify.num_tables += count; - value.data.set(schema.id, schemaToModify); - } - return { - ...value, - data: new Map(value.data), - }; - }); - } -} - -export function addCountToSchemaNumExplorations( - schemaId: SchemaEntry['id'], - count: number, -) { - const store = findStoreBySchemaId(schemaId); - if (store) { - store.update((value) => { - const schemaToModify = value.data.get(schemaId); + const schemaToModify = value.data.get(schema.oid); if (schemaToModify) { - schemaToModify.num_queries += count; - value.data.set(schemaId, schemaToModify); + schemaToModify.table_count += count; + value.data.set(schema.oid, schemaToModify); } return { ...value, @@ -153,10 +125,9 @@ async function refetchSchemasForDB( dbSchemasRequestMap.get(connectionId)?.cancel(); - const schemaRequest = schemasApi.list(connectionId); + const schemaRequest = api.schemas.list({ database_id: connectionId }).run(); dbSchemasRequestMap.set(connectionId, schemaRequest); - const response = await schemaRequest; - const schemas = response?.results || []; + const schemas = await schemaRequest; const dbSchemasStore = setDBSchemaStore(connectionId, schemas); @@ -201,8 +172,8 @@ function getSchemasStoreForDB( export function getSchemaInfo( connectionId: Connection['id'], - schemaId: SchemaEntry['id'], -): SchemaEntry | undefined { + schemaId: Schema['oid'], +): Schema | undefined { const store = dbSchemaStoreMap.get(connectionId); if (!store) { return undefined; @@ -212,32 +183,51 @@ export function getSchemaInfo( export async function createSchema( connectionId: Connection['id'], - schemaName: SchemaEntry['name'], - description: SchemaEntry['description'], -): Promise { - const response = await schemasApi.add({ - name: schemaName, + name: Schema['name'], + description: Schema['description'], +): Promise { + const schemaOid = await api.schemas + .add({ + database_id: connectionId, + name, + description, + }) + .run(); + updateSchemaInDBSchemaStore(connectionId, { + oid: schemaOid, + name, description, - connectionId, + table_count: 0, }); - updateSchemaInDBSchemaStore(connectionId, response); - return response; } export async function updateSchema( connectionId: Connection['id'], - schema: SchemaEntry, -): Promise { - const response = await schemasApi.update(schema); - updateSchemaInDBSchemaStore(connectionId, response); - return response; + schema: Schema, +): Promise { + await api.schemas + .patch({ + database_id: connectionId, + schema_oid: schema.oid, + patch: { + name: schema.name, + description: schema.description, + }, + }) + .run(); + updateSchemaInDBSchemaStore(connectionId, schema); } export async function deleteSchema( connectionId: Connection['id'], - schemaId: SchemaEntry['id'], + schemaId: Schema['oid'], ): Promise { - await schemasApi.delete(schemaId); + await api.schemas + .delete({ + database_id: connectionId, + schema_oid: schemaId, + }) + .run(); removeSchemaInDBSchemaStore(connectionId, schemaId); } @@ -264,7 +254,7 @@ export const schemas: Readable = derived( }, ); -export const currentSchema: Readable = derived( +export const currentSchema: Readable = derived( [currentSchemaId, schemas], ([$currentSchemaId, $schemas]) => $currentSchemaId ? $schemas.data.get($currentSchemaId) : undefined, diff --git a/mathesar_ui/src/stores/storeBasedUrls.ts b/mathesar_ui/src/stores/storeBasedUrls.ts index 2ac9ba02a5..7c858b6bd2 100644 --- a/mathesar_ui/src/stores/storeBasedUrls.ts +++ b/mathesar_ui/src/stores/storeBasedUrls.ts @@ -31,7 +31,7 @@ export const storeToGetRecordPageUrl = derived( recordId: unknown; }): string | undefined { const d = connectionId ?? connection?.id; - const s = schemaId ?? schema?.id; + const s = schemaId ?? schema?.oid; const t = tableId ?? table?.id; const r = recordId ?? undefined; if ( @@ -61,7 +61,7 @@ export const storeToGetTablePageUrl = derived( tableId?: number; }): string | undefined { const d = connectionId ?? connection?.id; - const s = schemaId ?? schema?.id; + const s = schemaId ?? schema?.oid; const t = tableId ?? table?.id; if (d === undefined || s === undefined || t === undefined) { return undefined; diff --git a/mathesar_ui/src/stores/tables.ts b/mathesar_ui/src/stores/tables.ts index 047d37d183..219cc4407c 100644 --- a/mathesar_ui/src/stores/tables.ts +++ b/mathesar_ui/src/stores/tables.ts @@ -35,7 +35,8 @@ import { patchAPI, postAPI, } from '@mathesar/api/rest/utils/requestUtils'; -import type { DBObjectEntry, Database, SchemaEntry } from '@mathesar/AppTypes'; +import type { Schema } from '@mathesar/api/rpc/schemas'; +import type { DBObjectEntry, Database } from '@mathesar/AppTypes'; import { invalidIf } from '@mathesar/components/form'; import type { AtLeastOne } from '@mathesar/typeUtils'; import { preloadCommonData } from '@mathesar/utils/preloadData'; @@ -55,11 +56,11 @@ export interface DBTablesStoreData { } const schemaTablesStoreMap: Map< - SchemaEntry['id'], + Schema['oid'], Writable > = new Map(); const schemaTablesRequestMap: Map< - SchemaEntry['id'], + Schema['oid'], CancellablePromise> > = new Map(); @@ -68,7 +69,7 @@ function sortedTableEntries(tableEntries: TableEntry[]): TableEntry[] { } function setSchemaTablesStore( - schemaId: SchemaEntry['id'], + schemaId: Schema['oid'], tableEntries?: TableEntry[], ): Writable { const tables: DBTablesStoreData['data'] = new Map(); @@ -93,14 +94,12 @@ function setSchemaTablesStore( return store; } -export function removeTablesInSchemaTablesStore( - schemaId: SchemaEntry['id'], -): void { +export function removeTablesInSchemaTablesStore(schemaId: Schema['oid']): void { schemaTablesStoreMap.delete(schemaId); } export async function refetchTablesForSchema( - schemaId: SchemaEntry['id'], + schemaId: Schema['oid'], ): Promise { const store = schemaTablesStoreMap.get(schemaId); if (!store) { @@ -143,7 +142,7 @@ export async function refetchTablesForSchema( let preload = true; export function getTablesStoreForSchema( - schemaId: SchemaEntry['id'], + schemaId: Schema['oid'], ): Writable { let store = schemaTablesStoreMap.get(schemaId); if (!store) { @@ -195,7 +194,7 @@ function findAndUpdateTableStore(id: TableEntry['id'], tableEntry: TableEntry) { export function deleteTable( database: Database, - schema: SchemaEntry, + schema: Schema, tableId: TableEntry['id'], ): CancellablePromise { const promise = deleteAPI(`/api/db/v0/tables/${tableId}/`); @@ -203,7 +202,7 @@ export function deleteTable( (resolve, reject) => { void promise.then((value) => { addCountToSchemaNumTables(database, schema, -1); - schemaTablesStoreMap.get(schema.id)?.update((tableStoreData) => { + schemaTablesStoreMap.get(schema.oid)?.update((tableStoreData) => { tableStoreData.data.delete(tableId); return { ...tableStoreData, @@ -242,14 +241,14 @@ export function updateTableMetaData( export function createTable( database: Database, - schema: SchemaEntry, + schema: Schema, tableArgs: { name?: string; dataFiles?: [number, ...number[]]; }, ): CancellablePromise { const promise = postAPI('/api/db/v0/tables/', { - schema: schema.id, + schema: schema.oid, name: tableArgs.name, data_files: tableArgs.dataFiles, }); @@ -257,7 +256,7 @@ export function createTable( (resolve, reject) => { void promise.then((value) => { addCountToSchemaNumTables(database, schema, 1); - schemaTablesStoreMap.get(schema.id)?.update((existing) => { + schemaTablesStoreMap.get(schema.oid)?.update((existing) => { const tableEntryMap: DBTablesStoreData['data'] = new Map(); sortedTableEntries([...existing.data.values(), value]).forEach( (entry) => { diff --git a/mathesar_ui/src/stores/users.ts b/mathesar_ui/src/stores/users.ts index 8609761322..90e4c07a82 100644 --- a/mathesar_ui/src/stores/users.ts +++ b/mathesar_ui/src/stores/users.ts @@ -11,7 +11,8 @@ import userApi, { type UserRole, } from '@mathesar/api/rest/users'; import type { RequestStatus } from '@mathesar/api/rest/utils/requestUtils'; -import type { Database, SchemaEntry } from '@mathesar/AppTypes'; +import type { Schema } from '@mathesar/api/rpc/schemas'; +import type { Database } from '@mathesar/AppTypes'; import { getErrorMessage } from '@mathesar/utils/errors'; import { type AccessOperation, @@ -54,7 +55,7 @@ export class UserModel { hasPermission( dbObject: { database?: Pick; - schema?: Pick; + schema?: Pick; }, operation: AccessOperation, ): boolean { @@ -69,7 +70,7 @@ export class UserModel { } const roles: UserRole[] = []; if (schema) { - const userSchemaRole = this.schemaRoles.get(schema.id); + const userSchemaRole = this.schemaRoles.get(schema.oid); if (userSchemaRole) { roles.push(userSchemaRole.role); } @@ -87,8 +88,8 @@ export class UserModel { return this.databaseRoles.get(database.id); } - getRoleForSchema(schema: Pick) { - return this.schemaRoles.get(schema.id); + getRoleForSchema(schema: Pick) { + return this.schemaRoles.get(schema.oid); } hasDirectDbAccess(database: Pick) { @@ -99,14 +100,11 @@ export class UserModel { return this.hasDirectDbAccess(database) || this.isSuperUser; } - hasDirectSchemaAccess(schema: Pick) { - return this.schemaRoles.has(schema.id); + hasDirectSchemaAccess(schema: Pick) { + return this.schemaRoles.has(schema.oid); } - hasSchemaAccess( - database: Pick, - schema: Pick, - ) { + hasSchemaAccess(database: Pick, schema: Pick) { return this.hasDbAccess(database) || this.hasDirectSchemaAccess(schema); } @@ -298,10 +296,10 @@ class WritableUsersStore { async addSchemaRoleForUser( userId: number, - schema: Pick, + schema: Pick, role: UserRole, ) { - const schemaRole = await userApi.addSchemaRole(userId, schema.id, role); + const schemaRole = await userApi.addSchemaRole(userId, schema.oid, role); this.users.update((users) => users.map((user) => { if (user.id === userId) { @@ -313,10 +311,7 @@ class WritableUsersStore { void this.fetchUsersSilently(); } - async removeSchemaAccessForUser( - userId: number, - schema: Pick, - ) { + async removeSchemaAccessForUser(userId: number, schema: Pick) { const user = get(this.users).find((entry) => entry.id === userId); const schemaRole = user?.getRoleForSchema(schema); if (schemaRole) { @@ -345,7 +340,7 @@ class WritableUsersStore { ); } - getNormalUsersWithDirectSchemaRole(schema: Pick) { + getNormalUsersWithDirectSchemaRole(schema: Pick) { return derived(this.users, ($users) => $users.filter( (user) => !user.isSuperUser && user.hasDirectSchemaAccess(schema), @@ -353,7 +348,7 @@ class WritableUsersStore { ); } - getNormalUsersWithoutDirectSchemaRole(schema: Pick) { + getNormalUsersWithoutDirectSchemaRole(schema: Pick) { return derived(this.users, ($users) => $users.filter( (user) => !user.isSuperUser && !user.hasDirectSchemaAccess(schema), @@ -363,7 +358,7 @@ class WritableUsersStore { getUsersWithAccessToSchema( database: Pick, - schema: Pick, + schema: Pick, ) { return derived(this.users, ($users) => $users.filter((user) => user.hasSchemaAccess(database, schema)), diff --git a/mathesar_ui/src/systems/data-explorer/result-pane/QueryRunErrors.svelte b/mathesar_ui/src/systems/data-explorer/result-pane/QueryRunErrors.svelte index cca067558f..6efc59e488 100644 --- a/mathesar_ui/src/systems/data-explorer/result-pane/QueryRunErrors.svelte +++ b/mathesar_ui/src/systems/data-explorer/result-pane/QueryRunErrors.svelte @@ -89,7 +89,7 @@ class="btn btn-secondary" href={getExplorationEditorPageUrl( $currentDatabase.id, - $currentSchema.id, + $currentSchema.oid, $query.id, )} > diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte index f24e68428b..4c36f76d7a 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte @@ -34,7 +34,7 @@ $currentDatabase && $currentSchema ? createDataExplorerUrlToExploreATable( $currentDatabase?.id, - $currentSchema.id, + $currentSchema.oid, { id: $tabularData.id, name: $tables.data.get($tabularData.id)?.name ?? '', @@ -47,7 +47,7 @@ } return constructDataExplorerUrlToSummarizeFromGroup( $currentDatabase.id, - $currentSchema.id, + $currentSchema.oid, { baseTable: { id, name: $currentTable.name }, columns: $columns, @@ -72,7 +72,7 @@ const schema = $currentSchema; if (database && schema) { await deleteTable(database, schema, $tabularData.id); - router.goto(getSchemaPageUrl(database.id, schema.id), true); + router.goto(getSchemaPageUrl(database.id, schema.oid), true); } }, }); diff --git a/mathesar_ui/src/utils/preloadData.ts b/mathesar_ui/src/utils/preloadData.ts index 09a3acc76e..c857d24128 100644 --- a/mathesar_ui/src/utils/preloadData.ts +++ b/mathesar_ui/src/utils/preloadData.ts @@ -2,11 +2,12 @@ import type { Connection } from '@mathesar/api/rest/connections'; import type { QueryInstance } from '@mathesar/api/rest/types/queries'; import type { TableEntry } from '@mathesar/api/rest/types/tables'; import type { User } from '@mathesar/api/rest/users'; -import type { AbstractTypeResponse, SchemaResponse } from '@mathesar/AppTypes'; +import type { Schema } from '@mathesar/api/rpc/schemas'; +import type { AbstractTypeResponse } from '@mathesar/AppTypes'; export interface CommonData { connections: Connection[]; - schemas: SchemaResponse[]; + schemas: Schema[]; tables: TableEntry[]; queries: QueryInstance[]; current_connection: Connection['id'] | null; From d8e44fb0eec6f3e852c839dc9a900e76a74c8373 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 2 Jul 2024 11:10:43 +0530 Subject: [PATCH 0342/1141] allow only alterable fields in patch --- mathesar/utils/tables.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index 8b7875ea20..2b7cd65229 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -92,8 +92,11 @@ def get_tables_meta_data(database_id): def patch_table_meta_data(table_oid, metadata_dict, database_id): metadata_model = TableMetaData.objects.get(database__id=database_id, table_oid=table_oid) + alterable_fields = ( + 'import_verified', 'column_order', 'record_summary_customized', 'record_summary_template' + ) for field, value in metadata_dict.items(): - if hasattr(metadata_model, field): + if hasattr(metadata_model, field) and field in alterable_fields: setattr(metadata_model, field, value) metadata_model.save() return metadata_model From b8d138300f10625e7946fcc090842d9eedac4368 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 2 Jul 2024 11:21:10 +0530 Subject: [PATCH 0343/1141] minor change --- mathesar/utils/tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index 2b7cd65229..87d350a20e 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -96,7 +96,7 @@ def patch_table_meta_data(table_oid, metadata_dict, database_id): 'import_verified', 'column_order', 'record_summary_customized', 'record_summary_template' ) for field, value in metadata_dict.items(): - if hasattr(metadata_model, field) and field in alterable_fields: + if field in alterable_fields: setattr(metadata_model, field, value) metadata_model.save() return metadata_model From b4a759629e8695dd98338ff181b8776e4982b07c Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 2 Jul 2024 12:27:28 +0530 Subject: [PATCH 0344/1141] allow unsetting schema comment using NULL --- db/sql/00_msar.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 08382616fd..8933e44e29 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -865,8 +865,7 @@ CREATE OR REPLACE FUNCTION msar.set_schema_description( ) RETURNS void AS $$/* Set the PostgreSQL description (aka COMMENT) of a schema. -Descriptions are removed by passing an empty string. Passing a NULL description will cause -this function to return NULL without doing anything. +Descriptions are removed by passing an empty string or NULL. Args: sch_id: The OID of the schema. @@ -875,7 +874,7 @@ Args: BEGIN EXECUTE format('COMMENT ON SCHEMA %I IS %L', msar.get_schema_name(sch_id), description); END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +$$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION msar.patch_schema(sch_id oid, patch jsonb) RETURNS void AS $$/* From 9ffc737542bb6a5a441fa6f845d01421f9e9e622 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 2 Jul 2024 12:35:40 +0530 Subject: [PATCH 0345/1141] fix sql test --- db/sql/test_00_msar.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 8636345229..4e053e42d1 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -1325,10 +1325,9 @@ BEGIN PERFORM msar.patch_schema(sch_oid, '{"description": "yay"}'); RETURN NEXT is(obj_description(sch_oid), 'yay'); - -- Edge case: setting the description to null doesn't actually remove it. This behavior is - -- debatable. I did it this way because it was easier to implement. + -- Description is removed when NULL is passed. PERFORM msar.patch_schema(sch_oid, '{"description": null}'); - RETURN NEXT is(obj_description(sch_oid), 'yay'); + RETURN NEXT is(obj_description(sch_oid), NULL); -- Description is removed when an empty string is passed. PERFORM msar.patch_schema(sch_oid, '{"description": ""}'); From f59cb471e33399260b9794708324d59551fb591d Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 2 Jul 2024 17:55:49 +0800 Subject: [PATCH 0346/1141] add metadata patch utility function --- mathesar/utils/columns.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/mathesar/utils/columns.py b/mathesar/utils/columns.py index 7c48cd914e..9b555701f5 100644 --- a/mathesar/utils/columns.py +++ b/mathesar/utils/columns.py @@ -1,5 +1,19 @@ -from mathesar.models.base import ColumnMetaData +from mathesar.models.base import ColumnMetaData, Database def get_columns_meta_data(table_oid, database_id): - return ColumnMetaData.filter(database__id=database_id, table_oid=table_oid) + return ColumnMetaData.objects.filter( + database__id=database_id, table_oid=table_oid + ) + + +def patch_column_meta_data(column_meta_data_list, table_oid, database_id): + db_model = Database.objects.get(id=database_id) + for meta_data_dict in column_meta_data_list: + ColumnMetaData.objects.update_or_create( + database=db_model, + table_oid=table_oid, + attnum=meta_data_dict["attnum"], + defaults=meta_data_dict + ) + return get_columns_meta_data(table_oid, database_id) From d197737e5b8b26fe98071d167ab52de208f13bc5 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 2 Jul 2024 18:14:54 +0800 Subject: [PATCH 0347/1141] wire column metadata patching up to RPC endpoint --- mathesar/rpc/columns/metadata.py | 63 ++++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 5 +++ 2 files changed, 68 insertions(+) diff --git a/mathesar/rpc/columns/metadata.py b/mathesar/rpc/columns/metadata.py index f99118b6b5..d97f81778d 100644 --- a/mathesar/rpc/columns/metadata.py +++ b/mathesar/rpc/columns/metadata.py @@ -73,6 +73,42 @@ def from_model(cls, model): ) +class SettableColumnMetaData(TypedDict): + """ + Settable metadata fields for a column in a table. + + Attributes: + attnum: The attnum of the column in the table. + bool_input: How the input for a boolean column should be shown. + bool_true: A string to display for `true` values. + bool_false: A string to display for `false` values. + num_min_frac_digits: Minimum digits shown after the decimal point. + num_max_frac_digits: Maximum digits shown after the decimal point. + num_show_as_perc: Whether to show a numeric value as a percentage. + mon_currency_symbol: The currency symbol shown for money value. + mon_currency_location: Where the currency symbol should be shown. + time_format: A string representing the format of time values. + date_format: A string representing the format of date values. + duration_min: The smallest unit for displaying durations. + duration_max: The largest unit for displaying durations. + duration_show_units: Whether to show the units for durations. + """ + attnum: int + bool_input: Optional[Literal["dropdown", "checkbox"]] + bool_true: Optional[str] + bool_false: Optional[str] + num_min_frac_digits: Optional[int] + num_max_frac_digits: Optional[int] + num_show_as_perc: Optional[bool] + mon_currency_symbol: Optional[str] + mon_currency_location: Optional[Literal["after-minus", "end-with-space"]] + time_format: Optional[str] + date_format: Optional[str] + duration_min: Optional[str] + duration_max: Optional[str] + duration_show_units: Optional[bool] + + @rpc_method(name="columns.metadata.list") @http_basic_auth_login_required @handle_rpc_exceptions @@ -91,3 +127,30 @@ def list_(*, table_oid: int, database_id: int, **kwargs) -> list[ColumnMetaData] return [ ColumnMetaData.from_model(model) for model in columns_meta_data ] + + +@rpc_method(name="columns.metadata.patch") +@http_basic_auth_login_required +@handle_rpc_exceptions +def patch( + *, + column_meta_data_list: list[SettableColumnMetaData], + table_oid: int, + database_id: int, + **kwargs +) -> ColumnMetaData: + """ + Alter metadata settings associated with columns of a table for a database. + + Args: + column_meta_data_list: A list describing desired metadata alterations. + table_oid: Identity of the table whose metadata we'll modify. + database_id: The Django id of the database containing the table. + + Returns: + List of altered metadata objects. + """ + column_meta_data = patch_column_meta_data( + column_meta_data_list, table_oid, database_id + ) + return ColumnMetaData.from_model(table_meta_data) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index b786e99027..0d6c28a4e8 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -39,6 +39,11 @@ "columns.metadata.list", [user_is_authenticated] ), + ( + columns.metadata.patch, + "columns.metadata.patch", + [user_is_authenticated] + ), ( connections.add_from_known_connection, "connections.add_from_known_connection", From d2b0b827e685a27d65f05d819f343a36026d65f1 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 2 Jul 2024 11:37:18 -0400 Subject: [PATCH 0348/1141] Drop old function signature --- db/sql/00_msar.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 08382616fd..92abbc41a8 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2302,6 +2302,8 @@ SELECT __msar.exec_ddl( FROM col_cte, con_cte; $$ LANGUAGE SQL; +-- Drop function defined in Mathesar 0.1.7 with different argument names +DROP FUNCTION IF EXISTS msar.add_mathesar_table(oid, text, jsonb, jsonb, text); CREATE OR REPLACE FUNCTION msar.add_mathesar_table(sch_id oid, tab_name text, col_defs jsonb, con_defs jsonb, comment_ text) From a2529ce809ecadc6eea3403bfe4f74ce72e74a40 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 2 Jul 2024 23:43:46 +0530 Subject: [PATCH 0349/1141] alter description for schemas and tables only when description key exists --- db/sql/00_msar.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 8933e44e29..599cdeaf2f 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -885,11 +885,12 @@ Args: patch: A JSONB object with the following keys: - name: (optional) The new name of the schema - description: (optional) The new description of the schema. To remove a description, pass an - empty string. Passing a NULL description will have no effect on the description. + empty string or NULL. */ BEGIN PERFORM msar.rename_schema(sch_id, patch->>'name'); - PERFORM msar.set_schema_description(sch_id, patch->>'description'); + PERFORM CASE WHEN patch ? 'description' + THEN msar.set_schema_description(sch_id, patch->>'description') END; END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -1136,14 +1137,13 @@ Args: */ DECLARE new_tab_name text; - comment text; col_alters jsonb; BEGIN new_tab_name := tab_alters->>'name'; - comment := tab_alters->>'description'; col_alters := tab_alters->'columns'; PERFORM msar.rename_table(tab_id, new_tab_name); - PERFORM msar.comment_on_table(tab_id, comment); + PERFORM CASE WHEN tab_alters ? 'description' + THEN msar.comment_on_table(tab_id, tab_alters->>'description') END; PERFORM msar.alter_columns(tab_id, col_alters); RETURN __msar.get_qualified_relation_name_or_null(tab_id); END; From 66d963ad15e95c97f13c994b1070feb2b4e5d9db Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 3 Jul 2024 01:34:26 +0530 Subject: [PATCH 0350/1141] fix common_data --- mathesar/urls.py | 2 +- mathesar/views.py | 37 ++++++------------------------------- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/mathesar/urls.py b/mathesar/urls.py index a0b438376d..8f80565ad0 100644 --- a/mathesar/urls.py +++ b/mathesar/urls.py @@ -61,7 +61,7 @@ path('i18n/', include('django.conf.urls.i18n')), re_path( r'^db/(?P\w+)/(?P\w+)/', - views.schema_home, + views.schemas, name='schema_home' ), ] diff --git a/mathesar/views.py b/mathesar/views.py index e88b596881..4199ee79ee 100644 --- a/mathesar/views.py +++ b/mathesar/views.py @@ -7,6 +7,7 @@ from rest_framework.decorators import api_view from rest_framework.response import Response +from mathesar.rpc.schemas import list_ as schemas_list from mathesar.api.db.permissions.database import DatabaseAccessPolicy from mathesar.api.db.permissions.query import QueryAccessPolicy from mathesar.api.db.permissions.schema import SchemaAccessPolicy @@ -26,14 +27,10 @@ def get_schema_list(request, database): - qs = Schema.objects.filter(database=database) - permission_restricted_qs = SchemaAccessPolicy.scope_queryset(request, qs) - schema_serializer = SchemaSerializer( - permission_restricted_qs, - many=True, - context={'request': request} - ) - return schema_serializer.data + if database is not None: + return schemas_list(request=request, database_id=database.id) + else: + return [] def _get_permissible_db_queryset(request): @@ -180,19 +177,6 @@ def get_current_database(request, connection_id): return current_database -def get_current_schema(request, schema_id, database): - # if there's a schema ID passed in, try to retrieve the schema, or return a 404 error. - if schema_id is not None: - permitted_schemas = SchemaAccessPolicy.scope_queryset(request, Schema.objects.all()) - return get_object_or_404(permitted_schemas, id=schema_id) - else: - try: - # Try to get the first schema in the DB - return Schema.objects.filter(database=database).order_by('id').first() - except Schema.DoesNotExist: - return None - - def render_schema(request, database, schema): # if there's no schema available, redirect to the schemas page. if not schema: @@ -300,16 +284,7 @@ def admin_home(request, **kwargs): @login_required -def schema_home(request, connection_id, schema_id, **kwargs): - database = get_current_database(request, connection_id) - schema = get_current_schema(request, schema_id, database) - return render(request, 'mathesar/index.html', { - 'common_data': get_common_data(request, database, schema) - }) - - -@login_required -def schemas(request, connection_id): +def schemas(request, connection_id, **kwargs): database = get_current_database(request, connection_id) return render(request, 'mathesar/index.html', { 'common_data': get_common_data(request, database, None) From ce43b90c22bc9b35f02cb8d44f157b25fd983604 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 2 Jul 2024 21:49:08 -0400 Subject: [PATCH 0351/1141] Begin creating constraints RPC methods --- config/settings/common_settings.py | 1 + db/connection.py | 17 +++ db/constraints/operations/create.py | 21 ++- db/constraints/operations/select.py | 7 +- db/sql/00_msar.sql | 135 ++++++++++++++---- .../constraints/operations/test_create.py | 2 +- db/tests/dependents/test_dependents.py | 6 +- docs/docs/api/rpc.md | 10 ++ mathesar/models/deprecated.py | 4 +- mathesar/rpc/constraints.py | 45 +++++- mathesar/tests/rpc/test_endpoints.py | 6 + 11 files changed, 213 insertions(+), 41 deletions(-) diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index c574b418d5..7ff925588f 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -66,6 +66,7 @@ def pipe_delim(pipe_string): MODERNRPC_METHODS_MODULES = [ 'mathesar.rpc.connections', + 'mathesar.rpc.constraints', 'mathesar.rpc.columns', 'mathesar.rpc.columns.metadata', 'mathesar.rpc.schemas', diff --git a/db/connection.py b/db/connection.py index 5af890107f..2f6f64a56b 100644 --- a/db/connection.py +++ b/db/connection.py @@ -1,5 +1,6 @@ from sqlalchemy import text import psycopg +from psycopg.rows import dict_row def execute_msar_func_with_engine(engine, func_name, *args): @@ -55,6 +56,22 @@ def exec_msar_func(conn, func_name, *args): ) +def select_from_msar_func(conn, func_name, *args): + """ + Select all records from an msar function using a psycopg (3) connection. + + Args: + conn: a psycopg connection + func_name: The unqualified msar_function name (danger; not sanitized) + *args: The list of parameters to pass + """ + cursor = conn.execute( + f"SELECT * FROM msar.{func_name}({','.join(['%s']*len(args))})", args + ) + cursor.row_factory = dict_row + return cursor.fetchall() + + def load_file_with_engine(engine, file_handle): """Run an SQL script from a file, using psycopg.""" conn_str = str(engine.url) diff --git a/db/constraints/operations/create.py b/db/constraints/operations/create.py index d12d8591a5..738b65db55 100644 --- a/db/constraints/operations/create.py +++ b/db/constraints/operations/create.py @@ -1,13 +1,12 @@ -from db.connection import execute_msar_func_with_engine +from db.connection import execute_msar_func_with_engine, exec_msar_func -def add_constraint(constraint_obj, engine): +def add_constraint_via_sql_alchemy(constraint_obj, engine): """ Add a constraint. Args: - constraint_obj: A constraint object instantiatated with appropriate - params. + constraint_obj: (See __msar.process_con_def_jsonb for details) engine: SQLAlchemy engine object for connecting. Returns: @@ -19,3 +18,17 @@ def add_constraint(constraint_obj, engine): constraint_obj.table_oid, constraint_obj.get_constraint_def_json() ).fetchone()[0] + + +def create_constraint(constraint_obj, conn): + """ + Create a constraint using a psycopg connection. + + Args: + constraint_obj: (See __msar.process_con_def_jsonb for details) + conn: a psycopg connection + + Returns: + Returns a list of oid(s) of constraints for a given table. + """ + return exec_msar_func(conn, 'add_constraints', constraint_obj).fetchone()[0] diff --git a/db/constraints/operations/select.py b/db/constraints/operations/select.py index df0e73fc9d..de47bacb1d 100644 --- a/db/constraints/operations/select.py +++ b/db/constraints/operations/select.py @@ -1,7 +1,12 @@ +from sqlalchemy import select, and_ + +from db.connection import select_from_msar_func from db.utils import get_pg_catalog_table from db.metadata import get_empty_metadata -from sqlalchemy import select, and_ + +def get_constraints_for_table(table_oid, conn): + return select_from_msar_func(conn, 'get_constraints_for_table', table_oid) def get_constraints_with_oids(engine, table_oid=None): diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 08382616fd..a9ee34a207 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -246,6 +246,52 @@ END $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION msar.get_relation_name(rel_oid oid) RETURNS TEXT AS $$/* +Return the UNQUOTED name of a given relation (e.g., table). + +If the relation does not exist, an exception will be raised. + +Args: + rel_oid: The OID of the relation. +*/ +DECLARE rel_name text; +BEGIN + SELECT relname INTO rel_name FROM pg_class WHERE oid=rel_oid; + + IF rel_name IS NULL THEN + RAISE EXCEPTION 'Relation with OID % does not exist', rel_oid + USING ERRCODE = '42P01'; -- undefined_table + END IF; + + RETURN rel_name; +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION msar.get_relation_schema_name(rel_oid oid) RETURNS TEXT AS $$/* +Return the UNQUOTED name of the schema which contains a given relation (e.g., table). + +If the relation does not exist, an exception will be raised. + +Args: + rel_oid: The OID of the relation. +*/ +DECLARE sch_name text; +BEGIN + SELECT n.nspname INTO sch_name + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = rel_oid; + + IF sch_name IS NULL THEN + RAISE EXCEPTION 'Relation with OID % does not exist', rel_oid + USING ERRCODE = '42P01'; -- undefined_table + END IF; + + RETURN sch_name; +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + DROP FUNCTION IF EXISTS msar.get_relation_oid(text, text) CASCADE; CREATE OR REPLACE FUNCTION @@ -451,6 +497,59 @@ END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION msar.get_constraint_type_api_code(contype "char") RETURNS TEXT AS $$/* +This function returns a string that represents the constraint type code used to describe +constraints when listing them within the Mathesar API. + +PostgreSQL constraint types are documented by the `contype` field here: +https://www.postgresql.org/docs/current/catalog-pg-constraint.html + +Notably, we don't include 't' (trigger) because triggers a bit different structurally and we don't +support working with them (yet?) in Mathesar. +*/ +SELECT CASE contype + WHEN 'c' THEN 'check' + WHEN 'f' THEN 'foreignkey' + WHEN 'p' THEN 'primary' + WHEN 'u' THEN 'unique' + WHEN 'x' THEN 'exclude' +END; +$$ LANGUAGE sql; + + +CREATE OR REPLACE FUNCTION msar.get_constraints_for_table(tab_id oid) RETURNS TABLE +( + oid oid, + name text, + columns smallint[], + type text, + referent_table_oid oid, + referent_columns smallint[] +) +AS $$/* +Return data describing the constraints set on a given table. + +Args: + tab_id: The OID of the table. +*/ +WITH constraints AS ( + SELECT + oid, + conname AS name, + conkey AS columns, + msar.get_constraint_type_api_code(contype) AS type, + confrelid AS referent_table_oid, + confkey AS referent_columns + FROM pg_catalog.pg_constraint + WHERE conrelid = tab_id +) +SELECT * +FROM constraints +-- Only return constraints with types that we're able to classify +WHERE type IS NOT NULL +$$ LANGUAGE sql; + + CREATE OR REPLACE FUNCTION msar.get_constraint_name(con_id oid) RETURNS text AS $$/* Return the UNQUOTED constraint name of the corresponding constraint oid. @@ -2220,28 +2319,9 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- --- Drop constraint --------------------------------------------------------------------------------- - - -CREATE OR REPLACE FUNCTION -__msar.drop_constraint(tab_name text, con_name text) RETURNS text AS $$/* -Drop a constraint, returning the command executed. - -Args: - tab_name: A qualified & quoted name of the table that has the constraint to be dropped. - con_name: Name of the constraint to drop, properly quoted. -*/ -BEGIN - RETURN __msar.exec_ddl( - 'ALTER TABLE %s DROP CONSTRAINT %s', tab_name, con_name - ); -END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; - - CREATE OR REPLACE FUNCTION -msar.drop_constraint(sch_name text, tab_name text, con_name text) RETURNS text AS $$/* -Drop a constraint, returning the command executed. +msar.drop_constraint(sch_name text, tab_name text, con_name text) RETURNS void AS $$/* +Drop a constraint Args: sch_name: The name of the schema where the table with constraint to be dropped resides, unquoted. @@ -2249,25 +2329,24 @@ Args: con_name: Name of the constraint to drop, unquoted. */ BEGIN - RETURN __msar.drop_constraint( - __msar.build_qualified_name_sql(sch_name, tab_name), quote_ident(con_name) - ); + EXECUTE format('ALTER TABLE %I.%I DROP CONSTRAINT %I', sch_name, tab_name, con_name); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION msar.drop_constraint(tab_id oid, con_id oid) RETURNS TEXT AS $$/* -Drop a constraint, returning the command executed. +Drop a constraint Args: tab_id: OID of the table that has the constraint to be dropped. con_id: OID of the constraint to be dropped. */ BEGIN - RETURN __msar.drop_constraint( - __msar.get_qualified_relation_name(tab_id), - quote_ident(msar.get_constraint_name(con_id)) + PERFORM msar.drop_constraint( + msar.get_relation_schema_name(tab_id), + msar.get_relation_name(tab_id), + msar.get_constraint_name(con_id) ); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; diff --git a/db/tests/constraints/operations/test_create.py b/db/tests/constraints/operations/test_create.py index da59b1afe6..476ef1c74b 100644 --- a/db/tests/constraints/operations/test_create.py +++ b/db/tests/constraints/operations/test_create.py @@ -23,7 +23,7 @@ def test_add_constraint_db(engine_with_schema, constraint_obj): engine = engine_with_schema with patch.object(con_create, 'execute_msar_func_with_engine') as mock_exec: - con_create.add_constraint( + con_create.add_constraint_via_sql_alchemy( engine=engine, constraint_obj=constraint_obj ) diff --git a/db/tests/dependents/test_dependents.py b/db/tests/dependents/test_dependents.py index 1b8e0f5afa..fc838bec23 100644 --- a/db/tests/dependents/test_dependents.py +++ b/db/tests/dependents/test_dependents.py @@ -1,7 +1,7 @@ import pytest from sqlalchemy import MetaData, select, Index from sqlalchemy_utils import create_view -from db.constraints.operations.create import add_constraint +from db.constraints.operations.create import add_constraint_via_sql_alchemy from db.constraints.base import ForeignKeyConstraint from db.dependents.dependents_utils import get_dependents_graph from db.constraints.operations.select import get_constraint_oid_by_name_and_table_oid @@ -93,7 +93,7 @@ def test_self_reference(engine_with_schema, library_tables_oids): fk_column_attnum = create_column(engine, publishers_oid, {'name': 'Parent Publisher', 'type': PostgresType.INTEGER.id})[0] pk_column_attnum = get_column_attnum_from_name(publishers_oid, 'id', engine, metadata=get_empty_metadata()) fk_constraint = ForeignKeyConstraint('Publishers_Publisher_fkey', publishers_oid, [fk_column_attnum], publishers_oid, [pk_column_attnum], {}) - add_constraint(fk_constraint, engine) + add_constraint_via_sql_alchemy(fk_constraint, engine) publishers_oid = library_tables_oids['Publishers'] publishers_dependents_graph = get_dependents_graph(publishers_oid, engine, []) @@ -114,7 +114,7 @@ def test_circular_reference(engine_with_schema, library_tables_oids): fk_column_attnum = create_column(engine, publishers_oid, {'name': 'Top Publication', 'type': PostgresType.INTEGER.id})[0] publications_pk_column_attnum = get_column_attnum_from_name(publications_oid, 'id', engine, metadata=get_empty_metadata()) fk_constraint = ForeignKeyConstraint('Publishers_Publications_fkey', publishers_oid, [fk_column_attnum], publications_oid, [publications_pk_column_attnum], {}) - add_constraint(fk_constraint, engine) + add_constraint_via_sql_alchemy(fk_constraint, engine) publishers_dependents_graph = get_dependents_graph(publishers_oid, engine, []) publications_dependents_oids = _get_object_dependents_oids(publishers_dependents_graph, publications_oid) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index f3fd42eb25..90753fb728 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -47,6 +47,16 @@ To use an RPC function: - grant_access_to_user - DBModelReturn +## Constraints + +::: mathesar.rpc.constraints + options: + members: + - add_from_known_connection + - add_from_scratch + - grant_access_to_user + - DBModelReturn + ## Schemas ::: schemas diff --git a/mathesar/models/deprecated.py b/mathesar/models/deprecated.py index 510bc92ca4..68087f3fae 100644 --- a/mathesar/models/deprecated.py +++ b/mathesar/models/deprecated.py @@ -17,7 +17,7 @@ get_column_attnum_from_names_as_map, get_column_name_from_attnum, get_map_of_attnum_to_column_name, get_map_of_attnum_and_table_oid_to_column_name, ) -from db.constraints.operations.create import add_constraint +from db.constraints.operations.create import add_constraint_via_sql_alchemy from db.constraints.operations.drop import drop_constraint from db.constraints.operations.select import get_constraint_record_from_oid from db.constraints import utils as constraint_utils @@ -558,7 +558,7 @@ def add_constraint(self, constraint_obj): # the most newly-created constraint. Other methods (e.g., trying to get # a constraint by name when it wasn't set here) are even less robust. constraint_oid = max( - add_constraint(constraint_obj, engine=self._sa_engine) + add_constraint_via_sql_alchemy(constraint_obj, engine=self._sa_engine) ) result = Constraint.current_objects.create(oid=constraint_oid, table=self) reset_reflection(db_name=self.schema.database.name) diff --git a/mathesar/rpc/constraints.py b/mathesar/rpc/constraints.py index fe726464d0..0a04e696bf 100644 --- a/mathesar/rpc/constraints.py +++ b/mathesar/rpc/constraints.py @@ -1,8 +1,49 @@ """ Classes and functions exposed to the RPC endpoint for managing table constraints. """ -from typing import TypedDict +from typing import * +from modernrpc.core import rpc_method, REQUEST_KEY +from modernrpc.auth.basic import http_basic_auth_login_required + +from db.constraints.operations.select import get_constraints_for_table +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions +from mathesar.rpc.utils import connect class CreatableConstraintInfo(TypedDict): - pass + name: Optional[str] + # TODO + + +class Constraint(TypedDict): + """ + Information about a constraint + + Attributes: + oid: The OID of the schema + name: The name of the schema + description: A description of the schema + table_count: The number of tables in the schema + """ + oid: int + table_oid: int + # TODO + + +@rpc_method(name="constraints.list") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_(*, table_oid: int, database_id: int, **kwargs) -> list[Constraint]: + """ + List information about constraints in a table. Exposed as `list`. + + Args: + database_id: The Django id of the database containing the table. + table_oid: The oid of the table to list constraints for. + + Returns: + A list of Constraint objects + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + return get_constraints_for_table(table_oid, conn) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index b786e99027..cd8efc3dbd 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -10,6 +10,7 @@ from mathesar.rpc import columns from mathesar.rpc import connections +from mathesar.rpc import constraints from mathesar.rpc import schemas from mathesar.rpc import tables @@ -54,6 +55,11 @@ "connections.grant_access_to_user", [user_is_superuser] ), + ( + constraints.list_, + "constraints.list", + [user_is_authenticated] + ), ( schemas.add, "schemas.add", From 4004345cecbe493ceeb50e55a82fd93807a1d4bd Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 3 Jul 2024 12:13:14 +0800 Subject: [PATCH 0352/1141] fix bug and signature for RPC function --- mathesar/rpc/columns/metadata.py | 8 +++++--- mathesar/utils/columns.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mathesar/rpc/columns/metadata.py b/mathesar/rpc/columns/metadata.py index d97f81778d..e9fbe86f61 100644 --- a/mathesar/rpc/columns/metadata.py +++ b/mathesar/rpc/columns/metadata.py @@ -138,7 +138,7 @@ def patch( table_oid: int, database_id: int, **kwargs -) -> ColumnMetaData: +) -> list[ColumnMetaData]: """ Alter metadata settings associated with columns of a table for a database. @@ -150,7 +150,9 @@ def patch( Returns: List of altered metadata objects. """ - column_meta_data = patch_column_meta_data( + columns_meta_data = patch_column_meta_data( column_meta_data_list, table_oid, database_id ) - return ColumnMetaData.from_model(table_meta_data) + return [ + ColumnMetaData.from_model(model) for model in columns_meta_data + ] diff --git a/mathesar/utils/columns.py b/mathesar/utils/columns.py index 9b555701f5..371901199e 100644 --- a/mathesar/utils/columns.py +++ b/mathesar/utils/columns.py @@ -10,6 +10,7 @@ def get_columns_meta_data(table_oid, database_id): def patch_column_meta_data(column_meta_data_list, table_oid, database_id): db_model = Database.objects.get(id=database_id) for meta_data_dict in column_meta_data_list: + # TODO decide if this is worth the trouble of doing in bulk. ColumnMetaData.objects.update_or_create( database=db_model, table_oid=table_oid, From 31c72e6f366bcf96399d2a5d88fa212e41a145af Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 3 Jul 2024 12:36:39 +0800 Subject: [PATCH 0353/1141] fix import issue, change name to match getter --- mathesar/rpc/columns/metadata.py | 4 ++-- mathesar/utils/columns.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mathesar/rpc/columns/metadata.py b/mathesar/rpc/columns/metadata.py index e9fbe86f61..3dd8df5931 100644 --- a/mathesar/rpc/columns/metadata.py +++ b/mathesar/rpc/columns/metadata.py @@ -7,7 +7,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -from mathesar.utils.columns import get_columns_meta_data +from mathesar.utils.columns import get_columns_meta_data, patch_columns_meta_data class ColumnMetaData(TypedDict): @@ -150,7 +150,7 @@ def patch( Returns: List of altered metadata objects. """ - columns_meta_data = patch_column_meta_data( + columns_meta_data = patch_columns_meta_data( column_meta_data_list, table_oid, database_id ) return [ diff --git a/mathesar/utils/columns.py b/mathesar/utils/columns.py index 371901199e..67b3deec75 100644 --- a/mathesar/utils/columns.py +++ b/mathesar/utils/columns.py @@ -7,7 +7,7 @@ def get_columns_meta_data(table_oid, database_id): ) -def patch_column_meta_data(column_meta_data_list, table_oid, database_id): +def patch_columns_meta_data(column_meta_data_list, table_oid, database_id): db_model = Database.objects.get(id=database_id) for meta_data_dict in column_meta_data_list: # TODO decide if this is worth the trouble of doing in bulk. From 2a658a272a31e8e92e0fff122e66ef2904982e19 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 3 Jul 2024 12:37:29 +0800 Subject: [PATCH 0354/1141] add test for metadata.patch RPC function --- mathesar/tests/rpc/columns/test_c_metadata.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/mathesar/tests/rpc/columns/test_c_metadata.py b/mathesar/tests/rpc/columns/test_c_metadata.py index f9b6e087c0..6e3ea21244 100644 --- a/mathesar/tests/rpc/columns/test_c_metadata.py +++ b/mathesar/tests/rpc/columns/test_c_metadata.py @@ -57,3 +57,57 @@ def mock_get_columns_meta_data(_table_oid, _database_id): ] actual_metadata_list = metadata.list_(table_oid=table_oid, database_id=database_id) assert actual_metadata_list == expect_metadata_list + + +# TODO consider mocking out ColumnMetaData queryset for this test +def test_columns_meta_data_patch(monkeypatch): + database_id = 2 + table_oid = 123456 + expect_metadata_list = [ + metadata.ColumnMetaData( + database_id=database_id, table_oid=table_oid, attnum=2, + bool_input="dropdown", bool_true="TRUE", bool_false="FALSE", + num_min_frac_digits=5, num_max_frac_digits=10, num_show_as_perc=False, + mon_currency_symbol="EUR", mon_currency_location="end-with-space", + time_format=None, date_format=None, + duration_min=None, duration_max=None, duration_show_units=True, + ), + metadata.ColumnMetaData( + database_id=database_id, table_oid=table_oid, attnum=8, + bool_input="checkbox", bool_true="true", bool_false="false", + num_min_frac_digits=2, num_max_frac_digits=8, num_show_as_perc=True, + mon_currency_symbol="$", mon_currency_location="after-minus", + time_format=None, date_format=None, + duration_min=None, duration_max=None, duration_show_units=True, + ), + ] + + def mock_patch_columns_meta_data(column_meta_data_list, _table_oid, _database_id): + server_model = Server(id=2, host='example.com', port=5432) + db_model = Database(id=_database_id, name='mymathesardb', server=server_model) + return [ + ColumnMetaData( + database=db_model, table_oid=_table_oid, attnum=2, + bool_input="dropdown", bool_true="TRUE", bool_false="FALSE", + num_min_frac_digits=5, num_max_frac_digits=10, num_show_as_perc=False, + mon_currency_symbol="EUR", mon_currency_location="end-with-space", + time_format=None, date_format=None, + duration_min=None, duration_max=None, duration_show_units=True, + ), + ColumnMetaData( + database=db_model, table_oid=_table_oid, attnum=8, + bool_input="checkbox", bool_true="true", bool_false="false", + num_min_frac_digits=2, num_max_frac_digits=8, num_show_as_perc=True, + mon_currency_symbol="$", mon_currency_location="after-minus", + time_format=None, date_format=None, + duration_min=None, duration_max=None, duration_show_units=True, + ) + ] + + monkeypatch.setattr(metadata, "patch_columns_meta_data", mock_patch_columns_meta_data) + actual_metadata_list = metadata.patch( + column_meta_data_list=expect_metadata_list, + table_oid=table_oid, + database_id=database_id + ) + assert actual_metadata_list == expect_metadata_list From 51fa9dc89f6cac59c2c8094c9606dcf5b8f86002 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 3 Jul 2024 14:04:31 +0800 Subject: [PATCH 0355/1141] update docs with metadata patch function --- docs/docs/api/rpc.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index f3fd42eb25..27dcf9d911 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -107,7 +107,9 @@ To use an RPC function: options: members: - list_ + - patch - ColumnMetaData + - SettableColumnMetaData ## Responses From 346702d4ec08c078498c6c04b361daaee81e5841 Mon Sep 17 00:00:00 2001 From: pavish Date: Wed, 3 Jul 2024 16:44:44 +0530 Subject: [PATCH 0356/1141] Implement endpoint for retrieving roles --- config/settings/common_settings.py | 1 + db/roles/__init__.py | 0 db/roles/operations/__init__.py | 0 db/roles/operations/select.py | 5 +++ db/sql/00_msar.sql | 51 ++++++++++++++++++++++++++++++ mathesar/rpc/roles.py | 47 +++++++++++++++++++++++++++ 6 files changed, 104 insertions(+) create mode 100644 db/roles/__init__.py create mode 100644 db/roles/operations/__init__.py create mode 100644 db/roles/operations/select.py create mode 100644 mathesar/rpc/roles.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index c574b418d5..a28c482e5c 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -68,6 +68,7 @@ def pipe_delim(pipe_string): 'mathesar.rpc.connections', 'mathesar.rpc.columns', 'mathesar.rpc.columns.metadata', + 'mathesar.rpc.roles', 'mathesar.rpc.schemas', 'mathesar.rpc.tables', 'mathesar.rpc.tables.metadata' diff --git a/db/roles/__init__.py b/db/roles/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/db/roles/operations/__init__.py b/db/roles/operations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py new file mode 100644 index 0000000000..46ea86526c --- /dev/null +++ b/db/roles/operations/select.py @@ -0,0 +1,5 @@ +from db.connection import exec_msar_func + + +def get_roles(conn): + return exec_msar_func(conn, 'get_roles').fetchone()[0] diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 08382616fd..1419dd16f1 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -792,6 +792,57 @@ FROM ( $$ LANGUAGE sql; +CREATE OR REPLACE FUNCTION msar.get_roles() RETURNS jsonb AS $$/* +Return a json array of objects with the list of roles in a database server, +excluding pg system roles. + +Each returned JSON object in the array has the form: + { + "oid": + "name": + "super": + "inherits": + "create_role": + "create_db": + "login": + "description": + "members": <[ + { "oid": , "admin": } + ]|null> + } +*/ +WITH rolemembers as ( + SELECT + pgr.oid AS oid, + jsonb_agg( + jsonb_build_object( + 'oid', pgm.member, + 'admin', pgm.admin_option + ) + ) AS members + FROM pg_catalog.pg_roles pgr + INNER JOIN pg_catalog.pg_auth_members pgm ON pgr.oid=pgm.roleid + GROUP BY pgr.oid +) +SELECT jsonb_agg(role_data) +FROM ( + SELECT + r.oid AS oid, + r.rolname AS name, + r.rolsuper AS super, + r.rolinherit AS inherits, + r.rolcreaterole AS create_role, + r.rolcreatedb AS create_db, + r.rolcanlogin AS login, + pg_catalog.obj_description(r.oid) AS description, + rolemembers.members AS members + FROM pg_catalog.pg_roles r + LEFT OUTER JOIN rolemembers ON r.oid = rolemembers.oid + WHERE r.rolname NOT LIKE 'pg_%' +) AS role_data; +$$ LANGUAGE sql; + + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -- ROLE MANIPULATION FUNCTIONS diff --git a/mathesar/rpc/roles.py b/mathesar/rpc/roles.py new file mode 100644 index 0000000000..16a02d4ea1 --- /dev/null +++ b/mathesar/rpc/roles.py @@ -0,0 +1,47 @@ +""" +Classes and functions exposed to the RPC endpoint for managing table columns. +""" +from typing import Optional, TypedDict + +from modernrpc.core import rpc_method, REQUEST_KEY +from modernrpc.auth.basic import http_basic_auth_login_required + +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions +from mathesar.rpc.utils import connect +from db.roles.operations.select import get_roles + + +class RoleMembers(TypedDict): + oid: int + admin: bool + + +class RoleInfo(TypedDict): + oid: int + name: str + super: bool + inherits: bool + create_role: bool + create_db: bool + login: bool + description: Optional[str] + members: Optional[list[RoleMembers]] + + +@rpc_method(name="roles.list") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_(*, database_id: int, **kwargs) -> list[RoleInfo]: + """ + List information about roles for a database server. Exposed as `list`. + Requires a database id inorder to connect to the server. + Args: + database_id: The Django id of the database containing the table. + Returns: + A list of roles present on the database server. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + roles = get_roles(conn) + + return roles From 58e550e92a0ac9b099f4d5e2d9160c5241e0dca4 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 4 Jul 2024 00:39:01 +0530 Subject: [PATCH 0357/1141] some cleanups and update docstrings --- db/sql/00_msar.sql | 10 +++++----- mathesar/rpc/constraints.py | 20 +++++++++++++------- mathesar/rpc/tables/base.py | 3 +++ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index a9ee34a207..2406607bc0 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -280,7 +280,7 @@ DECLARE sch_name text; BEGIN SELECT n.nspname INTO sch_name FROM pg_catalog.pg_class c - JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE c.oid = rel_oid; IF sch_name IS NULL THEN @@ -497,7 +497,7 @@ END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION msar.get_constraint_type_api_code(contype "char") RETURNS TEXT AS $$/* +CREATE OR REPLACE FUNCTION msar.get_constraint_type_api_code(contype char) RETURNS TEXT AS $$/* This function returns a string that represents the constraint type code used to describe constraints when listing them within the Mathesar API. @@ -521,8 +521,8 @@ CREATE OR REPLACE FUNCTION msar.get_constraints_for_table(tab_id oid) RETURNS TA ( oid oid, name text, - columns smallint[], type text, + columns smallint[], referent_table_oid oid, referent_columns smallint[] ) @@ -536,8 +536,8 @@ WITH constraints AS ( SELECT oid, conname AS name, - conkey AS columns, msar.get_constraint_type_api_code(contype) AS type, + conkey AS columns, confrelid AS referent_table_oid, confkey AS referent_columns FROM pg_catalog.pg_constraint @@ -547,7 +547,7 @@ SELECT * FROM constraints -- Only return constraints with types that we're able to classify WHERE type IS NOT NULL -$$ LANGUAGE sql; +$$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION diff --git a/mathesar/rpc/constraints.py b/mathesar/rpc/constraints.py index 0a04e696bf..69d359a851 100644 --- a/mathesar/rpc/constraints.py +++ b/mathesar/rpc/constraints.py @@ -1,7 +1,7 @@ """ Classes and functions exposed to the RPC endpoint for managing table constraints. """ -from typing import * +from typing import Optional, TypedDict from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required @@ -10,6 +10,7 @@ from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect + class CreatableConstraintInfo(TypedDict): name: Optional[str] # TODO @@ -20,14 +21,19 @@ class Constraint(TypedDict): Information about a constraint Attributes: - oid: The OID of the schema - name: The name of the schema - description: A description of the schema - table_count: The number of tables in the schema + oid: The OID of the constraint. + name: The name of the constraint. + type: The type of the constraint. + columns: List of constrained columns. + referent_table_oid: The OID of the referent table. + referent_columns: List of referent column(s). """ oid: int table_oid: int - # TODO + type: str + columns: list[int] + referent_table_oid: Optional[int] + referent_columns: Optional[list[int]] @rpc_method(name="constraints.list") @@ -38,8 +44,8 @@ def list_(*, table_oid: int, database_id: int, **kwargs) -> list[Constraint]: List information about constraints in a table. Exposed as `list`. Args: - database_id: The Django id of the database containing the table. table_oid: The oid of the table to list constraints for. + database_id: The Django id of the database containing the table. Returns: A list of Constraint objects diff --git a/mathesar/rpc/tables/base.py b/mathesar/rpc/tables/base.py index c73e5a35bc..408e6dbff9 100644 --- a/mathesar/rpc/tables/base.py +++ b/mathesar/rpc/tables/base.py @@ -1,3 +1,6 @@ +""" +Classes and functions exposed to the RPC endpoint for managing tables in a database. +""" from typing import Optional, TypedDict from modernrpc.core import rpc_method, REQUEST_KEY From ce089b76b76c71371134de727e2a1fe122234a82 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 4 Jul 2024 00:48:03 +0530 Subject: [PATCH 0358/1141] fix docs --- docs/docs/api/rpc.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 90753fb728..f2c91ce82a 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -49,13 +49,12 @@ To use an RPC function: ## Constraints -::: mathesar.rpc.constraints +::: constraints options: members: - - add_from_known_connection - - add_from_scratch - - grant_access_to_user - - DBModelReturn + - list_ + - Constraint + - CreatableConstraintInfo ## Schemas From e3dda1b3fc09863eccab357e5b1367ab389ea7dc Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 4 Jul 2024 01:17:17 +0530 Subject: [PATCH 0359/1141] add return in drop_constraint --- db/sql/00_msar.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 9b08266bb6..873d119eb2 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -514,7 +514,7 @@ SELECT CASE contype WHEN 'u' THEN 'unique' WHEN 'x' THEN 'exclude' END; -$$ LANGUAGE sql; +$$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION msar.get_constraints_for_table(tab_id oid) RETURNS TABLE @@ -536,7 +536,7 @@ WITH constraints AS ( SELECT oid, conname AS name, - msar.get_constraint_type_api_code(contype) AS type, + msar.get_constraint_type_api_code(contype::char) AS type, conkey AS columns, confrelid AS referent_table_oid, confkey AS referent_columns @@ -2342,7 +2342,7 @@ Args: con_id: OID of the constraint to be dropped. */ BEGIN - PERFORM msar.drop_constraint( + RETURN msar.drop_constraint( msar.get_relation_schema_name(tab_id), msar.get_relation_name(tab_id), msar.get_constraint_name(con_id) From 2417cf343a56acd9ed63e1c7f2d3b1ac24618b5b Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 4 Jul 2024 18:52:09 +0530 Subject: [PATCH 0360/1141] Fix retrieval of role description --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index fe5949144f..38c8adf11a 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -834,7 +834,7 @@ FROM ( r.rolcreaterole AS create_role, r.rolcreatedb AS create_db, r.rolcanlogin AS login, - pg_catalog.obj_description(r.oid) AS description, + pg_catalog.shobj_description(r.oid, 'pg_authid') AS description, rolemembers.members AS members FROM pg_catalog.pg_roles r LEFT OUTER JOIN rolemembers ON r.oid = rolemembers.oid From 5046e927aefb10bee003e389eb04e719aaa12ecb Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 4 Jul 2024 18:52:23 +0530 Subject: [PATCH 0361/1141] Add sql tests for fetching roles --- db/sql/test_00_msar.sql | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 4e053e42d1..b12fda83be 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2738,3 +2738,59 @@ BEGIN RETURN NEXT is(jsonb_array_length(msar.get_schemas()), initial_schema_count); END; $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_get_roles() RETURNS SETOF TEXT AS $$ +DECLARE + initial_role_count int; + foo_role jsonb; + bar_role jsonb; +BEGIN + SELECT jsonb_array_length(msar.get_roles()) INTO initial_role_count; + + -- Create role and check if role is present in response & count is increased + CREATE ROLE foo; + RETURN NEXT is(jsonb_array_length(msar.get_roles()), initial_role_count + 1); + SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; + + -- Check if role has expected properties + RETURN NEXT is(jsonb_typeof(foo_role), 'object'); + RETURN NEXT is((foo_role->>'super')::boolean, false); + RETURN NEXT is((foo_role->>'inherits')::boolean, true); + RETURN NEXT is((foo_role->>'create_role')::boolean, false); + RETURN NEXT is((foo_role->>'create_db')::boolean, false); + RETURN NEXT is((foo_role->>'login')::boolean, false); + RETURN NEXT is(jsonb_typeof(foo_role->'description'), 'null'); + RETURN NEXT is(jsonb_typeof(foo_role->'members'), 'null'); + + -- Modify properties and check role again + ALTER ROLE foo WITH CREATEDB CREATEROLE LOGIN NOINHERIT; + SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; + RETURN NEXT is((foo_role->>'super')::boolean, false); + RETURN NEXT is((foo_role->>'inherits')::boolean, false); + RETURN NEXT is((foo_role->>'create_role')::boolean, true); + RETURN NEXT is((foo_role->>'create_db')::boolean, true); + RETURN NEXT is((foo_role->>'login')::boolean, true); + + -- Add comment and check if comment is present + COMMENT ON ROLE foo IS 'A test role'; + SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; + RETURN NEXT is(foo_role->'description'#>>'{}', 'A test role'); + + -- Add members and check result + CREATE ROLE bar; + GRANT foo TO bar; + RETURN NEXT is(jsonb_array_length(msar.get_roles()), initial_role_count + 2); + SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; + SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "bar")') INTO bar_role; + RETURN NEXT is(jsonb_typeof(foo_role->'members'), 'array'); + RETURN NEXT is( + foo_role->'members'->0->>'oid', bar_role->>'oid' + ); + DROP ROLE bar; + + -- Drop role and ensure role is not present in response + DROP ROLE foo; + RETURN NEXT ok(NOT jsonb_path_exists(msar.get_roles(), '$[*] ? (@.name == "foo")')); +END; +$$ LANGUAGE plpgsql; From 17b5c1d02a2ffc98deb8b08a1acd7c70c8f0c030 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 4 Jul 2024 19:12:21 +0530 Subject: [PATCH 0362/1141] add constraints delete and add endpoints --- db/constraints/operations/create.py | 6 +- db/constraints/operations/drop.py | 18 +++- db/sql/00_msar.sql | 3 +- docs/docs/api/rpc.md | 23 +++-- mathesar/rpc/constraints.py | 133 +++++++++++++++++++++++++-- mathesar/tests/rpc/test_endpoints.py | 10 ++ 6 files changed, 173 insertions(+), 20 deletions(-) diff --git a/db/constraints/operations/create.py b/db/constraints/operations/create.py index 738b65db55..d6e4d0c305 100644 --- a/db/constraints/operations/create.py +++ b/db/constraints/operations/create.py @@ -20,15 +20,15 @@ def add_constraint_via_sql_alchemy(constraint_obj, engine): ).fetchone()[0] -def create_constraint(constraint_obj, conn): +def create_constraint(table_oid, constraint_obj_list, conn): """ Create a constraint using a psycopg connection. Args: - constraint_obj: (See __msar.process_con_def_jsonb for details) + constraint_obj_list: (See __msar.process_con_def_jsonb for details) conn: a psycopg connection Returns: Returns a list of oid(s) of constraints for a given table. """ - return exec_msar_func(conn, 'add_constraints', constraint_obj).fetchone()[0] + return exec_msar_func(conn, 'add_constraints', table_oid, constraint_obj_list).fetchone()[0] diff --git a/db/constraints/operations/drop.py b/db/constraints/operations/drop.py index c2f4807098..5faeb1bba9 100644 --- a/db/constraints/operations/drop.py +++ b/db/constraints/operations/drop.py @@ -1,4 +1,4 @@ -from db.connection import execute_msar_func_with_engine +from db.connection import execute_msar_func_with_engine, exec_msar_func def drop_constraint(table_name, schema_name, engine, constraint_name): @@ -17,3 +17,19 @@ def drop_constraint(table_name, schema_name, engine, constraint_name): return execute_msar_func_with_engine( engine, 'drop_constraint', schema_name, table_name, constraint_name ).fetchone()[0] + + +def drop_constraint_via_oid(table_oid, constraint_oid, conn): + """ + Drop a constraint. + + Args: + table_oid: Identity of the table to delete constraint for. + constraint_oid: The OID of the constraint to delete. + + Returns: + The name of the dropped constraint. + """ + return exec_msar_func( + conn, 'drop_constraint', table_oid, constraint_oid + ).fetchone()[0] diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 873d119eb2..cc57971b26 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2319,7 +2319,7 @@ $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; ---------------------------------------------------------------------------------------------------- CREATE OR REPLACE FUNCTION -msar.drop_constraint(sch_name text, tab_name text, con_name text) RETURNS void AS $$/* +msar.drop_constraint(sch_name text, tab_name text, con_name text) RETURNS TEXT AS $$/* Drop a constraint Args: @@ -2329,6 +2329,7 @@ Args: */ BEGIN EXECUTE format('ALTER TABLE %I.%I DROP CONSTRAINT %I', sch_name, tab_name, con_name); + RETURN con_name; END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 6dd18e048c..3ad8d6eeee 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -47,15 +47,6 @@ To use an RPC function: - grant_access_to_user - DBModelReturn -## Constraints - -::: constraints - options: - members: - - list_ - - Constraint - - CreatableConstraintInfo - ## Schemas ::: schemas @@ -120,6 +111,20 @@ To use an RPC function: - ColumnMetaData - SettableColumnMetaData +## Constraints + +::: constraints + options: + members: + - list_ + - add + - delete + - Constraint + - ForeignKeyConstraint + - PrimaryKeyConstraint + - UniqueConstraint + - CreatableConstraintInfo + ## Responses ### Success diff --git a/mathesar/rpc/constraints.py b/mathesar/rpc/constraints.py index 69d359a851..d2d992332f 100644 --- a/mathesar/rpc/constraints.py +++ b/mathesar/rpc/constraints.py @@ -7,16 +7,80 @@ from modernrpc.auth.basic import http_basic_auth_login_required from db.constraints.operations.select import get_constraints_for_table +from db.constraints.operations.create import create_constraint +from db.constraints.operations.drop import drop_constraint_via_oid from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect -class CreatableConstraintInfo(TypedDict): +class ForeignKeyConstraint(TypedDict): + """ + Information about a foreign key constraint. + + Attributes: + name: The name of the constraint. + type: The type of the constraint(`'f'` for foreign key constraint). + columns: List of columns to set a foreign key on. + deferrable: Whether to postpone constraint checking until the end of the transaction. + fkey_relation_id: The OID of the referent table. + fkey_columns: List of referent column(s). + fkey_update_action: Specifies what action should be taken when the referenced key is updated. + Valid options include `'a'(no action)`(default behavior), `'r'(restrict)`, `'c'(cascade)`, `'n'(set null)`, `'d'(set default)` + fkey_delete_action: Specifies what action should be taken when the referenced key is deleted. + Valid options include `'a'(no action)`(default behavior), `'r'(restrict)`, `'c'(cascade)`, `'n'(set null)`, `'d'(set default)` + fkey_match_type: Specifies how the foreign key matching should be performed. + Valid options include `'f'(full match)`, `'s'(simple match)`(default behavior). + """ + name: Optional[str] + type: str = 'f' + columns: list[int] + deferrable: Optional[bool] + fkey_relation_id: int + fkey_columns: list[int] + fkey_update_action: str = 'a' | 'r' | 'c' | 'n' | 'd' + fkey_delete_action: str = 'a' | 'r' | 'c' | 'n' | 'd' + fkey_match_type: str = 's' | 'f' + + +class PrimaryKeyConstraint(TypedDict): + """ + Information about a primary key constraint. + + Attributes: + name: The name of the constraint. + type: The type of the constraint(`'p'` for primary key constraint). + columns: List of columns to set a primary key on. + deferrable: Whether to postpone constraint checking until the end of the transaction. + """ name: Optional[str] - # TODO + type: str = 'p' + columns: list[int] + deferrable: Optional[bool] + + +class UniqueConstraint(TypedDict): + """ + Information about a unique constraint. + + Attributes: + name: The name of the constraint. + type: The type of the constraint(`'u'` for unique constraint). + columns: List of columns to set a unique constraint on. + deferrable: Whether to postpone constraint checking until the end of the transaction. + """ + name: Optional[str] + type: str = 'u' + columns: list[int] + deferrable: Optional[bool] + + +CreatableConstraintInfo = list[ForeignKeyConstraint | PrimaryKeyConstraint | UniqueConstraint] +""" +Type alias for a list of createable constraints which can be unique, primary key, or foreign key constraints. +""" -class Constraint(TypedDict): +class ConstraintInfo(TypedDict): """ Information about a constraint @@ -35,11 +99,22 @@ class Constraint(TypedDict): referent_table_oid: Optional[int] referent_columns: Optional[list[int]] + @classmethod + def from_dict(cls, con_info): + return cls( + oid=con_info["oid"], + table_oid=con_info["table_oid"], + type=con_info["type"], + columns=con_info["columns"], + referent_table_oid=con_info["referent_table_oid"], + referent_columns=con_info["referent_columns"] + ) + @rpc_method(name="constraints.list") @http_basic_auth_login_required @handle_rpc_exceptions -def list_(*, table_oid: int, database_id: int, **kwargs) -> list[Constraint]: +def list_(*, table_oid: int, database_id: int, **kwargs) -> list[ConstraintInfo]: """ List information about constraints in a table. Exposed as `list`. @@ -48,8 +123,54 @@ def list_(*, table_oid: int, database_id: int, **kwargs) -> list[Constraint]: database_id: The Django id of the database containing the table. Returns: - A list of Constraint objects + A list of constraint details. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + con_info = get_constraints_for_table(table_oid, conn) + return [ConstraintInfo.from_dict(con) for con in con_info] + + +@rpc_method(name="constraints.add") +@http_basic_auth_login_required +@handle_rpc_exceptions +def add( + *, + table_oid: int, + constraint_def_list: CreatableConstraintInfo, + database_id: int, **kwargs +) -> list[int]: + """ + Add constraint(s) on a table in bulk. + + Args: + table_oid: Identity of the table to delete constraint for. + constraint_def_list: A list describing the constraints to add. + database_id: The Django id of the database containing the table. + + Returns: + The oid(s) of the created constraints. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + return create_constraint(table_oid, constraint_def_list, conn) + + +@rpc_method(name="constraints.delete") +@http_basic_auth_login_required +@handle_rpc_exceptions +def delete(*, table_oid: int, constraint_oid: int, database_id: int, **kwargs) -> str: + """ + Delete a constraint from a table. + + Args: + table_oid: Identity of the table to delete constraint for. + constraint_oid: The OID of the constraint to delete. + database_id: The Django id of the database containing the table. + + Returns: + The name of the dropped constraint. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: - return get_constraints_for_table(table_oid, conn) + return drop_constraint_via_oid(table_oid, constraint_oid, conn) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 93eb4b8ed3..a9e8661982 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -65,6 +65,16 @@ "constraints.list", [user_is_authenticated] ), + ( + constraints.add, + "constraints.add" + [user_is_authenticated] + ), + ( + constraints.delete, + "constraints.delete" + [user_is_authenticated] + ), ( schemas.add, "schemas.add", From 6c8fe6083379e336b3a2e69d7a3c6fe209308c0f Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 4 Jul 2024 19:27:04 +0530 Subject: [PATCH 0363/1141] fix docstrings --- mathesar/rpc/constraints.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mathesar/rpc/constraints.py b/mathesar/rpc/constraints.py index d2d992332f..30e936e70c 100644 --- a/mathesar/rpc/constraints.py +++ b/mathesar/rpc/constraints.py @@ -1,7 +1,7 @@ """ Classes and functions exposed to the RPC endpoint for managing table constraints. """ -from typing import Optional, TypedDict +from typing import Optional, TypedDict, Union from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required @@ -37,9 +37,9 @@ class ForeignKeyConstraint(TypedDict): deferrable: Optional[bool] fkey_relation_id: int fkey_columns: list[int] - fkey_update_action: str = 'a' | 'r' | 'c' | 'n' | 'd' - fkey_delete_action: str = 'a' | 'r' | 'c' | 'n' | 'd' - fkey_match_type: str = 's' | 'f' + fkey_update_action: str + fkey_delete_action: str + fkey_match_type: str class PrimaryKeyConstraint(TypedDict): @@ -74,7 +74,7 @@ class UniqueConstraint(TypedDict): deferrable: Optional[bool] -CreatableConstraintInfo = list[ForeignKeyConstraint | PrimaryKeyConstraint | UniqueConstraint] +CreatableConstraintInfo = list[Union[ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint]] """ Type alias for a list of createable constraints which can be unique, primary key, or foreign key constraints. """ From f92677d65879e0e958812fc930f6623601b469ad Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 5 Jul 2024 10:23:34 +0800 Subject: [PATCH 0364/1141] add basic utilities to set up a new DB --- mathesar/rpc/connections.py | 2 +- mathesar/utils/permissions.py | 57 ++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/mathesar/rpc/connections.py b/mathesar/rpc/connections.py index 578b56f061..8c3fdf4ed0 100644 --- a/mathesar/rpc/connections.py +++ b/mathesar/rpc/connections.py @@ -143,4 +143,4 @@ def grant_access_to_user(*, connection_id: int, user_id: int): connection_id: The Django id of an old-style connection. user_id: The Django id of a user. """ - permissions.create_user_database_role_map(connection_id, user_id) + permissions.migrate_connection_for_user(connection_id, user_id) diff --git a/mathesar/utils/permissions.py b/mathesar/utils/permissions.py index c9272a036b..c05b53d53c 100644 --- a/mathesar/utils/permissions.py +++ b/mathesar/utils/permissions.py @@ -1,20 +1,63 @@ +from django.db import transaction +from django.conf import settings + +from db import install from mathesar.models.base import Server, Database, Role, UserDatabaseRoleMap from mathesar.models.deprecated import Connection from mathesar.models.users import User +from mathesar.utils.connections import BadInstallationTarget + +INTERNAL_DB_KEY = 'default' -def create_user_database_role_map(connection_id, user_id): +@transaction.atomic +def migrate_connection_for_user(connection_id, user_id): """Move data from old-style connection model to new models.""" conn = Connection.current_objects.get(id=connection_id) - - server = Server.objects.get_or_create(host=conn.host, port=conn.port)[0] - database = Database.objects.get_or_create(name=conn.db_name, server=server)[0] - role = Role.objects.get_or_create( - name=conn.username, server=server, password=conn.password - )[0] + server, database, role = _setup_connection_models( + conn.host, conn.port, conn.db_name, conn.username, conn.password + ) return UserDatabaseRoleMap.objects.create( user=User.objects.get(id=user_id), database=database, role=role, server=server ) + + +@transaction.atomic +def set_up_new_database_on_internal_server(database_name): + """Create a database on the internal server, install Mathesar.""" + conn_info = settings.DATABASES[INTERNAL_DB_KEY] + if database_name == conn_info["NAME"]: + raise BadInstallationTarget( + "Mathesar can't be installed in the internal database." + ) + _setup_connection_models( + conn_info["HOST"], + conn_info["PORT"], + conn_info["NAME"], + conn_info["USER"], + conn_info["PASSWORD"], + ) + install.install_mathesar( + database_name, + conn_info["USER"], + conn_info["PASSWORD"], + conn_info["HOST"], + conn_info["PORT"], + True, + root_db=conn_info["NAME"], + ) + + + +def _setup_connection_models(host, port, db_name, role_name, password): + server, _ = Server.objects.get_or_create(host=host, port=port) + database, _ = Database.objects.get_or_create(name=db_name, server=server) + role, _ = Role.objects.get_or_create( + name=role_name, + server=server, + defaults={"password": password}, + ) + return server, database, role From 96c87569e9439a7e53803e721ed260b9d05c1602 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 5 Jul 2024 10:37:08 +0800 Subject: [PATCH 0365/1141] move user mapping into setup function --- mathesar/utils/permissions.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/mathesar/utils/permissions.py b/mathesar/utils/permissions.py index c05b53d53c..de87a196e0 100644 --- a/mathesar/utils/permissions.py +++ b/mathesar/utils/permissions.py @@ -10,24 +10,22 @@ INTERNAL_DB_KEY = 'default' -@transaction.atomic def migrate_connection_for_user(connection_id, user_id): """Move data from old-style connection model to new models.""" conn = Connection.current_objects.get(id=connection_id) - server, database, role = _setup_connection_models( - conn.host, conn.port, conn.db_name, conn.username, conn.password - ) - return UserDatabaseRoleMap.objects.create( - user=User.objects.get(id=user_id), - database=database, - role=role, - server=server + user = User.objects.get(id=user_id) + return _setup_connection_models( + conn.host, conn.port, conn.db_name, conn.username, conn.password, user ) @transaction.atomic -def set_up_new_database_on_internal_server(database_name): - """Create a database on the internal server, install Mathesar.""" +def set_up_new_database_for_user_on_internal_server(database_name, user): + """ + Create a database on the internal server and install Mathesar. + + This database will be set up to be accessible for the given user. + """ conn_info = settings.DATABASES[INTERNAL_DB_KEY] if database_name == conn_info["NAME"]: raise BadInstallationTarget( @@ -39,6 +37,7 @@ def set_up_new_database_on_internal_server(database_name): conn_info["NAME"], conn_info["USER"], conn_info["PASSWORD"], + user ) install.install_mathesar( database_name, @@ -51,8 +50,8 @@ def set_up_new_database_on_internal_server(database_name): ) - -def _setup_connection_models(host, port, db_name, role_name, password): +@transaction.atomic +def _setup_connection_models(host, port, db_name, role_name, password, user): server, _ = Server.objects.get_or_create(host=host, port=port) database, _ = Database.objects.get_or_create(name=db_name, server=server) role, _ = Role.objects.get_or_create( @@ -60,4 +59,9 @@ def _setup_connection_models(host, port, db_name, role_name, password): server=server, defaults={"password": password}, ) - return server, database, role + return UserDatabaseRoleMap.objects.create( + user=user, + database=database, + role=role, + server=server + ) From 7c354c3a04475d2af8be74f177f3826111e4e47b Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 5 Jul 2024 13:58:13 +0800 Subject: [PATCH 0366/1141] rename connection object to match deprecated model --- mathesar/rpc/connections.py | 32 +++++++++++++------------- mathesar/tests/rpc/test_connections.py | 4 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/mathesar/rpc/connections.py b/mathesar/rpc/connections.py index 8c3fdf4ed0..c0486b1498 100644 --- a/mathesar/rpc/connections.py +++ b/mathesar/rpc/connections.py @@ -10,12 +10,12 @@ from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -class DBModelReturn(TypedDict): +class ConnectionReturn(TypedDict): """ - Information about a database model. + Information about a connection model. Attributes: - id (int): The Django id of the Database object added. + id (int): The Django id of the Connection object added. nickname (str): Used to identify the added connection. database (str): The name of the database on the server. username (str): The username of the role for the connection. @@ -30,14 +30,14 @@ class DBModelReturn(TypedDict): port: int @classmethod - def from_db_model(cls, db_model): + def from_model(cls, connection): return cls( - id=db_model.id, - nickname=db_model.name, - database=db_model.db_name, - username=db_model.username, - host=db_model.host, - port=db_model.port + id=connection.id, + nickname=connection.name, + database=connection.db_name, + username=connection.username, + host=connection.host, + port=connection.port ) @@ -51,7 +51,7 @@ def add_from_known_connection( create_db: bool = False, connection_id: int = None, sample_data: list[str] = [], -) -> DBModelReturn: +) -> ConnectionReturn: """ Add a new connection from an already existing one. @@ -80,10 +80,10 @@ def add_from_known_connection( 'connection_type': connection_type, 'connection_id': connection_id } - db_model = connections.copy_connection_from_preexisting( + connection_model = connections.copy_connection_from_preexisting( connection, nickname, database, create_db, sample_data ) - return DBModelReturn.from_db_model(db_model) + return ConnectionReturn.from_model(connection_model) @rpc_method(name='connections.add_from_scratch') @@ -98,7 +98,7 @@ def add_from_scratch( host: str, port: int, sample_data: list[str] = [], -) -> DBModelReturn: +) -> ConnectionReturn: """ Add a new connection to a PostgreSQL server from scratch. @@ -121,10 +121,10 @@ def add_from_scratch( Returns: Metadata about the Database associated with the connection. """ - db_model = connections.create_connection_from_scratch( + connection_model = connections.create_connection_from_scratch( user, password, host, port, nickname, database, sample_data ) - return DBModelReturn.from_db_model(db_model) + return ConnectionReturn.from_model(connection_model) @rpc_method(name='connections.grant_access_to_user') diff --git a/mathesar/tests/rpc/test_connections.py b/mathesar/tests/rpc/test_connections.py index da884c6daf..ea6cf14661 100644 --- a/mathesar/tests/rpc/test_connections.py +++ b/mathesar/tests/rpc/test_connections.py @@ -18,7 +18,7 @@ ] ) def test_add_from_known_connection(create_db, connection_id, sample_data): - with patch.object(rpc_conn, 'DBModelReturn'): + with patch.object(rpc_conn, 'ConnectionReturn'): with patch.object( rpc_conn.connections, 'copy_connection_from_preexisting' @@ -56,7 +56,7 @@ def test_add_from_known_connection(create_db, connection_id, sample_data): ] ) def test_add_from_scratch(port, sample_data): - with patch.object(rpc_conn, 'DBModelReturn'): + with patch.object(rpc_conn, 'ConnectionReturn'): with patch.object( rpc_conn.connections, 'create_connection_from_scratch' From de1f46a2b3af3e0e1a4ec50a8272b14490cf413e Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 5 Jul 2024 14:11:59 +0800 Subject: [PATCH 0367/1141] use correct database name in models --- mathesar/utils/permissions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mathesar/utils/permissions.py b/mathesar/utils/permissions.py index de87a196e0..eaf67812d7 100644 --- a/mathesar/utils/permissions.py +++ b/mathesar/utils/permissions.py @@ -31,10 +31,10 @@ def set_up_new_database_for_user_on_internal_server(database_name, user): raise BadInstallationTarget( "Mathesar can't be installed in the internal database." ) - _setup_connection_models( + user_db_role_map = _setup_connection_models( conn_info["HOST"], conn_info["PORT"], - conn_info["NAME"], + database_name, conn_info["USER"], conn_info["PASSWORD"], user @@ -48,6 +48,7 @@ def set_up_new_database_for_user_on_internal_server(database_name, user): True, root_db=conn_info["NAME"], ) + return user_db_role_map @transaction.atomic From 8f691a0b4127d11b2d3ef85d07c93d77c6f5e991 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 5 Jul 2024 15:19:23 +0800 Subject: [PATCH 0368/1141] move data sample loading to new functions --- mathesar/examples/library_dataset.py | 30 ++++++++++++-------------- mathesar/examples/movies_dataset.py | 32 ++++++++++++++-------------- mathesar/utils/connections.py | 16 +------------- mathesar/utils/permissions.py | 32 +++++++++++++++++++++++----- 4 files changed, 58 insertions(+), 52 deletions(-) diff --git a/mathesar/examples/library_dataset.py b/mathesar/examples/library_dataset.py index 1c04a52907..0afeae0efa 100644 --- a/mathesar/examples/library_dataset.py +++ b/mathesar/examples/library_dataset.py @@ -1,29 +1,27 @@ """This module contains functions to load the Library Management dataset.""" -from sqlalchemy import text +from psycopg import sql from mathesar.examples.base import LIBRARY_MANAGEMENT, LIBRARY_ONE, LIBRARY_TWO -def load_library_dataset(engine, safe_mode=False): +def load_library_dataset(conn): """ Load the library dataset into a "Library Management" schema. Args: - engine: an SQLAlchemy engine defining the connection to load data into. - safe_mode: When True, we will throw an error if the "Library Management" - schema already exists instead of dropping it. + conn: a psycopg (3) connection for loading the data. - Uses given engine to define database to load into. - Destructive, and will knock out any previous "Library Management" - schema in the given database, unless safe_mode=True. + Uses given connection to define database to load into. Raises an + Exception if the "Library Management" schema already exists. """ - drop_schema_query = text(f"""DROP SCHEMA IF EXISTS "{LIBRARY_MANAGEMENT}" CASCADE;""") - create_schema_query = text(f"""CREATE SCHEMA "{LIBRARY_MANAGEMENT}";""") - set_search_path = text(f"""SET search_path="{LIBRARY_MANAGEMENT}";""") - with engine.begin() as conn, open(LIBRARY_ONE) as f1, open(LIBRARY_TWO) as f2: - if safe_mode is False: - conn.execute(drop_schema_query) + create_schema_query = sql.SQL("CREATE SCHEMA {}").format( + sql.Identifier(LIBRARY_MANAGEMENT) + ) + set_search_path = sql.SQL("SET search_path={}").format( + sql.Identifier(LIBRARY_MANAGEMENT) + ) + with open(LIBRARY_ONE) as f1, open(LIBRARY_TWO) as f2: conn.execute(create_schema_query) conn.execute(set_search_path) - conn.execute(text(f1.read())) - conn.execute(text(f2.read())) + conn.execute(f1.read()) + conn.execute(f2.read()) diff --git a/mathesar/examples/movies_dataset.py b/mathesar/examples/movies_dataset.py index 854a3d80c4..735ed24611 100644 --- a/mathesar/examples/movies_dataset.py +++ b/mathesar/examples/movies_dataset.py @@ -1,31 +1,31 @@ """This module contains functions to load the Movie Collection dataset.""" import os -from sqlalchemy import text +from psycopg import sql from mathesar.examples.base import ( MOVIE_COLLECTION, MOVIES_SQL_TABLES, MOVIES_CSV, MOVIES_SQL_FKS ) -def load_movies_dataset(engine, safe_mode=False): +def load_movies_dataset(conn): """ Load the movie example data set. Args: - engine: an SQLAlchemy engine defining the connection to load data into. - safe_mode: When True, we will throw an error if the "Movie Collection" - schema already exists instead of dropping it. + conn: a psycopg (3) connection for loading the data. + + Uses given connection to define database to load into. Raises an + Exception if the "Movie Collection" schema already exists. """ - drop_schema_query = text(f"""DROP SCHEMA IF EXISTS "{MOVIE_COLLECTION}" CASCADE;""") - with engine.begin() as conn, open(MOVIES_SQL_TABLES) as f, open(MOVIES_SQL_FKS) as f2: - if safe_mode is False: - conn.execute(drop_schema_query) - conn.execute(text(f.read())) + with open(MOVIES_SQL_TABLES) as f, open(MOVIES_SQL_FKS) as f2: + conn.execute(f.read()) for file in os.scandir(MOVIES_CSV): table_name = file.name.split('.csv')[0] - with open(file, 'r') as csv_file: - conn.connection.cursor().copy_expert( - f"""COPY "{MOVIE_COLLECTION}"."{table_name}" FROM STDIN DELIMITER ',' CSV HEADER""", - csv_file - ) - conn.execute(text(f2.read())) + copy_sql = sql.SQL( + "COPY {}.{} FROM STDIN DELIMITER ',' CSV HEADER" + ).format( + sql.Identifier(MOVIE_COLLECTION), sql.Identifier(table_name) + ) + with open(file, 'r') as csv, conn.cursor().copy(copy_sql) as copy: + copy.write(csv.read()) + conn.execute(f2.read()) diff --git a/mathesar/utils/connections.py b/mathesar/utils/connections.py index 95fd15bd79..ae5add0224 100644 --- a/mathesar/utils/connections.py +++ b/mathesar/utils/connections.py @@ -4,8 +4,6 @@ from mathesar.models.deprecated import Connection from db import install, connection as dbconn from mathesar.state import reset_reflection -from mathesar.examples.library_dataset import load_library_dataset -from mathesar.examples.movies_dataset import load_movies_dataset class BadInstallationTarget(Exception): @@ -85,19 +83,7 @@ def _save_and_install( def _load_sample_data(engine, sample_data): - DATASET_MAP = { - 'library_management': load_library_dataset, - 'movie_collection': load_movies_dataset, - } - for key in sample_data: - try: - DATASET_MAP[key](engine, safe_mode=True) - except ProgrammingError as e: - if isinstance(e.orig, DuplicateSchema): - # We swallow this error, since otherwise we'll raise an error on the - # front end even though installation generally succeeded. - continue - reset_reflection() + pass def _validate_conn_model(conn_model): diff --git a/mathesar/utils/permissions.py b/mathesar/utils/permissions.py index eaf67812d7..bfa22e0555 100644 --- a/mathesar/utils/permissions.py +++ b/mathesar/utils/permissions.py @@ -1,7 +1,9 @@ from django.db import transaction from django.conf import settings -from db import install +from db.install import install_mathesar +from mathesar.examples.library_dataset import load_library_dataset +from mathesar.examples.movies_dataset import load_movies_dataset from mathesar.models.base import Server, Database, Role, UserDatabaseRoleMap from mathesar.models.deprecated import Connection from mathesar.models.users import User @@ -20,7 +22,9 @@ def migrate_connection_for_user(connection_id, user_id): @transaction.atomic -def set_up_new_database_for_user_on_internal_server(database_name, user): +def set_up_new_database_for_user_on_internal_server( + database_name, user, sample_data=[] +): """ Create a database on the internal server and install Mathesar. @@ -31,7 +35,7 @@ def set_up_new_database_for_user_on_internal_server(database_name, user): raise BadInstallationTarget( "Mathesar can't be installed in the internal database." ) - user_db_role_map = _setup_connection_models( + user_database_role = _setup_connection_models( conn_info["HOST"], conn_info["PORT"], database_name, @@ -39,7 +43,7 @@ def set_up_new_database_for_user_on_internal_server(database_name, user): conn_info["PASSWORD"], user ) - install.install_mathesar( + install_mathesar( database_name, conn_info["USER"], conn_info["PASSWORD"], @@ -48,7 +52,9 @@ def set_up_new_database_for_user_on_internal_server(database_name, user): True, root_db=conn_info["NAME"], ) - return user_db_role_map + with user_database_role.connection as conn: + _load_sample_data(conn, sample_data) + return user_database_role @transaction.atomic @@ -66,3 +72,19 @@ def _setup_connection_models(host, port, db_name, role_name, password, user): role=role, server=server ) + + +def _load_sample_data(conn, sample_data): + DATASET_MAP = { + 'library_management': load_library_dataset, + 'movie_collection': load_movies_dataset, + } + for key in sample_data: + try: + DATASET_MAP[key](conn) + except ProgrammingError as e: + if isinstance(e.orig, DuplicateSchema): + # We swallow this error, since otherwise we'll raise an + # error on the front end even though installation + # generally succeeded. + continue From 0dc212a7239a00b5b90718769133e452954df5f0 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 5 Jul 2024 16:24:50 +0800 Subject: [PATCH 0369/1141] wire up create_new to the RPC endpoint --- config/settings/common_settings.py | 1 + mathesar/rpc/database_setup.py | 62 ++++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 6 +++ 3 files changed, 69 insertions(+) create mode 100644 mathesar/rpc/database_setup.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index c574b418d5..3eb67f194e 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -68,6 +68,7 @@ def pipe_delim(pipe_string): 'mathesar.rpc.connections', 'mathesar.rpc.columns', 'mathesar.rpc.columns.metadata', + 'mathesar.rpc.database_setup', 'mathesar.rpc.schemas', 'mathesar.rpc.tables', 'mathesar.rpc.tables.metadata' diff --git a/mathesar/rpc/database_setup.py b/mathesar/rpc/database_setup.py new file mode 100644 index 0000000000..ea48b1f194 --- /dev/null +++ b/mathesar/rpc/database_setup.py @@ -0,0 +1,62 @@ +""" +RPC functions for setting up database connections. +""" +from typing import TypedDict + +from modernrpc.core import rpc_method, REQUEST_KEY +from modernrpc.auth.basic import http_basic_auth_superuser_required + +from mathesar.utils import permissions +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions + + +class DatabaseConnectionResult(TypedDict): + """ + Info about the objects resulting from calling the setup functions. + + These functions will get or create an instance of the Server, + Database, and Role models, as well as a UserDatabaseRoleMap entry. + + Attributes: + server_id: The Django ID of the Server model instance. + database_id: The Django ID of the Database model instance. + role_id: The Django ID of the Role model instance. + """ + server_id: int + database_id: int + role_id: int + + @classmethod + def from_model(cls, model): + return cls( + server_id=model.server.id, + database_id=model.database.id, + role_id=model.role.id, + ) + +@rpc_method(name='database_setup.create_new') +@http_basic_auth_superuser_required +@handle_rpc_exceptions +def create_new( + *, + database_name: str, + sample_data: list[str] = [], + **kwargs +) -> DatabaseConnectionResult: + """ + Set up a new database on the internal server. + + The calling user will get access to that database using the default + role stored in Django settings. + + Args: + database_name: The name of the new database. + sample_data: A list of strings requesting that some example data + sets be installed on the underlying database. Valid list + members are 'library_management' and 'movie_collection'. + """ + user = kwargs.get(REQUEST_KEY).user + result = permissions.set_up_new_database_for_user_on_internal_server( + database_name, user, sample_data=sample_data + ) + return DatabaseConnectionResult.from_model(result) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index b786e99027..890e7d4538 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -10,6 +10,7 @@ from mathesar.rpc import columns from mathesar.rpc import connections +from mathesar.rpc import database_setup from mathesar.rpc import schemas from mathesar.rpc import tables @@ -54,6 +55,11 @@ "connections.grant_access_to_user", [user_is_superuser] ), + ( + database_setup.create_new, + "database_setup.create_new", + [user_is_superuser] + ), ( schemas.add, "schemas.add", From c11ad16b36c92e03079337489776950fa3b5cd44 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 5 Jul 2024 14:45:50 +0530 Subject: [PATCH 0370/1141] fix typo --- mathesar/tests/rpc/test_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index a9e8661982..6328340408 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -67,12 +67,12 @@ ), ( constraints.add, - "constraints.add" + "constraints.add", [user_is_authenticated] ), ( constraints.delete, - "constraints.delete" + "constraints.delete", [user_is_authenticated] ), ( From 47bbb6eb27b30280c681497b5277f569aef4c62a Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 5 Jul 2024 15:06:10 +0530 Subject: [PATCH 0371/1141] Add unit tests --- db/tests/roles/__init__.py | 0 db/tests/roles/operations/__init__.py | 0 db/tests/roles/operations/test_select.py | 10 ++++++++++ mathesar/tests/rpc/test_endpoints.py | 6 ++++++ 4 files changed, 16 insertions(+) create mode 100644 db/tests/roles/__init__.py create mode 100644 db/tests/roles/operations/__init__.py create mode 100644 db/tests/roles/operations/test_select.py diff --git a/db/tests/roles/__init__.py b/db/tests/roles/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/db/tests/roles/operations/__init__.py b/db/tests/roles/operations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/db/tests/roles/operations/test_select.py b/db/tests/roles/operations/test_select.py new file mode 100644 index 0000000000..40579e99ac --- /dev/null +++ b/db/tests/roles/operations/test_select.py @@ -0,0 +1,10 @@ +from unittest.mock import patch +from db.roles.operations import select as ma_sel + + +def test_get_roles(): + with patch.object(ma_sel, 'exec_msar_func') as mock_exec: + mock_exec.return_value.fetchone = lambda: ('a', 'b') + result = ma_sel.get_roles('conn') + mock_exec.assert_called_once_with('conn', 'get_roles') + assert result == 'a' diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index b786e99027..50f109d1ac 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -10,6 +10,7 @@ from mathesar.rpc import columns from mathesar.rpc import connections +from mathesar.rpc import roles from mathesar.rpc import schemas from mathesar.rpc import tables @@ -54,6 +55,11 @@ "connections.grant_access_to_user", [user_is_superuser] ), + ( + roles.list_, + "roles.list", + [user_is_authenticated] + ), ( schemas.add, "schemas.add", From c381e3cb189f3129a14653fe2e2bce790e8187fa Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 5 Jul 2024 15:35:01 +0530 Subject: [PATCH 0372/1141] Add documentation --- docs/docs/api/rpc.md | 9 +++++++++ mathesar/rpc/roles.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 27dcf9d911..dbcbe9b315 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -111,6 +111,15 @@ To use an RPC function: - ColumnMetaData - SettableColumnMetaData +## Roles + +::: roles + options: + members: + - list_ + - RoleInfo + - RoleMember + ## Responses ### Success diff --git a/mathesar/rpc/roles.py b/mathesar/rpc/roles.py index 16a02d4ea1..0db0c10c3c 100644 --- a/mathesar/rpc/roles.py +++ b/mathesar/rpc/roles.py @@ -11,12 +11,38 @@ from db.roles.operations.select import get_roles -class RoleMembers(TypedDict): +class RoleMember(TypedDict): + """ + Information about a member role of an inherited role. + + Attributes: + oid: The OID of the member role. + admin: Whether the member role has ADMIN option on the inherited role. + """ oid: int admin: bool class RoleInfo(TypedDict): + """ + Information about a role. + + Attributes: + oid: The OID of the role. + name: Name of the role. + super: Whether the role has SUPERUSER status. + inherits: Whether the role has INHERIT attribute. + create_role: Whether the role has CREATEROLE attribute. + create_db: Whether the role has CREATEDB attribute. + login: Whether the role has LOGIN attribute. + description: A description of the role + members: The member roles that inherit the role. + + Refer PostgreSQL documenation on: + - [pg_roles table](https://www.postgresql.org/docs/current/view-pg-roles.html). + - [Role attributes](https://www.postgresql.org/docs/current/role-attributes.html) + - [Role membership](https://www.postgresql.org/docs/current/role-membership.html) + """ oid: int name: str super: bool @@ -25,7 +51,7 @@ class RoleInfo(TypedDict): create_db: bool login: bool description: Optional[str] - members: Optional[list[RoleMembers]] + members: Optional[list[RoleMember]] @rpc_method(name="roles.list") @@ -35,8 +61,10 @@ def list_(*, database_id: int, **kwargs) -> list[RoleInfo]: """ List information about roles for a database server. Exposed as `list`. Requires a database id inorder to connect to the server. + Args: database_id: The Django id of the database containing the table. + Returns: A list of roles present on the database server. """ From 80940a302803cd6d30397879c532c530ce1bc55a Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 5 Jul 2024 18:11:59 +0530 Subject: [PATCH 0373/1141] fix create_constraint --- db/constraints/operations/create.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/constraints/operations/create.py b/db/constraints/operations/create.py index d6e4d0c305..45b9b4600f 100644 --- a/db/constraints/operations/create.py +++ b/db/constraints/operations/create.py @@ -1,3 +1,5 @@ +import json + from db.connection import execute_msar_func_with_engine, exec_msar_func @@ -31,4 +33,4 @@ def create_constraint(table_oid, constraint_obj_list, conn): Returns: Returns a list of oid(s) of constraints for a given table. """ - return exec_msar_func(conn, 'add_constraints', table_oid, constraint_obj_list).fetchone()[0] + return exec_msar_func(conn, 'add_constraints', table_oid, json.dumps(constraint_obj_list)).fetchone()[0] From 18329ecedca03f28069f62e7dd3c1440785c9261 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 5 Jul 2024 18:23:27 +0530 Subject: [PATCH 0374/1141] specify optionals in docstrings --- mathesar/rpc/constraints.py | 10 +++++----- mathesar/tests/rpc/test_constraints.py | 0 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 mathesar/tests/rpc/test_constraints.py diff --git a/mathesar/rpc/constraints.py b/mathesar/rpc/constraints.py index 30e936e70c..fe021540bf 100644 --- a/mathesar/rpc/constraints.py +++ b/mathesar/rpc/constraints.py @@ -37,9 +37,9 @@ class ForeignKeyConstraint(TypedDict): deferrable: Optional[bool] fkey_relation_id: int fkey_columns: list[int] - fkey_update_action: str - fkey_delete_action: str - fkey_match_type: str + fkey_update_action: Optional[str] + fkey_delete_action: Optional[str] + fkey_match_type: Optional[str] class PrimaryKeyConstraint(TypedDict): @@ -93,7 +93,7 @@ class ConstraintInfo(TypedDict): referent_columns: List of referent column(s). """ oid: int - table_oid: int + name: str type: str columns: list[int] referent_table_oid: Optional[int] @@ -103,7 +103,7 @@ class ConstraintInfo(TypedDict): def from_dict(cls, con_info): return cls( oid=con_info["oid"], - table_oid=con_info["table_oid"], + name=con_info["name"], type=con_info["type"], columns=con_info["columns"], referent_table_oid=con_info["referent_table_oid"], diff --git a/mathesar/tests/rpc/test_constraints.py b/mathesar/tests/rpc/test_constraints.py new file mode 100644 index 0000000000..e69de29bb2 From bc6e7790d20b98facfb9a2e0beead1dc10430201 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 5 Jul 2024 18:26:53 +0530 Subject: [PATCH 0375/1141] reorder arguments --- mathesar/rpc/constraints.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mathesar/rpc/constraints.py b/mathesar/rpc/constraints.py index fe021540bf..2cdaaee322 100644 --- a/mathesar/rpc/constraints.py +++ b/mathesar/rpc/constraints.py @@ -18,12 +18,12 @@ class ForeignKeyConstraint(TypedDict): Information about a foreign key constraint. Attributes: - name: The name of the constraint. type: The type of the constraint(`'f'` for foreign key constraint). columns: List of columns to set a foreign key on. - deferrable: Whether to postpone constraint checking until the end of the transaction. fkey_relation_id: The OID of the referent table. fkey_columns: List of referent column(s). + name: The name of the constraint. + deferrable: Whether to postpone constraint checking until the end of the transaction. fkey_update_action: Specifies what action should be taken when the referenced key is updated. Valid options include `'a'(no action)`(default behavior), `'r'(restrict)`, `'c'(cascade)`, `'n'(set null)`, `'d'(set default)` fkey_delete_action: Specifies what action should be taken when the referenced key is deleted. @@ -31,12 +31,12 @@ class ForeignKeyConstraint(TypedDict): fkey_match_type: Specifies how the foreign key matching should be performed. Valid options include `'f'(full match)`, `'s'(simple match)`(default behavior). """ - name: Optional[str] type: str = 'f' columns: list[int] - deferrable: Optional[bool] fkey_relation_id: int fkey_columns: list[int] + name: Optional[str] + deferrable: Optional[bool] fkey_update_action: Optional[str] fkey_delete_action: Optional[str] fkey_match_type: Optional[str] @@ -47,14 +47,14 @@ class PrimaryKeyConstraint(TypedDict): Information about a primary key constraint. Attributes: - name: The name of the constraint. type: The type of the constraint(`'p'` for primary key constraint). columns: List of columns to set a primary key on. + name: The name of the constraint. deferrable: Whether to postpone constraint checking until the end of the transaction. """ - name: Optional[str] type: str = 'p' columns: list[int] + name: Optional[str] deferrable: Optional[bool] @@ -63,14 +63,14 @@ class UniqueConstraint(TypedDict): Information about a unique constraint. Attributes: - name: The name of the constraint. type: The type of the constraint(`'u'` for unique constraint). columns: List of columns to set a unique constraint on. + name: The name of the constraint. deferrable: Whether to postpone constraint checking until the end of the transaction. """ - name: Optional[str] type: str = 'u' columns: list[int] + name: Optional[str] deferrable: Optional[bool] From 2a4e78c59ee4e7a84e0b7da762b6800d2005f87f Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 5 Jul 2024 18:54:19 +0530 Subject: [PATCH 0376/1141] add pytests --- mathesar/rpc/constraints.py | 2 +- mathesar/tests/rpc/test_constraints.py | 156 +++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 1 deletion(-) diff --git a/mathesar/rpc/constraints.py b/mathesar/rpc/constraints.py index 2cdaaee322..dd5aef8fb4 100644 --- a/mathesar/rpc/constraints.py +++ b/mathesar/rpc/constraints.py @@ -149,7 +149,7 @@ def add( database_id: The Django id of the database containing the table. Returns: - The oid(s) of the created constraints. + The oid(s) of all the constraints on the table. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: diff --git a/mathesar/tests/rpc/test_constraints.py b/mathesar/tests/rpc/test_constraints.py index e69de29bb2..6153594dd0 100644 --- a/mathesar/tests/rpc/test_constraints.py +++ b/mathesar/tests/rpc/test_constraints.py @@ -0,0 +1,156 @@ +""" +This file tests the constraint RPC functions. + +Fixtures: + monkeypatch(pytest): Lets you monkeypatch an object for testing. +""" +from contextlib import contextmanager + +from mathesar.rpc import constraints +from mathesar.models.users import User + + +def test_constraints_list(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + table_oid = 2254444 + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_constaints_list(_table_oid, conn): + if _table_oid != table_oid: + raise AssertionError('incorrect parameters passed') + return [ + { + 'oid': 2254567, + 'name': 'Movie Cast Map_Cast Member_fkey', + 'type': 'foreignkey', + 'columns': [4], + 'referent_table_oid': 2254492, + 'referent_columns': [1] + }, + { + 'oid': 2254572, + 'name': 'Movie Cast Map_Movie_fkey', + 'type': 'foreignkey', + 'columns': [3], + 'referent_table_oid': 2254483, + 'referent_columns': [1] + }, + { + 'oid': 2254544, + 'name': 'Movie Cast Map_pkey', + 'type': 'primary', + 'columns': [1], + 'referent_table_oid': 0, + 'referent_columns': None + } + ] + monkeypatch.setattr(constraints, 'connect', mock_connect) + monkeypatch.setattr(constraints, 'get_constraints_for_table', mock_constaints_list) + expect_constraints_list = [ + { + 'oid': 2254567, + 'name': 'Movie Cast Map_Cast Member_fkey', + 'type': 'foreignkey', + 'columns': [4], + 'referent_table_oid': 2254492, + 'referent_columns': [1] + }, + { + 'oid': 2254572, + 'name': 'Movie Cast Map_Movie_fkey', + 'type': 'foreignkey', + 'columns': [3], + 'referent_table_oid': 2254483, + 'referent_columns': [1] + }, + { + 'oid': 2254544, + 'name': 'Movie Cast Map_pkey', + 'type': 'primary', + 'columns': [1], + 'referent_table_oid': 0, + 'referent_columns': None + } + ] + actual_constraint_list = constraints.list_(table_oid=table_oid, database_id=11, request=request) + assert actual_constraint_list == expect_constraints_list + + +def test_constraints_drop(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + table_oid = 2254444 + constraint_oid = 2254567 + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_constaints_delete(_table_oid, _constraint_oid, conn): + if _table_oid != table_oid and _constraint_oid != constraint_oid: + raise AssertionError('incorrect parameters passed') + return 'Movie Cast Map_Cast Member_fkey' + monkeypatch.setattr(constraints, 'connect', mock_connect) + monkeypatch.setattr(constraints, 'drop_constraint_via_oid', mock_constaints_delete) + constraint_name = constraints.delete( + table_oid=table_oid, constraint_oid=constraint_oid, database_id=11, request=request + ) + assert constraint_name == 'Movie Cast Map_Cast Member_fkey' + + +def test_constraints_create(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + table_oid = 2254444 + constraint_def_list = [ + { + 'name': 'Movie Cast Map_Movie_fkey', + 'type': 'f', + 'columns': [3], + 'fkey_relation_id': 2254483, + 'fkey_columns': [1], + 'fkey_update_actions': 'a', + 'fkey_delete_action': 'a', + 'fkey_match_type': 's' + } + ] + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_constaints_create(_table_oid, _constraint_def_list, conn): + if _table_oid != table_oid and _constraint_def_list != constraint_def_list: + raise AssertionError('incorrect parameters passed') + return [2254833, 2254567, 2254544] + monkeypatch.setattr(constraints, 'connect', mock_connect) + monkeypatch.setattr(constraints, 'create_constraint', mock_constaints_create) + constraint_oids = constraints.add( + table_oid=table_oid, constraint_def_list=constraint_def_list, database_id=11, request=request + ) + assert constraint_oids == [2254833, 2254567, 2254544] From 8e50a96f97ac754203385dac70168ee7ccc29907 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 8 Jul 2024 15:14:33 +0800 Subject: [PATCH 0377/1141] add function for setting up preexisting DB for a user --- mathesar/rpc/database_setup.py | 1 + mathesar/utils/connections.py | 4 +--- mathesar/utils/permissions.py | 44 +++++++++++++++++++++++++++------- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/mathesar/rpc/database_setup.py b/mathesar/rpc/database_setup.py index ea48b1f194..27439f6287 100644 --- a/mathesar/rpc/database_setup.py +++ b/mathesar/rpc/database_setup.py @@ -34,6 +34,7 @@ def from_model(cls, model): role_id=model.role.id, ) + @rpc_method(name='database_setup.create_new') @http_basic_auth_superuser_required @handle_rpc_exceptions diff --git a/mathesar/utils/connections.py b/mathesar/utils/connections.py index ae5add0224..000efb7ea5 100644 --- a/mathesar/utils/connections.py +++ b/mathesar/utils/connections.py @@ -1,9 +1,7 @@ """Utilities to help with creating and managing connections in Mathesar.""" -from psycopg2.errors import DuplicateSchema -from sqlalchemy.exc import OperationalError, ProgrammingError +from sqlalchemy.exc import OperationalError from mathesar.models.deprecated import Connection from db import install, connection as dbconn -from mathesar.state import reset_reflection class BadInstallationTarget(Exception): diff --git a/mathesar/utils/permissions.py b/mathesar/utils/permissions.py index bfa22e0555..1c3ae45d82 100644 --- a/mathesar/utils/permissions.py +++ b/mathesar/utils/permissions.py @@ -1,5 +1,6 @@ from django.db import transaction from django.conf import settings +from psycopg.errors import DuplicateSchema from db.install import install_mathesar from mathesar.examples.library_dataset import load_library_dataset @@ -58,9 +59,37 @@ def set_up_new_database_for_user_on_internal_server( @transaction.atomic -def _setup_connection_models(host, port, db_name, role_name, password, user): +def set_up_preexisting_database_for_user( + host, port, database_name, role_name, password, user, sample_data=[] +): + internal_conn_info = settings.DATABASES[INTERNAL_DB_KEY] + if ( + host == internal_conn_info["HOST"] + and port == internal_conn_info["PORT"] + and database_name == internal_conn_info["NAME"] + ): + raise BadInstallationTarget( + "Mathesar can't be installed in the internal database." + ) + user_database_role = _setup_connection_models( + host, port, database_name, role_name, password, user + ) + install_mathesar( + database_name, role_name, password, host, port, True, create_db=False, + ) + with user_database_role.connection as conn: + _load_sample_data(conn, sample_data) + return user_database_role + + +@transaction.atomic +def _setup_connection_models( + host, port, database_name, role_name, password, user +): server, _ = Server.objects.get_or_create(host=host, port=port) - database, _ = Database.objects.get_or_create(name=db_name, server=server) + database, _ = Database.objects.get_or_create( + name=database_name, server=server + ) role, _ = Role.objects.get_or_create( name=role_name, server=server, @@ -82,9 +111,8 @@ def _load_sample_data(conn, sample_data): for key in sample_data: try: DATASET_MAP[key](conn) - except ProgrammingError as e: - if isinstance(e.orig, DuplicateSchema): - # We swallow this error, since otherwise we'll raise an - # error on the front end even though installation - # generally succeeded. - continue + except DuplicateSchema: + # We swallow this error, since otherwise we'll raise an + # error on the front end even though installation + # generally succeeded. + continue From bd1ddf07126c8805dc0ce6a54af278e213a31ba8 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 8 Jul 2024 16:10:12 +0800 Subject: [PATCH 0378/1141] wire up new create_existing to RPC endpoint --- mathesar/rpc/database_setup.py | 42 ++++++++++++++++++++++++++-- mathesar/tests/rpc/test_endpoints.py | 5 ++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/mathesar/rpc/database_setup.py b/mathesar/rpc/database_setup.py index 27439f6287..37ea89969b 100644 --- a/mathesar/rpc/database_setup.py +++ b/mathesar/rpc/database_setup.py @@ -40,7 +40,7 @@ def from_model(cls, model): @handle_rpc_exceptions def create_new( *, - database_name: str, + database: str, sample_data: list[str] = [], **kwargs ) -> DatabaseConnectionResult: @@ -51,13 +51,49 @@ def create_new( role stored in Django settings. Args: - database_name: The name of the new database. + database: The name of the new database. sample_data: A list of strings requesting that some example data sets be installed on the underlying database. Valid list members are 'library_management' and 'movie_collection'. """ user = kwargs.get(REQUEST_KEY).user result = permissions.set_up_new_database_for_user_on_internal_server( - database_name, user, sample_data=sample_data + database, user, sample_data=sample_data + ) + return DatabaseConnectionResult.from_model(result) + + +@rpc_method(name='database_setup.connect_existing') +@http_basic_auth_superuser_required +@handle_rpc_exceptions +def connect_existing( + *, + host: str, + port: int, + database: str, + role: str, + password: str, + sample_data: list[str] = [], + **kwargs +) -> DatabaseConnectionResult: + """ + Connect Mathesar to an existing database on a server. + + The calling user will get access to that database using the + credentials passed to this function. + + Args: + host: The host of the database server. + port: The port of the database server. + database: The name of the database on the server. + role: The role on the server to use for the connection. + password: A password valid for the role. + sample_data: A list of strings requesting that some example data + sets be installed on the underlying database. Valid list + members are 'library_management' and 'movie_collection'. + """ + user = kwargs.get(REQUEST_KEY).user + result = permissions.set_up_preexisting_database_for_user( + host, port, database, role, password, user, sample_data=sample_data ) return DatabaseConnectionResult.from_model(result) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 890e7d4538..e010d544ca 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -60,6 +60,11 @@ "database_setup.create_new", [user_is_superuser] ), + ( + database_setup.connect_existing, + "database_setup.connect_existing", + [user_is_superuser] + ), ( schemas.add, "schemas.add", From 2f046ca3a0ee45d5ecc2b77a42c268eafa521f56 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 8 Jul 2024 17:34:38 +0800 Subject: [PATCH 0379/1141] add wiring tests for database setup functions --- mathesar/tests/rpc/test_database_setup.py | 91 +++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 mathesar/tests/rpc/test_database_setup.py diff --git a/mathesar/tests/rpc/test_database_setup.py b/mathesar/tests/rpc/test_database_setup.py new file mode 100644 index 0000000000..1e30679fbd --- /dev/null +++ b/mathesar/tests/rpc/test_database_setup.py @@ -0,0 +1,91 @@ +""" +This file tests the database_setup RPC functions. + +Fixtures: + rf(pytest-django): Provides mocked `Request` objects. + monkeypatch(pytest): Lets you monkeypatch an object for testing. +""" +from mathesar.models.users import User +from mathesar.models.base import Database, Role, Server, UserDatabaseRoleMap +from mathesar.rpc import database_setup + + +def test_create_new(monkeypatch, rf): + test_sample_data = ["movie_collection"] + test_database = "mathesar42" + request = rf.post("/api/rpc/v0/", data={}) + request.user = User(username="alice", password="pass1234") + + def mock_set_up_new_for_user(database, user, sample_data=[]): + if not ( + database == test_database + and user == request.user + and sample_data == test_sample_data + ): + raise AssertionError("incorrect parameters passed") + server_model = Server(id=2, host="example.com", port=5432) + db_model = Database(id=3, name=test_database, server=server_model) + role_model = Role(id=4, name="matheuser", server=server_model) + return UserDatabaseRoleMap( + user=user, database=db_model, role=role_model, server=server_model + ) + + monkeypatch.setattr( + database_setup.permissions, + "set_up_new_database_for_user_on_internal_server", + mock_set_up_new_for_user, + ) + expect_response = database_setup.DatabaseConnectionResult( + server_id=2, database_id=3, role_id=4 + ) + + actual_response = database_setup.create_new( + database=test_database, sample_data=test_sample_data, request=request + ) + assert actual_response == expect_response + + +def test_connect_existing(monkeypatch, rf): + test_sample_data = ["movie_collection"] + test_database = "mathesar42" + test_host = "example.com" + test_port = 6543 + test_role = "ernie" + test_password = "ernie1234" + request = rf.post("/api/rpc/v0/", data={}) + request.user = User(username="alice", password="pass1234") + + def mock_set_up_preexisting_database_for_user( + host, port, database_name, role_name, password, user, sample_data=[] + ): + if not ( + host == test_host + and port == test_port + and database_name == test_database + and role_name == test_role + and password == test_password + and user == request.user + and sample_data == test_sample_data + ): + raise AssertionError("incorrect parameters passed") + server_model = Server(id=2, host="example.com", port=5432) + db_model = Database(id=3, name=test_database, server=server_model) + role_model = Role(id=4, name="matheuser", server=server_model) + return UserDatabaseRoleMap( + user=user, database=db_model, role=role_model, server=server_model + ) + + monkeypatch.setattr( + database_setup.permissions, + "set_up_preexisting_database_for_user", + mock_set_up_preexisting_database_for_user, + ) + expect_response = database_setup.DatabaseConnectionResult( + server_id=2, database_id=3, role_id=4 + ) + + actual_response = database_setup.connect_existing( + host=test_host, port=test_port, database=test_database, role=test_role, + password=test_password, sample_data=test_sample_data, request=request + ) + assert actual_response == expect_response From b2aa9c16bbe96889557b1a652ca2ec92e66f718f Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 8 Jul 2024 18:04:46 +0800 Subject: [PATCH 0380/1141] add databases.list RPC function --- config/settings/common_settings.py | 1 + mathesar/rpc/databases/__init__.py | 1 + mathesar/rpc/databases/base.py | 51 ++++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 6 ++++ 4 files changed, 59 insertions(+) create mode 100644 mathesar/rpc/databases/__init__.py create mode 100644 mathesar/rpc/databases/base.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index 3eb67f194e..db01dd848c 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -69,6 +69,7 @@ def pipe_delim(pipe_string): 'mathesar.rpc.columns', 'mathesar.rpc.columns.metadata', 'mathesar.rpc.database_setup', + 'mathesar.rpc.databases', 'mathesar.rpc.schemas', 'mathesar.rpc.tables', 'mathesar.rpc.tables.metadata' diff --git a/mathesar/rpc/databases/__init__.py b/mathesar/rpc/databases/__init__.py new file mode 100644 index 0000000000..4b40b38c84 --- /dev/null +++ b/mathesar/rpc/databases/__init__.py @@ -0,0 +1 @@ +from .base import * # noqa diff --git a/mathesar/rpc/databases/base.py b/mathesar/rpc/databases/base.py new file mode 100644 index 0000000000..f4f89ed0e4 --- /dev/null +++ b/mathesar/rpc/databases/base.py @@ -0,0 +1,51 @@ +from typing import TypedDict + +from modernrpc.core import rpc_method +from modernrpc.auth.basic import http_basic_auth_login_required + +from mathesar.models.base import Database +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions + + +class DatabaseInfo(TypedDict): + """ + Information about a database. + + id: the Django ID of the database model instance. + name: The name of the database on the server. + server_id: the Django ID of the server model instance for the database. + """ + id: int + name: str + server_id: int + + @classmethod + def from_model(cls, model): + return cls( + id=model.id, + name=model.name, + server_id=model.server.id + ) + + +@rpc_method(name="databases.list") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_(*, server_id: int = None, **kwargs) -> list[DatabaseInfo]: + """ + List information about databases for a server. Exposed as `list`. + + If called with no `server_id`, all databases for all servers are listed. + + Args: + server_id: The Django id of the server containing the databases. + + Returns: + A list of database details. + """ + if server_id is not None: + database_qs = Database.objects.filter(server__id=server_id) + else: + database_qs = Database.objects.all() + + return [DatabaseInfo.from_model(db_model) for db_model in database_qs] diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index e010d544ca..8f863e4cf7 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -11,6 +11,7 @@ from mathesar.rpc import columns from mathesar.rpc import connections from mathesar.rpc import database_setup +from mathesar.rpc import databases from mathesar.rpc import schemas from mathesar.rpc import tables @@ -65,6 +66,11 @@ "database_setup.connect_existing", [user_is_superuser] ), + ( + databases.list_, + "databases.list", + [user_is_authenticated] + ), ( schemas.add, "schemas.add", From 5feae6cedf8a354553da7c9479ee7aaa09cacfd5 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 8 Jul 2024 18:21:21 +0800 Subject: [PATCH 0381/1141] integrate new functions into docs --- docs/docs/api/rpc.md | 19 ++++++++++++++++++- mathesar/rpc/databases/base.py | 7 ++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index f3fd42eb25..05a66b6778 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -45,7 +45,24 @@ To use an RPC function: - add_from_known_connection - add_from_scratch - grant_access_to_user - - DBModelReturn + - ConnectionReturn + +## Databases + +::: databases + options: + members: + - list_ + - DatabaseInfo + +## Database Setup + +::: database_setup + options: + members: + - create_new + - connect_existing + - DatabaseConnectionResult ## Schemas diff --git a/mathesar/rpc/databases/base.py b/mathesar/rpc/databases/base.py index f4f89ed0e4..abc9d43b89 100644 --- a/mathesar/rpc/databases/base.py +++ b/mathesar/rpc/databases/base.py @@ -11,9 +11,10 @@ class DatabaseInfo(TypedDict): """ Information about a database. - id: the Django ID of the database model instance. - name: The name of the database on the server. - server_id: the Django ID of the server model instance for the database. + Attributes: + id: the Django ID of the database model instance. + name: The name of the database on the server. + server_id: the Django ID of the server model instance for the database. """ id: int name: str From ddf2704aef7d757a00203375090455ae1487e822 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 8 Jul 2024 21:13:53 -0400 Subject: [PATCH 0382/1141] Cast OIDs to bigint before putting in json --- db/sql/00_msar.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index e5fce3cf1a..56194ca149 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -716,9 +716,9 @@ Args: tab_id: The OID or name of the table. */ SELECT jsonb_build_object( - 'oid', oid, + 'oid', oid::bigint, 'name', relname, - 'schema', relnamespace, + 'schema', relnamespace::bigint, 'description', msar.obj_description(oid, 'pg_class') ) FROM pg_catalog.pg_class WHERE oid = tab_id; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -740,9 +740,9 @@ Args: */ SELECT jsonb_agg( jsonb_build_object( - 'oid', pgc.oid, + 'oid', pgc.oid::bigint, 'name', pgc.relname, - 'schema', pgc.relnamespace, + 'schema', pgc.relnamespace::bigint, 'description', msar.obj_description(pgc.oid, 'pg_class') ) ) @@ -772,7 +772,7 @@ Each returned JSON object in the array will have the form: SELECT jsonb_agg(schema_data) FROM ( SELECT - s.oid AS oid, + s.oid::bigint AS oid, s.nspname AS name, pg_catalog.obj_description(s.oid) AS description, COALESCE(count(c.oid), 0) AS table_count @@ -2120,7 +2120,7 @@ SELECT jsonb_agg( 'type', contype, 'columns', ARRAY[attname], 'deferrable', condeferrable, - 'fkey_relation_id', confrelid::integer, + 'fkey_relation_id', confrelid::bigint, 'fkey_columns', confkey, 'fkey_update_action', confupdtype, 'fkey_delete_action', confdeltype, From 4900cd2ca48228606f1fa4b3b7da45bb90c417ea Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 8 Jul 2024 22:14:30 -0400 Subject: [PATCH 0383/1141] Improve standard on casting OIDs to JSON --- db/sql/STANDARDS.md | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/db/sql/STANDARDS.md b/db/sql/STANDARDS.md index 81ead6d690..93f12bf518 100644 --- a/db/sql/STANDARDS.md +++ b/db/sql/STANDARDS.md @@ -62,23 +62,35 @@ Always qualify system catalog tables by prefixing them with `pg_catalog.`. If yo Always cast OID values to `bigint` before putting them in JSON (or jsonb). -_Don't_ cast OID values to `integer`. +- _Don't_ leave OID values in JSON without casting. -This is because the [`oid` type](https://www.postgresql.org/docs/current/datatype-oid.html) is an _unsigned_ 32-bit integer whereas the `integer` type is a _signed_ 32-bit integer. That means it's possible for a database to have OID values which don't fit into the `integer` type. + This is because (strangely!) raw OID values become _strings_ in JSON unless you cast them. -For example, putting a large OID value into JSON by casting it to an integer will cause overflow: + ```sql + SELECT jsonb_build_object('foo', 42::oid); -- ❌ Bad + ``` -```SQL -SELECT jsonb_build_object('foo', 3333333333::oid::integer); -- ❌ Bad -``` + > `{"foo": "42"}` -> `{"foo": -961633963}` + If you keep OID values as strings in JSON, it can cause bugs. For example, a client may later feed a received OID value back to the DB layer when making a modification to a DB object. If the client sends a stringifed OID back to the DB layer, it might get treated as a _name_ instead of an OID due to function overloading. + +- _Don't_ cast OID values to `integer`. + + This is because the [`oid` type](https://www.postgresql.org/docs/current/datatype-oid.html) is an _unsigned_ 32-bit integer whereas the `integer` type is a _signed_ 32-bit integer. That means it's possible for a database to have OID values which don't fit into the `integer` type. + + For example, putting a large OID value into JSON by casting it to an integer will cause overflow: + + ```SQL + SELECT jsonb_build_object('foo', 3333333333::oid::integer); -- ❌ Bad + ``` + + > `{"foo": -961633963}` -Instead, cast it to `bigint` + Instead, cast it to `bigint` -```SQL -SELECT jsonb_build_object('foo', 3333333333::oid::bigint); -- ✅ Good -``` + ```SQL + SELECT jsonb_build_object('foo', 3333333333::oid::bigint); -- ✅ Good + ``` -> `{"foo": 3333333333}` + > `{"foo": 3333333333}` From 4b89b3696540e0e6aae507062d5d7a77b0859a81 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 9 Jul 2024 18:56:39 +0530 Subject: [PATCH 0384/1141] add python functions for joinable tables --- db/tables/operations/select.py | 6 ++- mathesar/rpc/tables/base.py | 74 +++++++++++++++++++++++++++- mathesar/tests/rpc/test_endpoints.py | 5 ++ 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/db/tables/operations/select.py b/db/tables/operations/select.py index e25ad3c976..e0d4b7365e 100644 --- a/db/tables/operations/select.py +++ b/db/tables/operations/select.py @@ -3,7 +3,7 @@ ) from sqlalchemy.dialects.postgresql import JSONB -from db.connection import exec_msar_func +from db.connection import exec_msar_func, select_from_msar_func from db.utils import execute_statement, get_pg_catalog_table BASE = 'base' @@ -59,6 +59,10 @@ def get_table_info(schema, conn): return exec_msar_func(conn, 'get_table_info', schema).fetchone()[0] +def list_joinable_tables(table_oid, conn, max_depth): + return select_from_msar_func(conn, 'get_joinable_tables', max_depth, table_oid) + + def reflect_table(name, schema, engine, metadata, connection_to_use=None, keep_existing=False): extend_existing = not keep_existing autoload_with = engine if connection_to_use is None else connection_to_use diff --git a/mathesar/rpc/tables/base.py b/mathesar/rpc/tables/base.py index 408e6dbff9..f4d09d5e29 100644 --- a/mathesar/rpc/tables/base.py +++ b/mathesar/rpc/tables/base.py @@ -6,7 +6,7 @@ from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required -from db.tables.operations.select import get_table_info, get_table +from db.tables.operations.select import get_table_info, get_table, list_joinable_tables from db.tables.operations.drop import drop_table_from_database from db.tables.operations.create import create_table_on_database from db.tables.operations.alter import alter_table_on_database @@ -54,6 +54,52 @@ class SettableTableInfo(TypedDict): columns: Optional[list[SettableColumnInfo]] +class JoinableTableInfo(TypedDict): + """ + Information about a joinable table. + + Attributes: + base: The OID of the table from which the paths start + target: The OID of the table where the paths end. + join_path: A list describing joinable paths in the following form: + [ + [[L_oid0, L_attnum0], [R_oid0, R_attnum0]], + [[L_oid1, L_attnum1], [R_oid1, R_attnum1]], + [[L_oid2, L_attnum2], [R_oid2, R_attnum2]], + ... + ] + + Here, [L_oidN, L_attnumN] represents the left column of a join, and [R_oidN, R_attnumN] the right. + fkey_path: Same as `join_path` expressed in terms of foreign key constraints in the following form: + [ + [constraint_id0, reversed], + [constraint_id1, reversed], + ] + + In this form, `constraint_idN` is a foreign key constraint, and `reversed` is a boolean giving + whether to travel from referrer to referant (when False) or from referant to referrer (when True). + depth: Specifies how far to search for joinable tables. + multiple_results: Specifies whether the path included is reversed. + """ + base: int + target: int + join_path: list[list[list[int, int], list[int, int]]] + fkey_path: list[list[int, str]] + depth: int + multiple_results: bool + + @classmethod + def from_dict(cls, joinables): + return cls( + base=joinables["base"], + target=joinables["target"], + join_path=joinables["join_path"], + fkey_path=joinables["fkey_path"], + depth=joinables["depth"], + multiple_results=joinables["multiple_reslults"] + ) + + @rpc_method(name="tables.list") @http_basic_auth_login_required @handle_rpc_exceptions @@ -231,3 +277,29 @@ def get_import_preview( user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: return get_preview(table_oid, columns, conn, limit) + + +@rpc_method(name="tables.list_joinable") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_joinable( + *, + table_oid: int, + database_id: int, + max_depth: int = 3, + **kwargs +) -> JoinableTableInfo: + """ + List details for joinable tables. + + Args: + table_oid: Identity of the table to get joinable tables for. + database_id: The Django id of the database containing the table. + max_depth: Specifies how far to search for joinable tables. + + Returns: + Joinable table details for a given table. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + return list_joinable_tables(table_oid, conn, max_depth) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index ca91d17b2e..87f4c95f90 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -136,6 +136,11 @@ "tables.get_import_preview", [user_is_authenticated] ), + ( + tables.list_joinable, + "tables.list_joinable", + [user_is_authenticated] + ), ( tables.metadata.list_, "tables.metadata.list", From a096c0934ed8c58b2ce3863381cd0cc4b69a23ae Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 9 Jul 2024 19:01:42 +0530 Subject: [PATCH 0385/1141] add docs --- docs/docs/api/rpc.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 37efba2c64..3284f38223 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -71,8 +71,10 @@ To use an RPC function: - patch - import_ - get_import_preview + - list_joinable - TableInfo - SettableTableInfo + - JoinableTableInfo ## Table Metadata From 4d580cda3189e467bbb1122fb038af9535fb3243 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Tue, 9 Jul 2024 09:43:34 -0400 Subject: [PATCH 0386/1141] Cast OID values to bigint in msar.get_roles --- db/sql/00_msar.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 366b4d2aaa..666c07a534 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -915,7 +915,7 @@ WITH rolemembers as ( pgr.oid AS oid, jsonb_agg( jsonb_build_object( - 'oid', pgm.member, + 'oid', pgm.member::bigint, 'admin', pgm.admin_option ) ) AS members @@ -926,7 +926,7 @@ WITH rolemembers as ( SELECT jsonb_agg(role_data) FROM ( SELECT - r.oid AS oid, + r.oid::bigint AS oid, r.rolname AS name, r.rolsuper AS super, r.rolinherit AS inherits, From 5e35ffa23d5dd1f85e8bd1f3a7074c9d3e3c8083 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 9 Jul 2024 19:14:59 +0530 Subject: [PATCH 0387/1141] install joinable_tables sql --- db/sql/install.py | 3 +++ mathesar/rpc/tables/base.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/db/sql/install.py b/db/sql/install.py index 8de4c7e2f2..eb46915338 100644 --- a/db/sql/install.py +++ b/db/sql/install.py @@ -3,6 +3,7 @@ FILE_DIR = os.path.abspath(os.path.dirname(__file__)) MSAR_SQL = os.path.join(FILE_DIR, '00_msar.sql') +MSAR_JOIN_SQL = os.path.join(FILE_DIR, '10_msar_joinable_tables.sql') MSAR_AGGREGATE_SQL = os.path.join(FILE_DIR, '30_msar_custom_aggregates.sql') @@ -10,5 +11,7 @@ def install(engine): """Install SQL pieces using the given engine.""" with open(MSAR_SQL) as file_handle: load_file_with_engine(engine, file_handle) + with open(MSAR_JOIN_SQL) as file_handle: + load_file_with_engine(engine, file_handle) with open(MSAR_AGGREGATE_SQL) as custom_aggregates: load_file_with_engine(engine, custom_aggregates) diff --git a/mathesar/rpc/tables/base.py b/mathesar/rpc/tables/base.py index f4d09d5e29..76b8bba731 100644 --- a/mathesar/rpc/tables/base.py +++ b/mathesar/rpc/tables/base.py @@ -83,8 +83,8 @@ class JoinableTableInfo(TypedDict): """ base: int target: int - join_path: list[list[list[int, int], list[int, int]]] - fkey_path: list[list[int, str]] + join_path: list + fkey_path: list depth: int multiple_results: bool @@ -302,4 +302,5 @@ def list_joinable( """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: - return list_joinable_tables(table_oid, conn, max_depth) + joinables = list_joinable_tables(table_oid, conn, max_depth) + return [JoinableTableInfo.from_dict(joinable) for joinable in joinables] From 1a2742940462675082d0e79773c69d9456d1f4bc Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 9 Jul 2024 19:34:42 +0530 Subject: [PATCH 0388/1141] add tests --- mathesar/tests/rpc/tables/test_t_base.py | 125 +++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/mathesar/tests/rpc/tables/test_t_base.py b/mathesar/tests/rpc/tables/test_t_base.py index 693f9b901d..26ce8ece30 100644 --- a/mathesar/tests/rpc/tables/test_t_base.py +++ b/mathesar/tests/rpc/tables/test_t_base.py @@ -267,3 +267,128 @@ def mock_table_preview(_table_oid, columns, conn, limit): {'id': 3, 'length': Decimal('4.0')}, {'id': 4, 'length': Decimal('5.22')} ] + + +def test_list_joinable(rf, monkeypatch): + request = rf.post('/api/rpc/v0', data={}) + request.user = User(username='alice', password='pass1234') + table_oid = 2254329 + database_id = 11 + + @contextmanager + def mock_connect(_database_id, user): + if _database_id == database_id and user.username == 'alice': + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_list_joinable_tables(_table_oid, conn, max_depth): + if _table_oid != table_oid: + raise AssertionError('incorrect parameters passed') + return [ + { + 'base': 2254329, + 'target': 2254334, + 'join_path': [[[2254329, 2], [2254334, 1]]], + 'fkey_path': [['2254406', False]], + 'depth': 1, + 'multiple_results': False + }, + { + 'base': 2254329, + 'target': 2254350, + 'join_path': [[[2254329, 3], [2254350, 1]]], + 'fkey_path': [['2254411', False]], + 'depth': 1, + 'multiple_results': False + }, + { + 'base': 2254329, + 'target': 2254321, + 'join_path': [[[2254329, 2], [2254334, 1]], [[2254334, 5], [2254321, 1]]], + 'fkey_path': [['2254406', False], ['2254399', False]], + 'depth': 2, + 'multiple_results': False + }, + { + 'base': 2254329, + 'target': 2254358, + 'join_path': [ + [[2254329, 2], [2254334, 1]], + [[2254334, 5], [2254321, 1]], + [[2254321, 11], [2254358, 1]] + ], + 'fkey_path': [['2254406', False], ['2254399', False], ['2254394', False]], + 'depth': 3, + 'multiple_results': False + }, + { + 'base': 2254329, + 'target': 2254313, + 'join_path': [ + [[2254329, 2], [2254334, 1]], + [[2254334, 5], [2254321, 1]], + [[2254321, 10], [2254313, 1]] + ], + 'fkey_path': [['2254406', False], ['2254399', False], ['2254389', False]], + 'depth': 3, + 'multiple_results': False + } + ] + expected_list = [ + { + 'base': 2254329, + 'target': 2254334, + 'join_path': [[[2254329, 2], [2254334, 1]]], + 'fkey_path': [['2254406', False]], + 'depth': 1, + 'multiple_results': False + }, + { + 'base': 2254329, + 'target': 2254350, + 'join_path': [[[2254329, 3], [2254350, 1]]], + 'fkey_path': [['2254411', False]], + 'depth': 1, + 'multiple_results': False + }, + { + 'base': 2254329, + 'target': 2254321, + 'join_path': [[[2254329, 2], [2254334, 1]], [[2254334, 5], [2254321, 1]]], + 'fkey_path': [['2254406', False], ['2254399', False]], + 'depth': 2, + 'multiple_results': False + }, + { + 'base': 2254329, + 'target': 2254358, + 'join_path': [ + [[2254329, 2], [2254334, 1]], + [[2254334, 5], [2254321, 1]], + [[2254321, 11], [2254358, 1]] + ], + 'fkey_path': [['2254406', False], ['2254399', False], ['2254394', False]], + 'depth': 3, + 'multiple_results': False + }, + { + 'base': 2254329, + 'target': 2254313, + 'join_path': [ + [[2254329, 2], [2254334, 1]], + [[2254334, 5], [2254321, 1]], + [[2254321, 10], [2254313, 1]] + ], + 'fkey_path': [['2254406', False], ['2254399', False], ['2254389', False]], + 'depth': 3, + 'multiple_results': False + } + ] + monkeypatch.setattr(tables.base, 'connect', mock_connect) + monkeypatch.setattr(tables.base, 'list_joinable_tables', mock_list_joinable_tables) + actual_list = tables.list_joinable(table_oid=2254329, database_id=11, max_depth=3) + assert expected_list == actual_list From 625ed0d49494a57fb4fd562ba4308f6105ee49f3 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 9 Jul 2024 20:07:40 +0530 Subject: [PATCH 0389/1141] change namespace for joinable_tables type --- db/sql/10_msar_joinable_tables.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/sql/10_msar_joinable_tables.sql b/db/sql/10_msar_joinable_tables.sql index 96e255a4fd..dd62775216 100644 --- a/db/sql/10_msar_joinable_tables.sql +++ b/db/sql/10_msar_joinable_tables.sql @@ -29,7 +29,7 @@ whether to travel from referrer to referant (when False) or from referant to ref */ -CREATE TYPE mathesar_types.joinable_tables AS ( +CREATE TYPE __msar.joinable_tables AS ( base integer, -- The OID of the table from which the paths start target integer, -- The OID of the table where the paths end join_path jsonb, -- A JSONB array of arrays of arrays @@ -40,8 +40,8 @@ CREATE TYPE mathesar_types.joinable_tables AS ( CREATE OR REPLACE FUNCTION -msar.get_joinable_tables(max_depth integer) RETURNS SETOF mathesar_types.joinable_tables AS $$/* -This function returns a table of mathesar_types.joinable_tables objects, giving paths to various +msar.get_joinable_tables(max_depth integer) RETURNS SETOF __msar.joinable_tables AS $$/* +This function returns a table of __msar.joinable_tables objects, giving paths to various joinable tables. Args: @@ -132,6 +132,6 @@ $$ LANGUAGE sql; CREATE OR REPLACE FUNCTION msar.get_joinable_tables(max_depth integer, table_id oid) RETURNS - SETOF mathesar_types.joinable_tables AS $$ + SETOF __msar.joinable_tables AS $$ SELECT * FROM msar.get_joinable_tables(max_depth) WHERE base=table_id $$ LANGUAGE sql; From fa2e7d56755cbe802eaac28783e9c4511ec14629 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 9 Jul 2024 20:23:10 +0530 Subject: [PATCH 0390/1141] fix test --- mathesar/rpc/tables/base.py | 2 +- mathesar/tests/rpc/tables/test_t_base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mathesar/rpc/tables/base.py b/mathesar/rpc/tables/base.py index 76b8bba731..606bbd7d11 100644 --- a/mathesar/rpc/tables/base.py +++ b/mathesar/rpc/tables/base.py @@ -96,7 +96,7 @@ def from_dict(cls, joinables): join_path=joinables["join_path"], fkey_path=joinables["fkey_path"], depth=joinables["depth"], - multiple_results=joinables["multiple_reslults"] + multiple_results=joinables["multiple_results"] ) diff --git a/mathesar/tests/rpc/tables/test_t_base.py b/mathesar/tests/rpc/tables/test_t_base.py index 26ce8ece30..16bcb24bf1 100644 --- a/mathesar/tests/rpc/tables/test_t_base.py +++ b/mathesar/tests/rpc/tables/test_t_base.py @@ -390,5 +390,5 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): ] monkeypatch.setattr(tables.base, 'connect', mock_connect) monkeypatch.setattr(tables.base, 'list_joinable_tables', mock_list_joinable_tables) - actual_list = tables.list_joinable(table_oid=2254329, database_id=11, max_depth=3) + actual_list = tables.list_joinable(table_oid=2254329, database_id=11, max_depth=3, request=request) assert expected_list == actual_list From b1bc21ff666583b3dd104e633adbccdeee9a2a2c Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 9 Jul 2024 22:11:46 +0530 Subject: [PATCH 0391/1141] move joinable_tables type from __msar to msar --- db/sql/10_msar_joinable_tables.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/sql/10_msar_joinable_tables.sql b/db/sql/10_msar_joinable_tables.sql index dd62775216..c1877903e9 100644 --- a/db/sql/10_msar_joinable_tables.sql +++ b/db/sql/10_msar_joinable_tables.sql @@ -29,7 +29,7 @@ whether to travel from referrer to referant (when False) or from referant to ref */ -CREATE TYPE __msar.joinable_tables AS ( +CREATE TYPE msar.joinable_tables AS ( base integer, -- The OID of the table from which the paths start target integer, -- The OID of the table where the paths end join_path jsonb, -- A JSONB array of arrays of arrays @@ -40,8 +40,8 @@ CREATE TYPE __msar.joinable_tables AS ( CREATE OR REPLACE FUNCTION -msar.get_joinable_tables(max_depth integer) RETURNS SETOF __msar.joinable_tables AS $$/* -This function returns a table of __msar.joinable_tables objects, giving paths to various +msar.get_joinable_tables(max_depth integer) RETURNS SETOF msar.joinable_tables AS $$/* +This function returns a table of msar.joinable_tables objects, giving paths to various joinable tables. Args: @@ -132,6 +132,6 @@ $$ LANGUAGE sql; CREATE OR REPLACE FUNCTION msar.get_joinable_tables(max_depth integer, table_id oid) RETURNS - SETOF __msar.joinable_tables AS $$ + SETOF msar.joinable_tables AS $$ SELECT * FROM msar.get_joinable_tables(max_depth) WHERE base=table_id $$ LANGUAGE sql; From 1b473ab65d73ab8e4b371aaa244eb7e76920612b Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 10 Jul 2024 18:10:41 +0530 Subject: [PATCH 0392/1141] convert oids from text -> bigint --- db/sql/10_msar_joinable_tables.sql | 12 ++++++------ mathesar/tests/rpc/tables/test_t_base.py | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/db/sql/10_msar_joinable_tables.sql b/db/sql/10_msar_joinable_tables.sql index c1877903e9..cf51e772a9 100644 --- a/db/sql/10_msar_joinable_tables.sql +++ b/db/sql/10_msar_joinable_tables.sql @@ -53,9 +53,9 @@ restrictions in either way. */ WITH RECURSIVE symmetric_fkeys AS ( SELECT - c.oid fkey_oid, - c.conrelid::INTEGER left_rel, - c.confrelid::INTEGER right_rel, + c.oid::BIGINT fkey_oid, + c.conrelid::BIGINT left_rel, + c.confrelid::BIGINT right_rel, c.conkey[1]::INTEGER left_col, c.confkey[1]::INTEGER right_col, false multiple_results, @@ -64,9 +64,9 @@ WITH RECURSIVE symmetric_fkeys AS ( WHERE c.contype='f' and array_length(c.conkey, 1)=1 UNION ALL SELECT - c.oid fkey_oid, - c.confrelid::INTEGER left_rel, - c.conrelid::INTEGER right_rel, + c.oid::BIGINT fkey_oid, + c.confrelid::BIGINT left_rel, + c.conrelid::BIGINT right_rel, c.confkey[1]::INTEGER left_col, c.conkey[1]::INTEGER right_col, true multiple_results, diff --git a/mathesar/tests/rpc/tables/test_t_base.py b/mathesar/tests/rpc/tables/test_t_base.py index 16bcb24bf1..3190502f9b 100644 --- a/mathesar/tests/rpc/tables/test_t_base.py +++ b/mathesar/tests/rpc/tables/test_t_base.py @@ -293,7 +293,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): 'base': 2254329, 'target': 2254334, 'join_path': [[[2254329, 2], [2254334, 1]]], - 'fkey_path': [['2254406', False]], + 'fkey_path': [[2254406, False]], 'depth': 1, 'multiple_results': False }, @@ -301,7 +301,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): 'base': 2254329, 'target': 2254350, 'join_path': [[[2254329, 3], [2254350, 1]]], - 'fkey_path': [['2254411', False]], + 'fkey_path': [[2254411, False]], 'depth': 1, 'multiple_results': False }, @@ -309,7 +309,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): 'base': 2254329, 'target': 2254321, 'join_path': [[[2254329, 2], [2254334, 1]], [[2254334, 5], [2254321, 1]]], - 'fkey_path': [['2254406', False], ['2254399', False]], + 'fkey_path': [[2254406, False], [2254399, False]], 'depth': 2, 'multiple_results': False }, @@ -321,7 +321,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): [[2254334, 5], [2254321, 1]], [[2254321, 11], [2254358, 1]] ], - 'fkey_path': [['2254406', False], ['2254399', False], ['2254394', False]], + 'fkey_path': [[2254406, False], [2254399, False], [2254394, False]], 'depth': 3, 'multiple_results': False }, @@ -333,7 +333,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): [[2254334, 5], [2254321, 1]], [[2254321, 10], [2254313, 1]] ], - 'fkey_path': [['2254406', False], ['2254399', False], ['2254389', False]], + 'fkey_path': [[2254406, False], [2254399, False], [2254389, False]], 'depth': 3, 'multiple_results': False } @@ -343,7 +343,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): 'base': 2254329, 'target': 2254334, 'join_path': [[[2254329, 2], [2254334, 1]]], - 'fkey_path': [['2254406', False]], + 'fkey_path': [[2254406, False]], 'depth': 1, 'multiple_results': False }, @@ -351,7 +351,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): 'base': 2254329, 'target': 2254350, 'join_path': [[[2254329, 3], [2254350, 1]]], - 'fkey_path': [['2254411', False]], + 'fkey_path': [[2254411, False]], 'depth': 1, 'multiple_results': False }, @@ -359,7 +359,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): 'base': 2254329, 'target': 2254321, 'join_path': [[[2254329, 2], [2254334, 1]], [[2254334, 5], [2254321, 1]]], - 'fkey_path': [['2254406', False], ['2254399', False]], + 'fkey_path': [[2254406, False], [2254399, False]], 'depth': 2, 'multiple_results': False }, @@ -371,7 +371,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): [[2254334, 5], [2254321, 1]], [[2254321, 11], [2254358, 1]] ], - 'fkey_path': [['2254406', False], ['2254399', False], ['2254394', False]], + 'fkey_path': [[2254406, False], [2254399, False], [2254394, False]], 'depth': 3, 'multiple_results': False }, @@ -383,7 +383,7 @@ def mock_list_joinable_tables(_table_oid, conn, max_depth): [[2254334, 5], [2254321, 1]], [[2254321, 10], [2254313, 1]] ], - 'fkey_path': [['2254406', False], ['2254399', False], ['2254389', False]], + 'fkey_path': [[2254406, False], [2254399, False], [2254389, False]], 'depth': 3, 'multiple_results': False } From 3240918db1758b1a13b8d87e1d4684ef074596c8 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 10 Jul 2024 18:19:12 +0530 Subject: [PATCH 0393/1141] add drop statement --- db/sql/10_msar_joinable_tables.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/sql/10_msar_joinable_tables.sql b/db/sql/10_msar_joinable_tables.sql index cf51e772a9..eab50b25ec 100644 --- a/db/sql/10_msar_joinable_tables.sql +++ b/db/sql/10_msar_joinable_tables.sql @@ -29,6 +29,7 @@ whether to travel from referrer to referant (when False) or from referant to ref */ +DROP TYPE IF EXISTS msar.joinable_tables CASCADE; CREATE TYPE msar.joinable_tables AS ( base integer, -- The OID of the table from which the paths start target integer, -- The OID of the table where the paths end From a7630b9d64d5c57189fcc5f06822202088a32780 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Jul 2024 14:00:26 -0400 Subject: [PATCH 0394/1141] Fix setting of table metadata via RPC - Rename from patch to set - Make it work when no metadata record exists yet - Rename the metadata_dict parameter to metadata --- docs/docs/api/rpc.md | 6 ++-- mathesar/models/base.py | 8 +++++ mathesar/rpc/tables/metadata.py | 32 +++++++++----------- mathesar/tests/rpc/tables/test_t_metadata.py | 17 +++++------ mathesar/tests/rpc/test_endpoints.py | 4 +-- mathesar/utils/tables.py | 16 ++++------ 6 files changed, 41 insertions(+), 42 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 37efba2c64..2ec24382f0 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -80,9 +80,9 @@ To use an RPC function: options: members: - list_ - - patch - - TableMetaData - - SettableTableMetaData + - set_ + - TableMetaDataBlob + - TableMetaDataRecord ## Columns diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 1ea3b0be70..ae6186b486 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -138,3 +138,11 @@ class Meta: name="unique_table_metadata" ) ] + + +table_metadata_fields = { + 'import_verified', + 'column_order', + 'record_summary_customized', + 'record_summary_template', +} diff --git a/mathesar/rpc/tables/metadata.py b/mathesar/rpc/tables/metadata.py index 2bc28382c8..f477bc44ee 100644 --- a/mathesar/rpc/tables/metadata.py +++ b/mathesar/rpc/tables/metadata.py @@ -7,10 +7,10 @@ from modernrpc.auth.basic import http_basic_auth_login_required from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -from mathesar.utils.tables import get_tables_meta_data, patch_table_meta_data +from mathesar.utils.tables import get_tables_meta_data, set_table_meta_data -class TableMetaData(TypedDict): +class TableMetaDataRecord(TypedDict): """ Metadata for a table in a database. @@ -46,9 +46,9 @@ def from_model(cls, model): ) -class SettableTableMetaData(TypedDict): +class TableMetaDataBlob(TypedDict): """ - Settable metadata fields for a table in a database. + The metadata fields which can be set on a table Attributes: import_verified: Specifies whether a file has been successfully imported into a table. @@ -65,7 +65,7 @@ class SettableTableMetaData(TypedDict): @rpc_method(name="tables.metadata.list") @http_basic_auth_login_required @handle_rpc_exceptions -def list_(*, database_id: int, **kwargs) -> list[TableMetaData]: +def list_(*, database_id: int, **kwargs) -> list[TableMetaDataRecord]: """ List metadata associated with tables for a database. @@ -77,26 +77,22 @@ def list_(*, database_id: int, **kwargs) -> list[TableMetaData]: """ table_meta_data = get_tables_meta_data(database_id) return [ - TableMetaData.from_model(model) for model in table_meta_data + TableMetaDataRecord.from_model(model) for model in table_meta_data ] -@rpc_method(name="tables.metadata.patch") +@rpc_method(name="tables.metadata.set") @http_basic_auth_login_required @handle_rpc_exceptions -def patch( - *, table_oid: int, metadata_dict: SettableTableMetaData, database_id: int, **kwargs -) -> TableMetaData: +def set_( + *, table_oid: int, metadata: TableMetaDataBlob, database_id: int, **kwargs +) -> TableMetaDataRecord: """ - Alter metadata settings associated with a table for a database. + Set metadata for a table. Args: - table_oid: Identity of the table whose metadata we'll modify. - metadata_dict: The dict describing desired table metadata alterations. + table_oid: The PostgreSQL OID of the table. + metadata: A TableMetaDataBlob object describing desired table metadata to set. database_id: The Django id of the database containing the table. - - Returns: - Altered metadata object. """ - table_meta_data = patch_table_meta_data(table_oid, metadata_dict, database_id) - return TableMetaData.from_model(table_meta_data) + set_table_meta_data(table_oid, metadata, database_id) diff --git a/mathesar/tests/rpc/tables/test_t_metadata.py b/mathesar/tests/rpc/tables/test_t_metadata.py index faad49b703..b7ec9629e2 100644 --- a/mathesar/tests/rpc/tables/test_t_metadata.py +++ b/mathesar/tests/rpc/tables/test_t_metadata.py @@ -29,12 +29,12 @@ def mock_get_tables_meta_data(_database_id): monkeypatch.setattr(metadata, "get_tables_meta_data", mock_get_tables_meta_data) expect_metadata_list = [ - metadata.TableMetaData( + metadata.TableMetaDataRecord( id=1, database_id=database_id, table_oid=1234, import_verified=True, column_order=[8, 9, 10], record_summary_customized=False, record_summary_template="{5555}" ), - metadata.TableMetaData( + metadata.TableMetaDataRecord( id=2, database_id=database_id, table_oid=4567, import_verified=False, column_order=[], record_summary_customized=True, record_summary_template="{5512} {1223}" @@ -44,11 +44,11 @@ def mock_get_tables_meta_data(_database_id): assert actual_metadata_list == expect_metadata_list -def test_tables_meta_data_patch(monkeypatch): +def test_tables_meta_data_set(monkeypatch): database_id = 2 - metadata_dict = {'import_verified': True, 'column_order': [1, 4, 12]} + metadata_values = {'import_verified': True, 'column_order': [1, 4, 12]} - def mock_patch_tables_meta_data(table_oid, metadata_dict, _database_id): + def mock_set_tables_meta_data(table_oid, metadata, _database_id): server_model = Server(id=2, host='example.com', port=5432) db_model = Database(id=_database_id, name='mymathesardb', server=server_model) return TableMetaData( @@ -56,12 +56,11 @@ def mock_patch_tables_meta_data(table_oid, metadata_dict, _database_id): import_verified=True, column_order=[1, 4, 12], record_summary_customized=False, record_summary_template="{5555}" ) - monkeypatch.setattr(metadata, "patch_table_meta_data", mock_patch_tables_meta_data) + monkeypatch.setattr(metadata, "set_table_meta_data", mock_set_tables_meta_data) - expect_metadata_object = metadata.TableMetaData( + expect_metadata_object = metadata.TableMetaDataRecord( id=1, database_id=database_id, table_oid=1234, import_verified=True, column_order=[1, 4, 12], record_summary_customized=False, record_summary_template="{5555}" ) - actual_metadata_object = metadata.patch(table_oid=1234, metadata_dict=metadata_dict, database_id=2) - assert actual_metadata_object == expect_metadata_object + metadata.set_(table_oid=1234, metadata=metadata_values, database_id=2) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index ca91d17b2e..8527b068ff 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -142,8 +142,8 @@ [user_is_authenticated] ), ( - tables.metadata.patch, - "tables.metadata.patch", + tables.metadata.set_, + "tables.metadata.set", [user_is_authenticated] ) ] diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index 87d350a20e..ab3c56d1e0 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -5,7 +5,7 @@ from mathesar.database.base import create_mathesar_engine from mathesar.imports.base import create_table_from_data_file from mathesar.models.deprecated import Table -from mathesar.models.base import TableMetaData +from mathesar.models.base import Database, TableMetaData, table_metadata_fields from mathesar.state.django import reflect_columns_from_tables from mathesar.state import get_cached_metadata @@ -90,13 +90,9 @@ def get_tables_meta_data(database_id): return TableMetaData.objects.filter(database__id=database_id) -def patch_table_meta_data(table_oid, metadata_dict, database_id): - metadata_model = TableMetaData.objects.get(database__id=database_id, table_oid=table_oid) - alterable_fields = ( - 'import_verified', 'column_order', 'record_summary_customized', 'record_summary_template' +def set_table_meta_data(table_oid, metadata, database_id): + TableMetaData.objects.update_or_create( + database=Database.objects.get(id=database_id), + table_oid=table_oid, + defaults={f: v for f, v in metadata.items() if f in table_metadata_fields} ) - for field, value in metadata_dict.items(): - if field in alterable_fields: - setattr(metadata_model, field, value) - metadata_model.save() - return metadata_model From 1625a6b8fa5e47905f0f631dda7b50f07d0c45d4 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Jul 2024 14:17:37 -0400 Subject: [PATCH 0395/1141] Remove default values for table metadata --- ...ter_tablemetadata_column_order_and_more.py | 33 +++++++++++++++++++ mathesar/models/base.py | 8 ++--- 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 mathesar/migrations/0010_alter_tablemetadata_column_order_and_more.py diff --git a/mathesar/migrations/0010_alter_tablemetadata_column_order_and_more.py b/mathesar/migrations/0010_alter_tablemetadata_column_order_and_more.py new file mode 100644 index 0000000000..8c11c8fdf3 --- /dev/null +++ b/mathesar/migrations/0010_alter_tablemetadata_column_order_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.11 on 2024-07-10 18:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('mathesar', '0009_add_column_metadata_model'), + ] + + operations = [ + migrations.AlterField( + model_name='tablemetadata', + name='column_order', + field=models.JSONField(null=True), + ), + migrations.AlterField( + model_name='tablemetadata', + name='import_verified', + field=models.BooleanField(null=True), + ), + migrations.AlterField( + model_name='tablemetadata', + name='record_summary_customized', + field=models.BooleanField(null=True), + ), + migrations.AlterField( + model_name='tablemetadata', + name='record_summary_template', + field=models.CharField(max_length=255, null=True), + ), + ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index ae6186b486..a912db0eed 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -126,10 +126,10 @@ class Meta: class TableMetaData(BaseModel): database = models.ForeignKey('Database', on_delete=models.CASCADE) table_oid = models.PositiveBigIntegerField() - import_verified = models.BooleanField(default=False) - column_order = models.JSONField(default=list) - record_summary_customized = models.BooleanField(default=False) - record_summary_template = models.CharField(max_length=255, blank=True) + import_verified = models.BooleanField(null=True) + column_order = models.JSONField(null=True) + record_summary_customized = models.BooleanField(null=True) + record_summary_template = models.CharField(max_length=255, null=True) class Meta: constraints = [ From 168e8b63638e27896bedbed19eca9a17c3e5bdd9 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Jul 2024 15:06:43 -0400 Subject: [PATCH 0396/1141] Add tables.list_with_metadata RPC method --- mathesar/rpc/tables/base.py | 28 ++++++++++++++++++++++++++++ mathesar/rpc/tables/metadata.py | 9 +++++++++ mathesar/tests/rpc/test_endpoints.py | 5 +++++ 3 files changed, 42 insertions(+) diff --git a/mathesar/rpc/tables/base.py b/mathesar/rpc/tables/base.py index 408e6dbff9..30f3710a0a 100644 --- a/mathesar/rpc/tables/base.py +++ b/mathesar/rpc/tables/base.py @@ -14,7 +14,9 @@ from mathesar.rpc.columns import CreatableColumnInfo, SettableColumnInfo, PreviewableColumnInfo from mathesar.rpc.constraints import CreatableConstraintInfo from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions +from mathesar.rpc.tables.metadata import TableMetaDataBlob from mathesar.rpc.utils import connect +from mathesar.utils.tables import get_tables_meta_data class TableInfo(TypedDict): @@ -231,3 +233,29 @@ def get_import_preview( user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: return get_preview(table_oid, columns, conn, limit) + + +@rpc_method(name="tables.list_with_metadata") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_with_metadata(*, schema_oid: int, database_id: int, **kwargs) -> list: + """ + List tables in a schema, along with the metadata associated with each table + + Args: + schema_oid: PostgreSQL OID of the schema containing the tables. + database_id: The Django id of the database containing the table. + + Returns: + A list of table details. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + tables = get_table_info(schema_oid, conn) + + metadata_records = get_tables_meta_data(database_id) + metadata_map = { + r.table_oid: TableMetaDataBlob.from_model(r) for r in metadata_records + } + + return [table | {"metadata": metadata_map.get(table["oid"])} for table in tables] diff --git a/mathesar/rpc/tables/metadata.py b/mathesar/rpc/tables/metadata.py index f477bc44ee..7ff9926995 100644 --- a/mathesar/rpc/tables/metadata.py +++ b/mathesar/rpc/tables/metadata.py @@ -61,6 +61,15 @@ class TableMetaDataBlob(TypedDict): record_summary_customized: Optional[bool] record_summary_template: Optional[str] + @classmethod + def from_model(cls, model): + return cls( + import_verified=model.import_verified, + column_order=model.column_order, + record_summary_customized=model.record_summary_customized, + record_summary_template=model.record_summary_template, + ) + @rpc_method(name="tables.metadata.list") @http_basic_auth_login_required diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 8527b068ff..1ff2f365d4 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -106,6 +106,11 @@ "tables.list", [user_is_authenticated] ), + ( + tables.list_with_metadata, + "tables.list_with_metadata", + [user_is_authenticated] + ), ( tables.get, "tables.get", From 243823f8ce7be40199b4d137ad6b4268da89ec10 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Wed, 10 Jul 2024 15:08:25 -0400 Subject: [PATCH 0397/1141] Remove test --- mathesar/tests/rpc/tables/test_t_metadata.py | 22 -------------------- 1 file changed, 22 deletions(-) diff --git a/mathesar/tests/rpc/tables/test_t_metadata.py b/mathesar/tests/rpc/tables/test_t_metadata.py index b7ec9629e2..c25753ffbc 100644 --- a/mathesar/tests/rpc/tables/test_t_metadata.py +++ b/mathesar/tests/rpc/tables/test_t_metadata.py @@ -42,25 +42,3 @@ def mock_get_tables_meta_data(_database_id): ] actual_metadata_list = metadata.list_(database_id=database_id) assert actual_metadata_list == expect_metadata_list - - -def test_tables_meta_data_set(monkeypatch): - database_id = 2 - metadata_values = {'import_verified': True, 'column_order': [1, 4, 12]} - - def mock_set_tables_meta_data(table_oid, metadata, _database_id): - server_model = Server(id=2, host='example.com', port=5432) - db_model = Database(id=_database_id, name='mymathesardb', server=server_model) - return TableMetaData( - id=1, database=db_model, table_oid=1234, - import_verified=True, column_order=[1, 4, 12], record_summary_customized=False, - record_summary_template="{5555}" - ) - monkeypatch.setattr(metadata, "set_table_meta_data", mock_set_tables_meta_data) - - expect_metadata_object = metadata.TableMetaDataRecord( - id=1, database_id=database_id, table_oid=1234, - import_verified=True, column_order=[1, 4, 12], record_summary_customized=False, - record_summary_template="{5555}" - ) - metadata.set_(table_oid=1234, metadata=metadata_values, database_id=2) From b8d6de5672712ad8a1e1bdeaed801c902a631fd5 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 11 Jul 2024 07:32:06 +0800 Subject: [PATCH 0398/1141] add record getter function in DB --- db/sql/00_msar.sql | 79 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 366b4d2aaa..2038713a56 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3207,8 +3207,7 @@ BEGIN $t$, -- %1$s This is a comma separated string of the extracted column names string_agg(quote_ident(col_def ->> 'name'), ', '), - -- %2$s This is the name of the original (remainder) table - __msar.get_qualified_relation_name(tab_id), + -- %2$s This is the name of the original (remainder) table __msar.get_qualified_relation_name(tab_id), -- %3$s This is the new extracted table name __msar.get_qualified_relation_name(extracted_table_id), -- %4$I This is the name of the fkey column in the remainder table. @@ -3228,3 +3227,79 @@ BEGIN RETURN jsonb_build_array(extracted_table_id, fkey_attnum); END; $f$ LANGUAGE plpgsql; + + +---------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- +-- DQL FUNCTIONS +-- +-- This set of functions is for getting records from python. +---------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- + + +CREATE OR REPLACE FUNCTION +msar.sanitize_direction(direction text) RETURNS text AS $$/* +*/ +SELECT CASE lower(direction) + WHEN 'asc' THEN 'ASC' + WHEN 'desc' THEN 'DESC' +END; +$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.build_order_by_expr(tab_id oid, order_ jsonb) RETURNS text AS $$/* +*/ +SELECT 'ORDER BY ' || string_agg(format('%I %s', field, msar.sanitize_direction(direction)), ', ') +FROM jsonb_to_recordset(order_) AS x(field integer, direction text); +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION +msar.build_selectable_column_expr(tab_id oid) RETURNS text AS $$/* +*/ +SELECT string_agg(format('%I AS %I', attname, attnum), ', ') +FROM pg_catalog.pg_attribute +WHERE + attrelid = tab_id AND attnum > 0 AND has_column_privilege(attrelid, attnum, 'SELECT'); +$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.get_records_from_table( + tab_id oid, + limit_ integer, + offset_ integer, + order_ jsonb, + filter_ jsonb, + group_ jsonb, + search_ jsonb +) RETURNS jsonb AS $$/* +*/ +DECLARE + records jsonb; +BEGIN + EXECUTE format( + $q$ + WITH count_cte AS ( + SELECT count(1) AS count FROM %2$I.%3$I + ), results_cte AS ( + SELECT %1$s FROM %2$I.%3$I %6$s LIMIT %4$L OFFSET %5$L + ) + SELECT jsonb_build_object( + 'results', jsonb_agg(row_to_json(results_cte.*)), + 'count', max(count_cte.count) + ) + FROM results_cte, count_cte + $q$, + msar.build_selectable_column_expr(tab_id), + msar.get_relation_schema_name(tab_id), + msar.get_relation_name(tab_id), + limit_, + offset_, + msar.build_order_by_expr(tab_id, order_) + ) INTO records; + RETURN records; +END; +$$ LANGUAGE plpgsql; From 90f34f48199de9fa7d9295f8f844059e5b83c54f Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 11 Jul 2024 13:29:59 +0800 Subject: [PATCH 0399/1141] remove field validation from utility function --- mathesar/models/base.py | 8 -------- mathesar/utils/tables.py | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/mathesar/models/base.py b/mathesar/models/base.py index a912db0eed..3781f5d98c 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -138,11 +138,3 @@ class Meta: name="unique_table_metadata" ) ] - - -table_metadata_fields = { - 'import_verified', - 'column_order', - 'record_summary_customized', - 'record_summary_template', -} diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index ab3c56d1e0..8c0664fa63 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -5,7 +5,7 @@ from mathesar.database.base import create_mathesar_engine from mathesar.imports.base import create_table_from_data_file from mathesar.models.deprecated import Table -from mathesar.models.base import Database, TableMetaData, table_metadata_fields +from mathesar.models.base import Database, TableMetaData from mathesar.state.django import reflect_columns_from_tables from mathesar.state import get_cached_metadata @@ -94,5 +94,5 @@ def set_table_meta_data(table_oid, metadata, database_id): TableMetaData.objects.update_or_create( database=Database.objects.get(id=database_id), table_oid=table_oid, - defaults={f: v for f, v in metadata.items() if f in table_metadata_fields} + defaults=metadata, ) From e2cd90f45ad26524ab4a390df2898c00eeca0cf3 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 11 Jul 2024 16:55:24 +0800 Subject: [PATCH 0400/1141] add functions to make order deterministic --- db/sql/00_msar.sql | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 2038713a56..449deeffa0 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3248,12 +3248,39 @@ END; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION msar.get_pkey_order(tab_id oid) RETURNS jsonb AS $$ +SELECT jsonb_agg(jsonb_build_object('field', f, 'direction', 'asc')) +FROM pg_constraint, LATERAL unnest(conkey) f WHERE contype='p' AND conrelid=tab_id; +$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION msar.get_total_order(tab_id oid) RETURNS jsonb AS $$ +WITH orderable_cte AS ( + SELECT DISTINCT attnum + FROM pg_catalog.pg_attribute + INNER JOIN pg_catalog.pg_cast ON atttypid=castsource + INNER JOIN pg_catalog.pg_operator ON casttarget=oprleft + WHERE attrelid=tab_id AND attnum>0 AND castcontext='i' AND oprname='<' + ORDER BY attnum +) +SELECT COALESCE(jsonb_agg(jsonb_build_object('field', attnum, 'direction', 'asc')), '[]'::jsonb) +FROM orderable_cte; +$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.build_order_by_expr(tab_id oid, order_ jsonb) RETURNS text AS $$/* */ SELECT 'ORDER BY ' || string_agg(format('%I %s', field, msar.sanitize_direction(direction)), ', ') -FROM jsonb_to_recordset(order_) AS x(field integer, direction text); -$$ LANGUAGE plpgsql; +FROM jsonb_to_recordset( + COALESCE( + COALESCE(order_, '[]'::jsonb) || msar.get_pkey_order(tab_id), + COALESCE(order_, '[]'::jsonb) || msar.get_total_order(tab_id) + ) +) + AS x(field smallint, direction text) +WHERE has_column_privilege(tab_id, field, 'SELECT'); +$$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION From 9cf8bbe87d8cd2fb87b19cfed8bc3f2123441317 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 11 Jul 2024 18:13:58 +0530 Subject: [PATCH 0401/1141] add types.list rpc endpoint --- config/settings/common_settings.py | 3 ++- mathesar/rpc/types.py | 35 ++++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 6 +++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 mathesar/rpc/types.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index 51a2786480..77efbb70e7 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -72,7 +72,8 @@ def pipe_delim(pipe_string): 'mathesar.rpc.roles', 'mathesar.rpc.schemas', 'mathesar.rpc.tables', - 'mathesar.rpc.tables.metadata' + 'mathesar.rpc.tables.metadata', + 'mathesar.rpc.types' ] TEMPLATES = [ diff --git a/mathesar/rpc/types.py b/mathesar/rpc/types.py new file mode 100644 index 0000000000..055b885cf3 --- /dev/null +++ b/mathesar/rpc/types.py @@ -0,0 +1,35 @@ +""" +Classes and functions exposed to the RPC endpoint for listing types in a database. +""" + +from typing import Optional, TypedDict + +from modernrpc.core import rpc_method +from modernrpc.auth.basic import http_basic_auth_login_required + +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions +from mathesar.database.types import UIType +from mathesar.api.display_options import DISPLAY_OPTIONS_BY_UI_TYPE + + +class TypeInfo(TypedDict): + identifier: str + name: str + db_types: list + display_options: Optional[dict] + + @classmethod + def from_dict(cls, type): + return cls( + identifier=type.id, + name=type.display_name, + db_types=[db_type.id for db_type in type.db_types], + display_options=DISPLAY_OPTIONS_BY_UI_TYPE.get(type, None) + ) + + +@rpc_method(name="types.list") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_() -> TypeInfo: + return [TypeInfo.from_dict(type) for type in UIType] diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index ca91d17b2e..b87df24f6e 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -14,6 +14,7 @@ from mathesar.rpc import roles from mathesar.rpc import schemas from mathesar.rpc import tables +from mathesar.rpc import types METHODS = [ ( @@ -101,6 +102,11 @@ "schemas.patch", [user_is_authenticated] ), + ( + types.list_, + "types.list", + [user_is_authenticated] + ), ( tables.list_, "tables.list", From ae64517f308767faed6f991a8ff16dc6e2e2e300 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 11 Jul 2024 18:34:45 +0530 Subject: [PATCH 0402/1141] add docstrings --- docs/docs/api/rpc.md | 8 ++++++++ mathesar/rpc/types.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 37efba2c64..eab7f37785 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -111,6 +111,14 @@ To use an RPC function: - ColumnMetaData - SettableColumnMetaData +## Types + +::: types + options: + members: + - list_ + - TypeInfo + ## Constraints ::: constraints diff --git a/mathesar/rpc/types.py b/mathesar/rpc/types.py index 055b885cf3..970d357408 100644 --- a/mathesar/rpc/types.py +++ b/mathesar/rpc/types.py @@ -13,6 +13,15 @@ class TypeInfo(TypedDict): + """ + Information about a type. + + Attributes: + identifier: Specifies the type class that db_type(s) belongs to. + name: Specifies the UI name for a type class. + db_types: Specifies the name(s) of types present on the database. + display_options: Specifies metadata related to a type class. + """ identifier: str name: str db_types: list @@ -32,4 +41,7 @@ def from_dict(cls, type): @http_basic_auth_login_required @handle_rpc_exceptions def list_() -> TypeInfo: + """ + List information about types available on the database. Exposed as `list`. + """ return [TypeInfo.from_dict(type) for type in UIType] From fb5b48707bcdf6eb1e44da1d4b37c5114d3343ba Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 11 Jul 2024 18:36:42 +0530 Subject: [PATCH 0403/1141] fix the return type --- mathesar/rpc/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/rpc/types.py b/mathesar/rpc/types.py index 970d357408..016b59b7a7 100644 --- a/mathesar/rpc/types.py +++ b/mathesar/rpc/types.py @@ -40,7 +40,7 @@ def from_dict(cls, type): @rpc_method(name="types.list") @http_basic_auth_login_required @handle_rpc_exceptions -def list_() -> TypeInfo: +def list_() -> list[TypeInfo]: """ List information about types available on the database. Exposed as `list`. """ From 3396114bd98f0898b93943ea5c8c75b3fbc6ee67 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Thu, 11 Jul 2024 09:39:09 -0400 Subject: [PATCH 0404/1141] Fix return type annotation --- mathesar/rpc/tables/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/rpc/tables/metadata.py b/mathesar/rpc/tables/metadata.py index 7ff9926995..7a6481b81b 100644 --- a/mathesar/rpc/tables/metadata.py +++ b/mathesar/rpc/tables/metadata.py @@ -95,7 +95,7 @@ def list_(*, database_id: int, **kwargs) -> list[TableMetaDataRecord]: @handle_rpc_exceptions def set_( *, table_oid: int, metadata: TableMetaDataBlob, database_id: int, **kwargs -) -> TableMetaDataRecord: +) -> None: """ Set metadata for a table. From 022819f453100fd995375a0340451efcc77309e1 Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 11 Jul 2024 19:32:14 +0530 Subject: [PATCH 0405/1141] Add servers.list endpoint --- config/settings/common_settings.py | 1 + docs/docs/api/rpc.md | 8 +++++ mathesar/rpc/servers.py | 44 ++++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 6 ++++ 4 files changed, 59 insertions(+) create mode 100644 mathesar/rpc/servers.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index f116f98f15..3309363645 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -73,6 +73,7 @@ def pipe_delim(pipe_string): 'mathesar.rpc.databases', 'mathesar.rpc.roles', 'mathesar.rpc.schemas', + 'mathesar.rpc.servers', 'mathesar.rpc.tables', 'mathesar.rpc.tables.metadata' ] diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index d5733d843d..36f9326cd7 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -151,6 +151,14 @@ To use an RPC function: - RoleInfo - RoleMember +## Servers + +::: servers + options: + members: + - list_ + - ServerInfo + ## Responses ### Success diff --git a/mathesar/rpc/servers.py b/mathesar/rpc/servers.py new file mode 100644 index 0000000000..71602e9a39 --- /dev/null +++ b/mathesar/rpc/servers.py @@ -0,0 +1,44 @@ +from typing import TypedDict + +from modernrpc.core import rpc_method +from modernrpc.auth.basic import http_basic_auth_login_required + +from mathesar.models.base import Server +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions + + +class ServerInfo(TypedDict): + """ + Information about a database server. + + Attributes: + id: the Django ID of the server model instance. + host: The host of the database server. + port: the port of the database server. + """ + id: int + host: str + port: int + + @classmethod + def from_model(cls, model): + return cls( + id=model.id, + host=model.host, + port=model.port + ) + + +@rpc_method(name="servers.list") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_() -> list[ServerInfo]: + """ + List information about servers. Exposed as `list`. + + Returns: + A list of server details. + """ + server_qs = Server.objects.all() + + return [ServerInfo.from_model(db_model) for db_model in server_qs] diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 87db102c57..9f619c11ef 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -15,6 +15,7 @@ from mathesar.rpc import databases from mathesar.rpc import roles from mathesar.rpc import schemas +from mathesar.rpc import servers from mathesar.rpc import tables METHODS = [ @@ -118,6 +119,11 @@ "schemas.patch", [user_is_authenticated] ), + ( + servers.list_, + "servers.list", + [user_is_authenticated] + ), ( tables.list_, "tables.list", From b72436d8ecb5f7e43dec6efbcb17da30d88513ed Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 11 Jul 2024 20:02:11 +0530 Subject: [PATCH 0406/1141] fix install script --- mathesar/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/install.py b/mathesar/install.py index e026bd9591..74c2c8c482 100644 --- a/mathesar/install.py +++ b/mathesar/install.py @@ -42,7 +42,6 @@ def main(skip_static_collection=False): def install_on_db_with_key(database_key, skip_confirm): from mathesar.models.deprecated import Connection db_model = Connection.create_from_settings_key(database_key) - db_model.save() try: install.install_mathesar( database_name=db_model.db_name, @@ -55,6 +54,7 @@ def install_on_db_with_key(database_key, skip_confirm): except OperationalError as e: db_model.delete() raise e + db_model.save() if __name__ == "__main__": From 4196340f32920a9b0fa8622de50bb8aa597a0587 Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 11 Jul 2024 20:36:09 +0530 Subject: [PATCH 0407/1141] Add endpoint for listing collaborators --- config/settings/common_settings.py | 1 + docs/docs/api/rpc.md | 8 ++++ mathesar/rpc/collaborators/__init__.py | 1 + mathesar/rpc/collaborators/base.py | 55 ++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 6 +++ 5 files changed, 71 insertions(+) create mode 100644 mathesar/rpc/collaborators/__init__.py create mode 100644 mathesar/rpc/collaborators/base.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index 3309363645..be2ed7edb1 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -65,6 +65,7 @@ def pipe_delim(pipe_string): ROOT_URLCONF = "config.urls" MODERNRPC_METHODS_MODULES = [ + 'mathesar.rpc.collaborators', 'mathesar.rpc.connections', 'mathesar.rpc.constraints', 'mathesar.rpc.columns', diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 36f9326cd7..08be0e398d 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -37,6 +37,14 @@ To use an RPC function: } ``` +## Collaborators + +::: collaborators + options: + members: + - list_ + - CollaboratorInfo + ## Connections ::: connections diff --git a/mathesar/rpc/collaborators/__init__.py b/mathesar/rpc/collaborators/__init__.py new file mode 100644 index 0000000000..4b40b38c84 --- /dev/null +++ b/mathesar/rpc/collaborators/__init__.py @@ -0,0 +1 @@ +from .base import * # noqa diff --git a/mathesar/rpc/collaborators/base.py b/mathesar/rpc/collaborators/base.py new file mode 100644 index 0000000000..b7393b54d5 --- /dev/null +++ b/mathesar/rpc/collaborators/base.py @@ -0,0 +1,55 @@ +from typing import TypedDict + +from modernrpc.core import rpc_method +from modernrpc.auth.basic import http_basic_auth_login_required + +from mathesar.models.base import UserDatabaseRoleMap +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions + + +class CollaboratorInfo(TypedDict): + """ + Information about a collaborator. + + Attributes: + id: the Django ID of the UserDatabaseRoleMap model instance. + user_id: The Django ID of the User model instance of the collaborator. + database_id: the Django ID of the Database model instance for the collaborator. + role_id: The Django ID of the Role model instance for the collaborator. + """ + id: int + user_id: int + database_id: int + role_id: int + + @classmethod + def from_model(cls, model): + return cls( + id=model.id, + user_id=model.user.id, + database_id=model.database.id, + role_id=model.role.id + ) + + +@rpc_method(name="collaborators.list") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_(*, database_id: int = None, **kwargs) -> list[CollaboratorInfo]: + """ + List information about collaborators. Exposed as `list`. + + If called with no `database_id`, all collaborators for all databases are listed. + + Args: + database_id: The Django id of the database associated with the collaborators. + + Returns: + A list of collaborators. + """ + if database_id is not None: + user_database_role_map_qs = UserDatabaseRoleMap.objects.filter(database__id=database_id) + else: + user_database_role_map_qs = UserDatabaseRoleMap.objects.all() + + return [CollaboratorInfo.from_model(db_model) for db_model in user_database_role_map_qs] diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 9f619c11ef..8f0192ec7d 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -8,6 +8,7 @@ import pytest from modernrpc.auth import user_is_authenticated, user_is_superuser +from mathesar.rpc import collaborators from mathesar.rpc import columns from mathesar.rpc import connections from mathesar.rpc import constraints @@ -19,6 +20,11 @@ from mathesar.rpc import tables METHODS = [ + ( + collaborators.list_, + "collaborators.list", + [user_is_authenticated] + ), ( columns.delete, "columns.delete", From c1784dd80fd7272415c41f6052e1e6d8e3edf865 Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 11 Jul 2024 20:51:07 +0530 Subject: [PATCH 0408/1141] Make rpc/collaborators an individual file instead of a module --- mathesar/rpc/{collaborators/base.py => collaborators.py} | 0 mathesar/rpc/collaborators/__init__.py | 1 - 2 files changed, 1 deletion(-) rename mathesar/rpc/{collaborators/base.py => collaborators.py} (100%) delete mode 100644 mathesar/rpc/collaborators/__init__.py diff --git a/mathesar/rpc/collaborators/base.py b/mathesar/rpc/collaborators.py similarity index 100% rename from mathesar/rpc/collaborators/base.py rename to mathesar/rpc/collaborators.py diff --git a/mathesar/rpc/collaborators/__init__.py b/mathesar/rpc/collaborators/__init__.py deleted file mode 100644 index 4b40b38c84..0000000000 --- a/mathesar/rpc/collaborators/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base import * # noqa From 4aa6a5cdeae6f930c3051b1a65d4e363e57f4c44 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 12 Jul 2024 00:15:10 +0530 Subject: [PATCH 0409/1141] Implement collaborators add, delete, and set_role endpoints --- mathesar/rpc/collaborators.py | 72 ++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/mathesar/rpc/collaborators.py b/mathesar/rpc/collaborators.py index b7393b54d5..27e683cb6b 100644 --- a/mathesar/rpc/collaborators.py +++ b/mathesar/rpc/collaborators.py @@ -1,9 +1,10 @@ from typing import TypedDict from modernrpc.core import rpc_method -from modernrpc.auth.basic import http_basic_auth_login_required +from modernrpc.auth.basic import http_basic_auth_login_required, http_basic_auth_superuser_required -from mathesar.models.base import UserDatabaseRoleMap +from mathesar.models.base import UserDatabaseRoleMap, Database, Role +from mathesar.models.users import User from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -53,3 +54,70 @@ def list_(*, database_id: int = None, **kwargs) -> list[CollaboratorInfo]: user_database_role_map_qs = UserDatabaseRoleMap.objects.all() return [CollaboratorInfo.from_model(db_model) for db_model in user_database_role_map_qs] + + +@rpc_method(name='collaborators.add') +@http_basic_auth_superuser_required +@handle_rpc_exceptions +def add( + *, + database_id: int, + user_id: int, + role_id: int, + **kwargs +) -> CollaboratorInfo: + """ + Set up a new collaborator for a database. + + Args: + database_id: The Django id of the Database to associate with the collaborator. + user_id: The Django id of the User who'd be the collaborator. + role_id: The Django id of the Role to associate with the collaborator. + """ + database = Database.objects.get(id=database_id) + user = User.objects.get(id=user_id) + role = Role.objects.get(id=role_id) + collaborator = UserDatabaseRoleMap.objects.create( + database=database, + user=user, + role=role, + server=role.server + ) + return CollaboratorInfo.from_model(collaborator) + + +@rpc_method(name='collaborators.delete') +@http_basic_auth_superuser_required +@handle_rpc_exceptions +def delete(*, collaborator_id: int, **kwargs): + """ + Delete a collaborator from a database. + + Args: + collaborator_id: The Django id of the UserDatabaseRoleMap model instance of the collaborator. + """ + collaborator = UserDatabaseRoleMap.objects.get(id=collaborator_id) + collaborator.delete() + + +@rpc_method(name='collaborators.set_role') +@http_basic_auth_superuser_required +@handle_rpc_exceptions +def set_role( + *, + collaborator_id: int, + role_id: int, + **kwargs +) -> CollaboratorInfo: + """ + Set the role of a collaborator for a database. + + Args: + collaborator_id: The Django id of the UserDatabaseRoleMap model instance of the collaborator. + role_id: The Django id of the Role to associate with the collaborator. + """ + collaborator = UserDatabaseRoleMap.objects.get(id=collaborator_id) + role = Role.objects.get(id=role_id) + collaborator.role = role + collaborator.save() + return CollaboratorInfo.from_model(collaborator) From 56a1fa3636c69a181519130e7301adf77a3a31c9 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 12 Jul 2024 00:23:50 +0530 Subject: [PATCH 0410/1141] Add endpoint tests and docs for collaborators --- docs/docs/api/rpc.md | 3 +++ mathesar/tests/rpc/test_endpoints.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 08be0e398d..926d080540 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -43,6 +43,9 @@ To use an RPC function: options: members: - list_ + - add + - delete + - set_role - CollaboratorInfo ## Connections diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 8f0192ec7d..3d2e7cfe4a 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -20,11 +20,26 @@ from mathesar.rpc import tables METHODS = [ + ( + collaborators.add, + "collaborators.add", + [user_is_superuser] + ), + ( + collaborators.delete, + "collaborators.delete", + [user_is_superuser] + ), ( collaborators.list_, "collaborators.list", [user_is_authenticated] ), + ( + collaborators.set_role, + "collaborators.set_role", + [user_is_superuser] + ), ( columns.delete, "columns.delete", From 8eba96332d8227f2103b1342d7039ce0acce92a8 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 12 Jul 2024 01:04:34 +0530 Subject: [PATCH 0411/1141] Rename Role model to ConfiguredRole --- .../0011_rename_role_configuredrole.py | 17 +++++++++++++++++ mathesar/models/base.py | 4 ++-- mathesar/rpc/collaborators.py | 14 +++++++------- mathesar/tests/rpc/test_database_setup.py | 6 +++--- mathesar/utils/permissions.py | 4 ++-- 5 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 mathesar/migrations/0011_rename_role_configuredrole.py diff --git a/mathesar/migrations/0011_rename_role_configuredrole.py b/mathesar/migrations/0011_rename_role_configuredrole.py new file mode 100644 index 0000000000..752d6012be --- /dev/null +++ b/mathesar/migrations/0011_rename_role_configuredrole.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.11 on 2024-07-11 19:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('mathesar', '0010_alter_tablemetadata_column_order_and_more'), + ] + + operations = [ + migrations.RenameModel( + old_name='Role', + new_name='ConfiguredRole', + ), + ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 3781f5d98c..98f1d2829b 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -40,7 +40,7 @@ class Meta: ] -class Role(BaseModel): +class ConfiguredRole(BaseModel): name = models.CharField(max_length=255) server = models.ForeignKey( 'Server', on_delete=models.CASCADE, related_name='roles' @@ -61,7 +61,7 @@ class Meta: class UserDatabaseRoleMap(BaseModel): user = models.ForeignKey('User', on_delete=models.CASCADE) database = models.ForeignKey('Database', on_delete=models.CASCADE) - role = models.ForeignKey('Role', on_delete=models.CASCADE) + role = models.ForeignKey('ConfiguredRole', on_delete=models.CASCADE) server = models.ForeignKey('Server', on_delete=models.CASCADE) class Meta: diff --git a/mathesar/rpc/collaborators.py b/mathesar/rpc/collaborators.py index 27e683cb6b..1851d03051 100644 --- a/mathesar/rpc/collaborators.py +++ b/mathesar/rpc/collaborators.py @@ -3,7 +3,7 @@ from modernrpc.core import rpc_method from modernrpc.auth.basic import http_basic_auth_login_required, http_basic_auth_superuser_required -from mathesar.models.base import UserDatabaseRoleMap, Database, Role +from mathesar.models.base import UserDatabaseRoleMap, Database, ConfiguredRole from mathesar.models.users import User from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -16,7 +16,7 @@ class CollaboratorInfo(TypedDict): id: the Django ID of the UserDatabaseRoleMap model instance. user_id: The Django ID of the User model instance of the collaborator. database_id: the Django ID of the Database model instance for the collaborator. - role_id: The Django ID of the Role model instance for the collaborator. + role_id: The Django ID of the ConfiguredRole model instance for the collaborator. """ id: int user_id: int @@ -71,12 +71,12 @@ def add( Args: database_id: The Django id of the Database to associate with the collaborator. - user_id: The Django id of the User who'd be the collaborator. - role_id: The Django id of the Role to associate with the collaborator. + user_id: The Django id of the User model instance who'd be the collaborator. + role_id: The Django id of the ConfiguredRole model instance to associate with the collaborator. """ database = Database.objects.get(id=database_id) user = User.objects.get(id=user_id) - role = Role.objects.get(id=role_id) + role = ConfiguredRole.objects.get(id=role_id) collaborator = UserDatabaseRoleMap.objects.create( database=database, user=user, @@ -114,10 +114,10 @@ def set_role( Args: collaborator_id: The Django id of the UserDatabaseRoleMap model instance of the collaborator. - role_id: The Django id of the Role to associate with the collaborator. + role_id: The Django id of the ConfiguredRole model instance to associate with the collaborator. """ collaborator = UserDatabaseRoleMap.objects.get(id=collaborator_id) - role = Role.objects.get(id=role_id) + role = ConfiguredRole.objects.get(id=role_id) collaborator.role = role collaborator.save() return CollaboratorInfo.from_model(collaborator) diff --git a/mathesar/tests/rpc/test_database_setup.py b/mathesar/tests/rpc/test_database_setup.py index 1e30679fbd..e77cd92b8d 100644 --- a/mathesar/tests/rpc/test_database_setup.py +++ b/mathesar/tests/rpc/test_database_setup.py @@ -6,7 +6,7 @@ monkeypatch(pytest): Lets you monkeypatch an object for testing. """ from mathesar.models.users import User -from mathesar.models.base import Database, Role, Server, UserDatabaseRoleMap +from mathesar.models.base import Database, ConfiguredRole, Server, UserDatabaseRoleMap from mathesar.rpc import database_setup @@ -25,7 +25,7 @@ def mock_set_up_new_for_user(database, user, sample_data=[]): raise AssertionError("incorrect parameters passed") server_model = Server(id=2, host="example.com", port=5432) db_model = Database(id=3, name=test_database, server=server_model) - role_model = Role(id=4, name="matheuser", server=server_model) + role_model = ConfiguredRole(id=4, name="matheuser", server=server_model) return UserDatabaseRoleMap( user=user, database=db_model, role=role_model, server=server_model ) @@ -70,7 +70,7 @@ def mock_set_up_preexisting_database_for_user( raise AssertionError("incorrect parameters passed") server_model = Server(id=2, host="example.com", port=5432) db_model = Database(id=3, name=test_database, server=server_model) - role_model = Role(id=4, name="matheuser", server=server_model) + role_model = ConfiguredRole(id=4, name="matheuser", server=server_model) return UserDatabaseRoleMap( user=user, database=db_model, role=role_model, server=server_model ) diff --git a/mathesar/utils/permissions.py b/mathesar/utils/permissions.py index 1c3ae45d82..b3beee022e 100644 --- a/mathesar/utils/permissions.py +++ b/mathesar/utils/permissions.py @@ -5,7 +5,7 @@ from db.install import install_mathesar from mathesar.examples.library_dataset import load_library_dataset from mathesar.examples.movies_dataset import load_movies_dataset -from mathesar.models.base import Server, Database, Role, UserDatabaseRoleMap +from mathesar.models.base import Server, Database, ConfiguredRole, UserDatabaseRoleMap from mathesar.models.deprecated import Connection from mathesar.models.users import User from mathesar.utils.connections import BadInstallationTarget @@ -90,7 +90,7 @@ def _setup_connection_models( database, _ = Database.objects.get_or_create( name=database_name, server=server ) - role, _ = Role.objects.get_or_create( + role, _ = ConfiguredRole.objects.get_or_create( name=role_name, server=server, defaults={"password": password}, From badf3d1813d415d58bd7fd77ce53ab4cc148c639 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 12 Jul 2024 01:35:13 +0530 Subject: [PATCH 0412/1141] Add endpoints for configured_roles --- config/settings/common_settings.py | 1 + mathesar/rpc/configured_roles.py | 109 +++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 21 ++++++ 3 files changed, 131 insertions(+) create mode 100644 mathesar/rpc/configured_roles.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index be2ed7edb1..3d47107fb5 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -66,6 +66,7 @@ def pipe_delim(pipe_string): MODERNRPC_METHODS_MODULES = [ 'mathesar.rpc.collaborators', + 'mathesar.rpc.configured_roles', 'mathesar.rpc.connections', 'mathesar.rpc.constraints', 'mathesar.rpc.columns', diff --git a/mathesar/rpc/configured_roles.py b/mathesar/rpc/configured_roles.py new file mode 100644 index 0000000000..3943e39cca --- /dev/null +++ b/mathesar/rpc/configured_roles.py @@ -0,0 +1,109 @@ +from typing import TypedDict + +from modernrpc.core import rpc_method +from modernrpc.auth.basic import http_basic_auth_login_required, http_basic_auth_superuser_required + +from mathesar.models.base import ConfiguredRole, Server +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions + + +class ConfiguredRoleInfo(TypedDict): + """ + Information about a role configured in Mathesar. + + Attributes: + id: the Django ID of the ConfiguredRole model instance. + name: The name of the role. + server: The Django ID of the Server model instance for the role. + """ + id: int + name: str + server_id: int + + @classmethod + def from_model(cls, model): + return cls( + id=model.id, + name=model.name, + server_id=model.server.id + ) + + +@rpc_method(name="configured_roles.list") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_(*, server_id: int, **kwargs) -> list[ConfiguredRoleInfo]: + """ + List information about roles configured in Mathesar. Exposed as `list`. + + Args: + server_id: The Django id of the Server containing the configured roles. + + Returns: + A list of configured roles. + """ + configured_role_qs = ConfiguredRole.objects.filter(server__id=server_id) + + return [ConfiguredRoleInfo.from_model(db_model) for db_model in configured_role_qs] + + +@rpc_method(name='configured_roles.add') +@http_basic_auth_superuser_required +@handle_rpc_exceptions +def add( + *, + server_id: str, + name: str, + password: str, + **kwargs +) -> ConfiguredRoleInfo: + """ + Configure a role in Mathesar for a database server. + + Args: + server_id: The Django id of the Server to contain the configured role. + name: The name of the role. + password: The password for the role. + """ + server = Server.objects.get(id=server_id) + configured_role = ConfiguredRole.objects.create( + server=server, + name=name, + password=password + ) + return ConfiguredRoleInfo.from_model(configured_role) + + +@rpc_method(name='configured_roles.delete') +@http_basic_auth_superuser_required +@handle_rpc_exceptions +def delete(*, configured_role_id: int, **kwargs): + """ + Delete a configured role for a server. + + Args: + configured_role_id: The Django id of the ConfiguredRole model instance. + """ + configured_role = ConfiguredRole.objects.get(id=configured_role_id) + configured_role.delete() + + +@rpc_method(name='configured_roles.set_password') +@http_basic_auth_superuser_required +@handle_rpc_exceptions +def set_password( + *, + configured_role_id: int, + password: str, + **kwargs +): + """ + Set the password of a configured role for a server. + + Args: + configured_role_id: The Django id of the ConfiguredRole model instance. + password: The password for the role. + """ + configured_role = ConfiguredRole.objects.get(id=configured_role_id) + configured_role.password = password + configured_role.save() diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index e7e413e90c..938ca9b2d5 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -10,6 +10,7 @@ from mathesar.rpc import collaborators from mathesar.rpc import columns +from mathesar.rpc import configured_roles from mathesar.rpc import connections from mathesar.rpc import constraints from mathesar.rpc import database_setup @@ -70,6 +71,26 @@ "columns.metadata.patch", [user_is_authenticated] ), + ( + configured_roles.add, + "configured_roles.add", + [user_is_superuser] + ), + ( + configured_roles.list_, + "configured_roles.list", + [user_is_authenticated] + ), + ( + configured_roles.delete, + "configured_roles.delete", + [user_is_superuser] + ), + ( + configured_roles.set_password, + "configured_roles.set_password", + [user_is_superuser] + ), ( connections.add_from_known_connection, "connections.add_from_known_connection", From 7233cd96271eb128c2d48bce1c80907c539f4508 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 12 Jul 2024 01:36:36 +0530 Subject: [PATCH 0413/1141] Add docs for ConfiguredRoles --- docs/docs/api/rpc.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 36369c57f3..9af62f0141 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -48,6 +48,17 @@ To use an RPC function: - set_role - CollaboratorInfo +## ConfiguredRoles + +::: configured_roles + options: + members: + - list_ + - add + - delete + - set_password + - ConfiguredRoleInfo + ## Connections ::: connections From 75f20c43e462e679ea9d3fd2798332c2cbc2ff6e Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 12 Jul 2024 14:01:37 +0800 Subject: [PATCH 0414/1141] improve documentation, change field -> attnum --- db/sql/00_msar.sql | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 449deeffa0..e6a4216c06 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3249,7 +3249,7 @@ $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION msar.get_pkey_order(tab_id oid) RETURNS jsonb AS $$ -SELECT jsonb_agg(jsonb_build_object('field', f, 'direction', 'asc')) +SELECT jsonb_agg(jsonb_build_object('attnum', f, 'direction', 'asc')) FROM pg_constraint, LATERAL unnest(conkey) f WHERE contype='p' AND conrelid=tab_id; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -3263,28 +3263,46 @@ WITH orderable_cte AS ( WHERE attrelid=tab_id AND attnum>0 AND castcontext='i' AND oprname='<' ORDER BY attnum ) -SELECT COALESCE(jsonb_agg(jsonb_build_object('field', attnum, 'direction', 'asc')), '[]'::jsonb) +SELECT COALESCE(jsonb_agg(jsonb_build_object('attnum', attnum, 'direction', 'asc')), '[]'::jsonb) FROM orderable_cte; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION msar.build_order_by_expr(tab_id oid, order_ jsonb) RETURNS text AS $$/* +Build an ORDER BY expression for the given table and order JSON. + +The ORDER BY expression will refer to columns by their attnum. This is designed to work together +with `msar.build_selectable_column_expr`. It will only use the columns to which the user has access. +Finally, this function will append either a primary key, or all columns to the produced ORDER BY so +the resulting ordering is totally defined (i.e., deterministic). + +Args: + tab_id: The OID of the table whose columns we'll order by. */ -SELECT 'ORDER BY ' || string_agg(format('%I %s', field, msar.sanitize_direction(direction)), ', ') +SELECT 'ORDER BY ' || string_agg(format('%I %s', attnum, msar.sanitize_direction(direction)), ', ') FROM jsonb_to_recordset( COALESCE( COALESCE(order_, '[]'::jsonb) || msar.get_pkey_order(tab_id), COALESCE(order_, '[]'::jsonb) || msar.get_total_order(tab_id) ) ) - AS x(field smallint, direction text) -WHERE has_column_privilege(tab_id, field, 'SELECT'); + AS x(attnum smallint, direction text) +WHERE has_column_privilege(tab_id, attnum, 'SELECT'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION msar.build_selectable_column_expr(tab_id oid) RETURNS text AS $$/* +Build an SQL select-target expression of only columns to which the user has access. + +Given columns with attnums 2, 3, and 4, and assuming the user has access only to columns 2 and 4, +this function will return an expression of the form: + +column_name AS "2", another_column_name AS "4" + +Args: + tab_id: The OID of the table containing the columns to select. */ SELECT string_agg(format('%I AS %I', attname, attnum), ', ') FROM pg_catalog.pg_attribute @@ -3303,6 +3321,19 @@ msar.get_records_from_table( group_ jsonb, search_ jsonb ) RETURNS jsonb AS $$/* +Get records from a table. Only columns to which the user has access are returned. + +Args: + tab_id: The OID of the table whose records we'll get + limit_: The maximum number of rows we'll return + offset_: The number of rows to skip before returning records from following rows. + order_: An array of ordering definition objects. + filter_: An array of filter definition objects. + group_: An array of group definition objects. + search_: An array of search definition objects. + +The order definition objects should have the form + {"attnum": , "direction": } */ DECLARE records jsonb; From 67c70a176099e985751827681417e96ccb4bf1ef Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 12 Jul 2024 14:02:58 +0800 Subject: [PATCH 0415/1141] fix linting mistake made by GH editor --- mathesar/rpc/tables/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/rpc/tables/base.py b/mathesar/rpc/tables/base.py index 62b553fa8d..8b076a94e5 100644 --- a/mathesar/rpc/tables/base.py +++ b/mathesar/rpc/tables/base.py @@ -331,4 +331,4 @@ def list_with_metadata(*, schema_oid: int, database_id: int, **kwargs) -> list: r.table_oid: TableMetaDataBlob.from_model(r) for r in metadata_records } - return [table | {"metadata": metadata_map.get(table["oid"])} for table in tables] \ No newline at end of file + return [table | {"metadata": metadata_map.get(table["oid"])} for table in tables] From 0f7d96073c5113326aeb48cb788c637a2c4fb3a2 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 12 Jul 2024 15:49:36 +0800 Subject: [PATCH 0416/1141] fix column extraction bug, rename to 'list' --- db/sql/00_msar.sql | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index ceb4bfff53..0ff571226d 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3207,7 +3207,8 @@ BEGIN $t$, -- %1$s This is a comma separated string of the extracted column names string_agg(quote_ident(col_def ->> 'name'), ', '), - -- %2$s This is the name of the original (remainder) table __msar.get_qualified_relation_name(tab_id), + -- %2$s This is the name of the original (remainder) table + __msar.get_qualified_relation_name(tab_id), -- %3$s This is the new extracted table name __msar.get_qualified_relation_name(extracted_table_id), -- %4$I This is the name of the fkey column in the remainder table. @@ -3249,8 +3250,9 @@ $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION msar.get_pkey_order(tab_id oid) RETURNS jsonb AS $$ -SELECT jsonb_agg(jsonb_build_object('attnum', f, 'direction', 'asc')) -FROM pg_constraint, LATERAL unnest(conkey) f WHERE contype='p' AND conrelid=tab_id; +SELECT jsonb_agg(jsonb_build_object('attnum', attnum, 'direction', 'asc')) +FROM pg_constraint, LATERAL unnest(conkey) attnum +WHERE contype='p' AND conrelid=tab_id AND has_column_privilege(tab_id, attnum, 'SELECT'); $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -3260,7 +3262,13 @@ WITH orderable_cte AS ( FROM pg_catalog.pg_attribute INNER JOIN pg_catalog.pg_cast ON atttypid=castsource INNER JOIN pg_catalog.pg_operator ON casttarget=oprleft - WHERE attrelid=tab_id AND attnum>0 AND castcontext='i' AND oprname='<' + WHERE + attrelid=tab_id + AND attnum>0 + AND castcontext='i' + AND oprname='<' + -- This privilege check is redundant in context, but may be useful for other callers. + AND has_column_privilege(tab_id, attnum, 'SELECT') ORDER BY attnum ) SELECT COALESCE(jsonb_agg(jsonb_build_object('attnum', attnum, 'direction', 'asc')), '[]'::jsonb) @@ -3312,7 +3320,7 @@ $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.get_records_from_table( +msar.list_records_from_table( tab_id oid, limit_ integer, offset_ integer, From 4546f260f0994cf727e728d568eabe8c860ea6a2 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 12 Jul 2024 15:59:08 +0800 Subject: [PATCH 0417/1141] add tests for building ORDER BY --- db/sql/test_00_msar.sql | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index b12fda83be..6542c2bad5 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2794,3 +2794,34 @@ BEGIN RETURN NEXT ok(NOT jsonb_path_exists(msar.get_roles(), '$[*] ? (@.name == "foo")')); END; $$ LANGUAGE plpgsql; + + +-- msar.build_order_by_expr ------------------------------------------------------------------------ + +CREATE OR REPLACE FUNCTION __setup_order_by_table() RETURNS SETOF TEXT AS $$ +BEGIN + CREATE TABLE atable (id integer PRIMARY KEY, col1 integer, col2 varchar, col3 json, col4 jsonb); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_build_order_by_expr() RETURNS SETOF TEXT AS $$ +DECLARE + rel_id oid; + order_by_expr text; +BEGIN + PERFORM __setup_order_by_table(); + rel_id := 'atable'::regclass::oid; + RETURN NEXT is(msar.build_order_by_expr(rel_id, null), 'ORDER BY "1" ASC'); + RETURN NEXT is( + msar.build_order_by_expr(rel_id, '[{"attnum": 1, "direction": "desc"}]'), + 'ORDER BY "1" DESC, "1" ASC' + ); + RETURN NEXT is( + msar.build_order_by_expr( + rel_id, '[{"attnum": 3, "direction": "asc"}, {"attnum": 5, "direction": "DESC"}]' + ), + 'ORDER BY "3" ASC, "5" DESC, "1" ASC' + ); +END; +$$ LANGUAGE plpgsql; From 9a50dafcee60cdee938b054803676cfae3ce82f7 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 12 Jul 2024 17:27:49 +0800 Subject: [PATCH 0418/1141] add tests for permissions fix jsonb ordering bug --- db/sql/00_msar.sql | 18 +++++++++++++----- db/sql/test_00_msar.sql | 12 ++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 0ff571226d..ded714e73c 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3208,7 +3208,7 @@ BEGIN -- %1$s This is a comma separated string of the extracted column names string_agg(quote_ident(col_def ->> 'name'), ', '), -- %2$s This is the name of the original (remainder) table - __msar.get_qualified_relation_name(tab_id), + -- __msar.get_qualified_relation_name(tab_id), -- %3$s This is the new extracted table name __msar.get_qualified_relation_name(extracted_table_id), -- %4$I This is the name of the fkey column in the remainder table. @@ -3258,7 +3258,7 @@ $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION msar.get_total_order(tab_id oid) RETURNS jsonb AS $$ WITH orderable_cte AS ( - SELECT DISTINCT attnum + SELECT attnum FROM pg_catalog.pg_attribute INNER JOIN pg_catalog.pg_cast ON atttypid=castsource INNER JOIN pg_catalog.pg_operator ON casttarget=oprleft @@ -3267,12 +3267,20 @@ WITH orderable_cte AS ( AND attnum>0 AND castcontext='i' AND oprname='<' - -- This privilege check is redundant in context, but may be useful for other callers. - AND has_column_privilege(tab_id, attnum, 'SELECT') + UNION SELECT attnum + FROM pg_catalog.pg_attribute + INNER JOIN pg_catalog.pg_operator ON atttypid=oprleft + WHERE + attrelid=tab_id + AND attnum>0 + AND oprname='<' ORDER BY attnum ) SELECT COALESCE(jsonb_agg(jsonb_build_object('attnum', attnum, 'direction', 'asc')), '[]'::jsonb) -FROM orderable_cte; +-- This privilege check is redundant in context, but may be useful for other callers. +FROM orderable_cte +-- This privilege check is redundant in context, but may be useful for other callers. +WHERE has_column_privilege(tab_id, attnum, 'SELECT'); $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 6542c2bad5..2f13ea1f18 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2823,5 +2823,17 @@ BEGIN ), 'ORDER BY "3" ASC, "5" DESC, "1" ASC' ); + CREATE ROLE intern_no_pkey; + GRANT USAGE ON SCHEMA msar, __msar TO intern_no_pkey; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA msar, __msar TO intern_no_pkey; + GRANT SELECT (col1, col2, col3, col4) ON TABLE atable TO intern_no_pkey; + SET ROLE intern_no_pkey; + RETURN NEXT is( + msar.build_order_by_expr(rel_id, null), 'ORDER BY "2" ASC, "3" ASC, "5" ASC' + ); + SET ROLE NONE; + REVOKE ALL ON TABLE atable FROM intern_no_pkey; + SET ROLE intern_no_pkey; + RETURN NEXT is(msar.build_order_by_expr(rel_id, null), null); END; $$ LANGUAGE plpgsql; From 8821bf2050b889bf654206c8cbbb28427664b997 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 12 Jul 2024 17:56:57 +0800 Subject: [PATCH 0419/1141] add tests for record listing --- db/sql/test_00_msar.sql | 100 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 5 deletions(-) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 2f13ea1f18..66a330f9bd 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2796,21 +2796,111 @@ END; $$ LANGUAGE plpgsql; --- msar.build_order_by_expr ------------------------------------------------------------------------ -CREATE OR REPLACE FUNCTION __setup_order_by_table() RETURNS SETOF TEXT AS $$ +-- msar.list_records_from_table -------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION __setup_list_records_table() RETURNS SETOF TEXT AS $$ BEGIN - CREATE TABLE atable (id integer PRIMARY KEY, col1 integer, col2 varchar, col3 json, col4 jsonb); + CREATE TABLE atable ( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + col1 integer, + col2 varchar, + col3 json, + col4 jsonb + ); + INSERT INTO atable (col1, col2, col3, col4) VALUES + (5, 'sdflkj', '"s"', '{"a": "val"}'), + (34, 'sdflfflsk', null, '[1, 2, 3, 4]'), + (2, 'abcde', '{"k": 3242348}', 'true'); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_list_records_from_table() RETURNS SETOF TEXT AS $$ +DECLARE + rel_id oid; +BEGIN + PERFORM __setup_list_records_table(); + rel_id := 'atable'::regclass::oid; + RETURN NEXT is( + msar.list_records_from_table(rel_id, null, null, null, null, null, null), + $j${ + "count": 3, + "results": [ + {"1": 1, "2": 5, "3": "sdflkj", "4": "s", "5": {"a": "val"}}, + {"1": 2, "2": 34, "3": "sdflfflsk", "4": null, "5": [1, 2, 3, 4]}, + {"1": 3, "2": 2, "3": "abcde", "4": {"k": 3242348}, "5": true} + ] + }$j$ + ); + RETURN NEXT is( + msar.list_records_from_table( + rel_id, 2, null, '[{"attnum": 2, "direction": "desc"}]', null, null, null + ), + $j${ + "count": 3, + "results": [ + {"1": 2, "2": 34, "3": "sdflfflsk", "4": null, "5": [1, 2, 3, 4]}, + {"1": 1, "2": 5, "3": "sdflkj", "4": "s", "5": {"a": "val"}} + ] + }$j$ + ); + RETURN NEXT is( + msar.list_records_from_table( + rel_id, null, 1, '[{"attnum": 1, "direction": "desc"}]', null, null, null + ), + $j${ + "count": 3, + "results": [ + {"1": 2, "2": 34, "3": "sdflfflsk", "4": null, "5": [1, 2, 3, 4]}, + {"1": 1, "2": 5, "3": "sdflkj", "4": "s", "5": {"a": "val"}} + ] + }$j$ + ); + CREATE ROLE intern_no_pkey; + GRANT USAGE ON SCHEMA msar, __msar TO intern_no_pkey; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA msar, __msar TO intern_no_pkey; + GRANT SELECT (col1, col2, col3, col4) ON TABLE atable TO intern_no_pkey; + SET ROLE intern_no_pkey; + RETURN NEXT is( + msar.list_records_from_table(rel_id, null, null, null, null, null, null), + $j${ + "count": 3, + "results": [ + {"2": 2, "3": "abcde", "4": {"k": 3242348}, "5": true}, + {"2": 5, "3": "sdflkj", "4": "s", "5": {"a": "val"}}, + {"2": 34, "3": "sdflfflsk", "4": null, "5": [1, 2, 3, 4]} + ] + }$j$ + ); + RETURN NEXT is( + msar.list_records_from_table( + rel_id, null, null, '[{"attnum": 3, "direction": "desc"}]', null, null, null + ), + $j${ + "count": 3, + "results": [ + {"2": 5, "3": "sdflkj", "4": "s", "5": {"a": "val"}}, + {"2": 34, "3": "sdflfflsk", "4": null, "5": [1, 2, 3, 4]}, + {"2": 2, "3": "abcde", "4": {"k": 3242348}, "5": true} + ] + }$j$ + ); +-- SET ROLE NONE; +-- REVOKE ALL ON TABLE atable FROM intern_no_pkey; +-- SET ROLE intern_no_pkey; +-- RETURN NEXT is(msar.build_order_by_expr(rel_id, null), null); END; $$ LANGUAGE plpgsql; +-- msar.build_order_by_expr ------------------------------------------------------------------------ + CREATE OR REPLACE FUNCTION test_build_order_by_expr() RETURNS SETOF TEXT AS $$ DECLARE rel_id oid; - order_by_expr text; BEGIN - PERFORM __setup_order_by_table(); + PERFORM __setup_list_records_table(); rel_id := 'atable'::regclass::oid; RETURN NEXT is(msar.build_order_by_expr(rel_id, null), 'ORDER BY "1" ASC'); RETURN NEXT is( From aeaef994abbef58c7c6d746cf15a152526c53ea0 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 12 Jul 2024 17:58:39 +0800 Subject: [PATCH 0420/1141] remove cruft --- db/sql/test_00_msar.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 66a330f9bd..fa45d24949 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2886,10 +2886,6 @@ BEGIN ] }$j$ ); --- SET ROLE NONE; --- REVOKE ALL ON TABLE atable FROM intern_no_pkey; --- SET ROLE intern_no_pkey; --- RETURN NEXT is(msar.build_order_by_expr(rel_id, null), null); END; $$ LANGUAGE plpgsql; From 2cb6d344a144e3afcc2e18a9208587d84e299980 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 12 Jul 2024 22:08:51 +0800 Subject: [PATCH 0421/1141] use get_or_create for the role map creation --- mathesar/utils/permissions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mathesar/utils/permissions.py b/mathesar/utils/permissions.py index 1c3ae45d82..3a907dd642 100644 --- a/mathesar/utils/permissions.py +++ b/mathesar/utils/permissions.py @@ -95,12 +95,12 @@ def _setup_connection_models( server=server, defaults={"password": password}, ) - return UserDatabaseRoleMap.objects.create( + return UserDatabaseRoleMap.objects.get_or_create( user=user, database=database, role=role, server=server - ) + )[0] def _load_sample_data(conn, sample_data): From 221408115735b7f67cfc590757548a4fd84f6a94 Mon Sep 17 00:00:00 2001 From: pavish Date: Sun, 14 Jul 2024 03:13:23 +0530 Subject: [PATCH 0422/1141] Rename role to configured_role for all Role model references --- .../0011_rename_role_configuredrole.py | 5 ++++ mathesar/models/base.py | 6 ++--- mathesar/rpc/collaborators.py | 24 +++++++++---------- mathesar/rpc/database_setup.py | 8 +++---- mathesar/tests/rpc/test_database_setup.py | 8 +++---- mathesar/utils/permissions.py | 4 ++-- 6 files changed, 30 insertions(+), 25 deletions(-) diff --git a/mathesar/migrations/0011_rename_role_configuredrole.py b/mathesar/migrations/0011_rename_role_configuredrole.py index 752d6012be..7dc768960e 100644 --- a/mathesar/migrations/0011_rename_role_configuredrole.py +++ b/mathesar/migrations/0011_rename_role_configuredrole.py @@ -14,4 +14,9 @@ class Migration(migrations.Migration): old_name='Role', new_name='ConfiguredRole', ), + migrations.RenameField( + model_name='userdatabaserolemap', + old_name='role', + new_name='configured_role', + ), ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index 98f1d2829b..ab34e92d75 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -61,7 +61,7 @@ class Meta: class UserDatabaseRoleMap(BaseModel): user = models.ForeignKey('User', on_delete=models.CASCADE) database = models.ForeignKey('Database', on_delete=models.CASCADE) - role = models.ForeignKey('ConfiguredRole', on_delete=models.CASCADE) + configured_role = models.ForeignKey('ConfiguredRole', on_delete=models.CASCADE) server = models.ForeignKey('Server', on_delete=models.CASCADE) class Meta: @@ -77,8 +77,8 @@ def connection(self): host=self.server.host, port=self.server.port, dbname=self.database.name, - user=self.role.name, - password=self.role.password, + user=self.configured_role.name, + password=self.configured_role.password, ) diff --git a/mathesar/rpc/collaborators.py b/mathesar/rpc/collaborators.py index 1851d03051..8273f9d4b7 100644 --- a/mathesar/rpc/collaborators.py +++ b/mathesar/rpc/collaborators.py @@ -16,12 +16,12 @@ class CollaboratorInfo(TypedDict): id: the Django ID of the UserDatabaseRoleMap model instance. user_id: The Django ID of the User model instance of the collaborator. database_id: the Django ID of the Database model instance for the collaborator. - role_id: The Django ID of the ConfiguredRole model instance for the collaborator. + configured_role_id: The Django ID of the ConfiguredRole model instance for the collaborator. """ id: int user_id: int database_id: int - role_id: int + configured_role_id: int @classmethod def from_model(cls, model): @@ -29,7 +29,7 @@ def from_model(cls, model): id=model.id, user_id=model.user.id, database_id=model.database.id, - role_id=model.role.id + configured_role_id=model.configured_role.id ) @@ -63,7 +63,7 @@ def add( *, database_id: int, user_id: int, - role_id: int, + configured_role_id: int, **kwargs ) -> CollaboratorInfo: """ @@ -72,16 +72,16 @@ def add( Args: database_id: The Django id of the Database to associate with the collaborator. user_id: The Django id of the User model instance who'd be the collaborator. - role_id: The Django id of the ConfiguredRole model instance to associate with the collaborator. + configured_role_id: The Django id of the ConfiguredRole model instance to associate with the collaborator. """ database = Database.objects.get(id=database_id) user = User.objects.get(id=user_id) - role = ConfiguredRole.objects.get(id=role_id) + configured_role = ConfiguredRole.objects.get(id=configured_role_id) collaborator = UserDatabaseRoleMap.objects.create( database=database, user=user, - role=role, - server=role.server + configured_role=configured_role, + server=configured_role.server ) return CollaboratorInfo.from_model(collaborator) @@ -106,7 +106,7 @@ def delete(*, collaborator_id: int, **kwargs): def set_role( *, collaborator_id: int, - role_id: int, + configured_role_id: int, **kwargs ) -> CollaboratorInfo: """ @@ -114,10 +114,10 @@ def set_role( Args: collaborator_id: The Django id of the UserDatabaseRoleMap model instance of the collaborator. - role_id: The Django id of the ConfiguredRole model instance to associate with the collaborator. + configured_role_id: The Django id of the ConfiguredRole model instance to associate with the collaborator. """ collaborator = UserDatabaseRoleMap.objects.get(id=collaborator_id) - role = ConfiguredRole.objects.get(id=role_id) - collaborator.role = role + configured_role = ConfiguredRole.objects.get(id=configured_role_id) + collaborator.configured_role = configured_role collaborator.save() return CollaboratorInfo.from_model(collaborator) diff --git a/mathesar/rpc/database_setup.py b/mathesar/rpc/database_setup.py index 37ea89969b..ce2ac09fb0 100644 --- a/mathesar/rpc/database_setup.py +++ b/mathesar/rpc/database_setup.py @@ -15,23 +15,23 @@ class DatabaseConnectionResult(TypedDict): Info about the objects resulting from calling the setup functions. These functions will get or create an instance of the Server, - Database, and Role models, as well as a UserDatabaseRoleMap entry. + Database, and ConfiguredRole models, as well as a UserDatabaseRoleMap entry. Attributes: server_id: The Django ID of the Server model instance. database_id: The Django ID of the Database model instance. - role_id: The Django ID of the Role model instance. + configured_role_id: The Django ID of the ConfiguredRole model instance. """ server_id: int database_id: int - role_id: int + configured_role_id: int @classmethod def from_model(cls, model): return cls( server_id=model.server.id, database_id=model.database.id, - role_id=model.role.id, + configured_role_id=model.configured_role.id, ) diff --git a/mathesar/tests/rpc/test_database_setup.py b/mathesar/tests/rpc/test_database_setup.py index e77cd92b8d..d9de031afc 100644 --- a/mathesar/tests/rpc/test_database_setup.py +++ b/mathesar/tests/rpc/test_database_setup.py @@ -27,7 +27,7 @@ def mock_set_up_new_for_user(database, user, sample_data=[]): db_model = Database(id=3, name=test_database, server=server_model) role_model = ConfiguredRole(id=4, name="matheuser", server=server_model) return UserDatabaseRoleMap( - user=user, database=db_model, role=role_model, server=server_model + user=user, database=db_model, configured_role=role_model, server=server_model ) monkeypatch.setattr( @@ -36,7 +36,7 @@ def mock_set_up_new_for_user(database, user, sample_data=[]): mock_set_up_new_for_user, ) expect_response = database_setup.DatabaseConnectionResult( - server_id=2, database_id=3, role_id=4 + server_id=2, database_id=3, configured_role_id=4 ) actual_response = database_setup.create_new( @@ -72,7 +72,7 @@ def mock_set_up_preexisting_database_for_user( db_model = Database(id=3, name=test_database, server=server_model) role_model = ConfiguredRole(id=4, name="matheuser", server=server_model) return UserDatabaseRoleMap( - user=user, database=db_model, role=role_model, server=server_model + user=user, database=db_model, configured_role=role_model, server=server_model ) monkeypatch.setattr( @@ -81,7 +81,7 @@ def mock_set_up_preexisting_database_for_user( mock_set_up_preexisting_database_for_user, ) expect_response = database_setup.DatabaseConnectionResult( - server_id=2, database_id=3, role_id=4 + server_id=2, database_id=3, configured_role_id=4 ) actual_response = database_setup.connect_existing( diff --git a/mathesar/utils/permissions.py b/mathesar/utils/permissions.py index e0a156fae6..34e2363be4 100644 --- a/mathesar/utils/permissions.py +++ b/mathesar/utils/permissions.py @@ -90,7 +90,7 @@ def _setup_connection_models( database, _ = Database.objects.get_or_create( name=database_name, server=server ) - role, _ = ConfiguredRole.objects.get_or_create( + configured_role, _ = ConfiguredRole.objects.get_or_create( name=role_name, server=server, defaults={"password": password}, @@ -98,7 +98,7 @@ def _setup_connection_models( return UserDatabaseRoleMap.objects.get_or_create( user=user, database=database, - role=role, + configured_role=configured_role, server=server )[0] From 004db00ce97684956abf93426cffcbd9fdcab889 Mon Sep 17 00:00:00 2001 From: pavish Date: Sun, 14 Jul 2024 16:19:56 +0530 Subject: [PATCH 0423/1141] Remove existing Access control models --- mathesar_ui/src/i18n/languages/en/dict.json | 1 - .../src/pages/database/DatabaseDetails.svelte | 14 - .../database/DbAccessControlModal.svelte | 68 ----- .../schema/SchemaAccessControlModal.svelte | 85 ------ .../src/pages/schema/SchemaPage.svelte | 19 +- .../AccessControlRow.svelte | 266 ------------------ .../AccessControlView.svelte | 192 ------------- .../systems/users-and-permissions/index.ts | 1 - 8 files changed, 1 insertion(+), 645 deletions(-) delete mode 100644 mathesar_ui/src/pages/database/DbAccessControlModal.svelte delete mode 100644 mathesar_ui/src/pages/schema/SchemaAccessControlModal.svelte delete mode 100644 mathesar_ui/src/systems/users-and-permissions/AccessControlRow.svelte delete mode 100644 mathesar_ui/src/systems/users-and-permissions/AccessControlView.svelte diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index c2edf16583..cb35003739 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -278,7 +278,6 @@ "loading_release_data": "Loading release data", "loading_spinner_no_progress_bar": "You will see a loading spinner but no progress bar.", "log_out": "Log Out", - "manage_access": "Manage Access", "manage_connections": "Manage Connections", "manage_database_access": "Manage [databaseName] Database Access", "manage_schema_access": "Manage [databaseName] Schema Access", diff --git a/mathesar_ui/src/pages/database/DatabaseDetails.svelte b/mathesar_ui/src/pages/database/DatabaseDetails.svelte index 6a1b655a3c..f66455305b 100644 --- a/mathesar_ui/src/pages/database/DatabaseDetails.svelte +++ b/mathesar_ui/src/pages/database/DatabaseDetails.svelte @@ -13,7 +13,6 @@ iconDatabase, iconDeleteMajor, iconEdit, - iconManageAccess, iconMoreActions, iconRefresh, } from '@mathesar/icons'; @@ -46,7 +45,6 @@ import SchemaRow from './SchemaRow.svelte'; const addEditModal = modal.spawnModalController(); - const accessControlModal = modal.spawnModalController(); const editConnectionModal = modal.spawnModalController(); const deleteConnectionModal = modal.spawnModalController(); @@ -106,10 +104,6 @@ }); } - function manageAccess() { - accessControlModal.open(); - } - function handleClearFilterQuery() { filterQuery = ''; } @@ -137,12 +131,6 @@ {#if canExecuteDDL || canEditPermissions}
- {#if canEditPermissions} - - {/if} - - - import { _ } from 'svelte-i18n'; - - import type { UserRole } from '@mathesar/api/rest/users'; - import type { Database } from '@mathesar/AppTypes'; - import Identifier from '@mathesar/components/Identifier.svelte'; - import ErrorBox from '@mathesar/components/message-boxes/ErrorBox.svelte'; - import { RichText } from '@mathesar/components/rich-text'; - import { - type UserModel, - setUsersStoreInContext, - } from '@mathesar/stores/users'; - import { AccessControlView } from '@mathesar/systems/users-and-permissions'; - import type { ObjectRoleMap } from '@mathesar/utils/permissions'; - import { - ControlledModal, - type ModalController, - } from '@mathesar-component-library'; - - export let controller: ModalController; - export let database: Database; - - const usersStore = setUsersStoreInContext(); - const { requestStatus } = usersStore; - - $: usersWithoutAccess = usersStore.getUsersWithoutAccessToDb(database); - $: usersWithAccess = usersStore.getUsersWithAccessToDb(database); - - async function addAccessForUser(user: UserModel, role: UserRole) { - await usersStore.addDatabaseRoleForUser(user.id, database, role); - } - - async function removeAccessForUser(user: UserModel) { - await usersStore.removeDatabaseAccessForUser(user.id, database); - } - - function getUserRoles(user: UserModel): ObjectRoleMap | undefined { - const dbRole = user.getRoleForDb(database); - return dbRole ? new Map([['database', dbRole.role]]) : undefined; - } - - - - - - {#if slotName === 'databaseName'} - {database.nickname} - {/if} - - - - {#if $requestStatus?.state === 'success'} - - {:else if $requestStatus?.state === 'processing'} -
{$_('loading')}
- {:else if $requestStatus?.state === 'failure'} - - {$requestStatus.errors.join(', ')} - - {/if} -
diff --git a/mathesar_ui/src/pages/schema/SchemaAccessControlModal.svelte b/mathesar_ui/src/pages/schema/SchemaAccessControlModal.svelte deleted file mode 100644 index cb22a2a557..0000000000 --- a/mathesar_ui/src/pages/schema/SchemaAccessControlModal.svelte +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - {#if slotName === 'databaseName'} - {schema.name} - {/if} - - - - {#if $requestStatus?.state === 'success'} - - {:else if $requestStatus?.state === 'processing'} -
{$_('loading')}
- {:else if $requestStatus?.state === 'failure'} - - {$requestStatus.errors.join(', ')} - - {/if} -
diff --git a/mathesar_ui/src/pages/schema/SchemaPage.svelte b/mathesar_ui/src/pages/schema/SchemaPage.svelte index fab522012b..6802f85a07 100644 --- a/mathesar_ui/src/pages/schema/SchemaPage.svelte +++ b/mathesar_ui/src/pages/schema/SchemaPage.svelte @@ -3,7 +3,7 @@ import type { Database, SchemaEntry } from '@mathesar/AppTypes'; import AppSecondaryHeader from '@mathesar/components/AppSecondaryHeader.svelte'; - import { iconEdit, iconManageAccess, iconSchema } from '@mathesar/icons'; + import { iconEdit, iconSchema } from '@mathesar/icons'; import LayoutWithHeader from '@mathesar/layouts/LayoutWithHeader.svelte'; import { makeSimplePageTitle } from '@mathesar/pages/pageTitleUtils'; import { @@ -24,7 +24,6 @@ import AddEditSchemaModal from '../database/AddEditSchemaModal.svelte'; import ExplorationSkeleton from './ExplorationSkeleton.svelte'; - import SchemaAccessControlModal from './SchemaAccessControlModal.svelte'; import SchemaExplorations from './SchemaExplorations.svelte'; import SchemaOverview from './SchemaOverview.svelte'; import SchemaTables from './SchemaTables.svelte'; @@ -42,13 +41,8 @@ $: canEditMetadata = userProfile?.hasPermission({ database, schema }, 'canEditMetadata') ?? false; - $: canEditPermissions = userProfile?.hasPermission( - { database, schema }, - 'canEditPermissions', - ); const addEditModal = modal.spawnModalController(); - const accessControlModal = modal.spawnModalController(); // NOTE: This has to be same as the name key in the paths prop of Route component type TabsKey = 'overview' | 'tables' | 'explorations'; @@ -92,10 +86,6 @@ $: isDefault = schema.name === 'public'; - function manageAccess() { - accessControlModal.open(); - } - logEvent('opened_schema', { database_name: database.nickname, schema_name: schema.name, @@ -127,12 +117,6 @@ {$_('edit_schema')} {/if} - {#if canEditPermissions} - - {/if}
@@ -191,7 +175,6 @@ - diff --git a/mathesar_ui/src/systems/users-and-permissions/AccessControlView.svelte b/mathesar_ui/src/systems/users-and-permissions/AccessControlView.svelte deleted file mode 100644 index 9aa9679357..0000000000 --- a/mathesar_ui/src/systems/users-and-permissions/AccessControlView.svelte +++ /dev/null @@ -1,192 +0,0 @@ - - -
- {#if !showAddForm} - - {/if} - - {#if showAddForm} -
-
- - - - - a?.id === b?.id} + valuesAreEqual={(a, b) => a?.oid === b?.oid} {autoSelect} bind:value on:change diff --git a/mathesar_ui/src/components/SelectTableWithinCurrentSchema.svelte b/mathesar_ui/src/components/SelectTableWithinCurrentSchema.svelte index 1c3c7b51f8..258c1bec27 100644 --- a/mathesar_ui/src/components/SelectTableWithinCurrentSchema.svelte +++ b/mathesar_ui/src/components/SelectTableWithinCurrentSchema.svelte @@ -1,14 +1,13 @@ diff --git a/mathesar_ui/src/components/TableName.svelte b/mathesar_ui/src/components/TableName.svelte index 80cb9b2de5..70705713f8 100644 --- a/mathesar_ui/src/components/TableName.svelte +++ b/mathesar_ui/src/components/TableName.svelte @@ -1,7 +1,7 @@ diff --git a/mathesar_ui/src/components/breadcrumb/EntitySelector.svelte b/mathesar_ui/src/components/breadcrumb/EntitySelector.svelte index c3c182f93a..5232e8d356 100644 --- a/mathesar_ui/src/components/breadcrumb/EntitySelector.svelte +++ b/mathesar_ui/src/components/breadcrumb/EntitySelector.svelte @@ -3,7 +3,7 @@ import { meta } from 'tinro'; import type { QueryInstance } from '@mathesar/api/rest/types/queries'; - import type { TableEntry } from '@mathesar/api/rest/types/tables'; + import type { Table } from '@mathesar/api/rest/types/tables'; import type { Schema } from '@mathesar/api/rpc/schemas'; import type { Database } from '@mathesar/AppTypes'; import { iconTable } from '@mathesar/icons'; @@ -26,7 +26,7 @@ export let schema: Schema; function makeTableBreadcrumbSelectorItem( - table: TableEntry, + table: Table, ): BreadcrumbSelectorEntryForTable { return { type: 'table', @@ -35,7 +35,7 @@ href: getLinkForTableItem(database.id, schema.oid, table), icon: iconTable, isActive() { - return table.id === $currentTableId; + return table.oid === $currentTableId; }, }; } diff --git a/mathesar_ui/src/components/breadcrumb/breadcrumbTypes.ts b/mathesar_ui/src/components/breadcrumb/breadcrumbTypes.ts index 3e761149fa..b9a437894f 100644 --- a/mathesar_ui/src/components/breadcrumb/breadcrumbTypes.ts +++ b/mathesar_ui/src/components/breadcrumb/breadcrumbTypes.ts @@ -1,5 +1,5 @@ import type { QueryInstance } from '@mathesar/api/rest/types/queries'; -import type { TableEntry } from '@mathesar/api/rest/types/tables'; +import type { Table } from '@mathesar/api/rest/types/tables'; import type { Schema } from '@mathesar/api/rpc/schemas'; import type { Database } from '@mathesar/AppTypes'; import type { @@ -26,7 +26,7 @@ export interface BreadcrumbItemTable { type: 'table'; database: Database; schema: Schema; - table: TableEntry; + table: Table; } export interface BreadcrumbItemSimple { @@ -40,7 +40,7 @@ export interface BreadcrumbItemRecord { type: 'record'; database: Database; schema: Schema; - table: TableEntry; + table: Table; record: { pk: string; summary: string; @@ -78,7 +78,7 @@ export interface SimpleBreadcrumbSelectorEntry export interface BreadcrumbSelectorEntryForTable extends BaseBreadcrumbSelectorEntry { type: 'table'; - table: TableEntry; + table: Table; } export type BreadcrumbSelectorEntry = diff --git a/mathesar_ui/src/components/cell-fabric/utils.ts b/mathesar_ui/src/components/cell-fabric/utils.ts index f70664dccc..3e9a501731 100644 --- a/mathesar_ui/src/components/cell-fabric/utils.ts +++ b/mathesar_ui/src/components/cell-fabric/utils.ts @@ -1,4 +1,4 @@ -import type { TableEntry } from '@mathesar/api/rest/types/tables'; +import type { Table } from '@mathesar/api/rest/types/tables'; import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { CellInfo } from '@mathesar/stores/abstract-types/types'; import type { RecordSummariesForSheet } from '@mathesar/stores/table-data/record-summaries/recordSummaryUtils'; @@ -24,12 +24,12 @@ export function getCellCap({ * When the cell falls within an FK column, this value will give the id of the * table to which the FK points. */ - fkTargetTableId?: TableEntry['id']; + fkTargetTableId?: Table['oid']; /** * When the cell falls within a PK column, this value will give the id of the * table. */ - pkTargetTableId?: TableEntry['id']; + pkTargetTableId?: Table['oid']; }): ComponentAndProps { if (fkTargetTableId) { const props: LinkedRecordCellExternalProps = { @@ -52,7 +52,7 @@ export function getCellCap({ export function getDbTypeBasedInputCap( column: CellColumnLike, - fkTargetTableId?: TableEntry['id'], + fkTargetTableId?: Table['oid'], optionalCellInfo?: CellInfo, ): ComponentAndProps { if (fkTargetTableId) { @@ -71,7 +71,7 @@ export function getDbTypeBasedInputCap( export function getInitialInputValue( column: CellColumnLike, - fkTargetTableId?: TableEntry['id'], + fkTargetTableId?: Table['oid'], optionalCellInfo?: CellInfo, ): unknown { if (fkTargetTableId) { diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte index 838161b297..9e123b5897 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte @@ -4,7 +4,7 @@ import { columnsApi } from '@mathesar/api/rest/columns'; import type { DataFile } from '@mathesar/api/rest/types/dataFiles'; - import type { TableEntry } from '@mathesar/api/rest/types/tables'; + import type { Table } from '@mathesar/api/rest/types/tables'; import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { Schema } from '@mathesar/api/rpc/schemas'; import type { Database } from '@mathesar/AppTypes'; @@ -62,7 +62,7 @@ export let database: Database; export let schema: Schema; - export let table: TableEntry; + export let table: Table; export let dataFile: DataFile; export let useColumnTypeInference = false; @@ -70,7 +70,7 @@ let columnPropertiesMap = buildColumnPropertiesMap([]); $: otherTableNames = [...$tables.data.values()] - .filter((t) => t.id !== table.id) + .filter((t) => t.oid !== table.oid) .map((t) => t.name); $: customizedTableName = requiredField(table.name, [ uniqueWith(otherTableNames, $_('table_name_already_exists')), @@ -82,7 +82,7 @@ $: processedColumns = processColumns(columns, $currentDbAbstractTypes.data); async function init() { - const columnsResponse = await columnsFetch.run(table.id); + const columnsResponse = await columnsFetch.run(table.oid); const fetchedColumns = columnsResponse?.resolvedValue?.results; if (!fetchedColumns) { return; @@ -90,7 +90,7 @@ columns = fetchedColumns; columnPropertiesMap = buildColumnPropertiesMap(columns); if (useColumnTypeInference) { - const response = await typeSuggestionsRequest.run(table.id); + const response = await typeSuggestionsRequest.run(table.oid); if (response.settlement?.state === 'resolved') { const typeSuggestions = response.settlement.value; columns = columns.map((column) => ({ @@ -103,11 +103,8 @@ } $: table, useColumnTypeInference, void init(); - function reload(props: { - table?: TableEntry; - useColumnTypeInference?: boolean; - }) { - const tableId = props.table?.id ?? table.id; + function reload(props: { table?: Table; useColumnTypeInference?: boolean }) { + const tableId = props.table?.oid ?? table.oid; router.goto( getImportPreviewPageUrl(database.id, schema.oid, tableId, { useColumnTypeInference: @@ -155,12 +152,12 @@ async function finishImport() { try { - await patchTable(table.id, { + await patchTable(table.oid, { name: $customizedTableName, import_verified: true, columns: finalizeColumns(columns, columnPropertiesMap), }); - router.goto(getTablePageUrl(database.id, schema.oid, table.id), true); + router.goto(getTablePageUrl(database.id, schema.oid, table.oid), true); } catch (err) { toast.fromError(err); } diff --git a/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts b/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts index 6d7474ca8c..e544c240e6 100644 --- a/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts +++ b/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts @@ -1,9 +1,6 @@ import { dataFilesApi } from '@mathesar/api/rest/dataFiles'; import type { DataFile } from '@mathesar/api/rest/types/dataFiles'; -import type { - MinimalColumnDetails, - TableEntry, -} from '@mathesar/api/rest/types/tables'; +import type { Table } from '@mathesar/api/rest/types/tables'; import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { Schema } from '@mathesar/api/rpc/schemas'; import type { Database } from '@mathesar/AppTypes'; @@ -53,14 +50,14 @@ export function makeHeaderUpdateRequest() { interface Props { database: Database; schema: Schema; - table: Pick; + table: Pick; dataFile: Pick; firstRowIsHeader: boolean; customizedTableName: string; } async function updateHeader(p: Props) { await Promise.all([ - deleteTable(p.database, p.schema, p.table.id), + deleteTable(p.database, p.schema, p.table.oid), dataFilesApi.update(p.dataFile.id, { header: p.firstRowIsHeader }), ]); return createTable(p.database, p.schema, { @@ -75,10 +72,10 @@ export function makeDeleteTableRequest() { interface Props { database: Database; schema: Schema; - table: Pick; + table: Pick; } return new AsyncStore((props: Props) => - deleteTable(props.database, props.schema, props.table.id), + deleteTable(props.database, props.schema, props.table.oid), ); } @@ -104,7 +101,7 @@ export function buildColumnPropertiesMap( export function finalizeColumns( columns: Column[], columnPropertiesMap: ColumnPropertiesMap, -): MinimalColumnDetails[] { +): Table['columns'] { return columns .filter((column) => columnPropertiesMap[column.id]?.selected) .map((column) => ({ diff --git a/mathesar_ui/src/pages/import/upload/ImportUploadPage.svelte b/mathesar_ui/src/pages/import/upload/ImportUploadPage.svelte index 64c670d4d1..75896e172a 100644 --- a/mathesar_ui/src/pages/import/upload/ImportUploadPage.svelte +++ b/mathesar_ui/src/pages/import/upload/ImportUploadPage.svelte @@ -119,7 +119,7 @@ const previewPage = getImportPreviewPageUrl( database.id, schema.oid, - table.id, + table.oid, { useColumnTypeInference: $useColumnTypeInference }, ); router.goto(previewPage, true); diff --git a/mathesar_ui/src/pages/record/RecordPage.svelte b/mathesar_ui/src/pages/record/RecordPage.svelte index bde3ee3ab9..ae9b4fe4af 100644 --- a/mathesar_ui/src/pages/record/RecordPage.svelte +++ b/mathesar_ui/src/pages/record/RecordPage.svelte @@ -1,5 +1,5 @@ diff --git a/mathesar_ui/src/pages/schema/CreateNewTableButton.svelte b/mathesar_ui/src/pages/schema/CreateNewTableButton.svelte index 9a22b3733e..343b3812a2 100644 --- a/mathesar_ui/src/pages/schema/CreateNewTableButton.svelte +++ b/mathesar_ui/src/pages/schema/CreateNewTableButton.svelte @@ -24,7 +24,7 @@ isCreatingNewTable = true; const tableInfo = await createTable(database, schema, {}); isCreatingNewTable = false; - router.goto(getTablePageUrl(database.id, schema.oid, tableInfo.id), false); + router.goto(getTablePageUrl(database.id, schema.oid, tableInfo.oid), false); } diff --git a/mathesar_ui/src/pages/schema/EditTable.svelte b/mathesar_ui/src/pages/schema/EditTable.svelte index 88c61ffa2f..0007deb56d 100644 --- a/mathesar_ui/src/pages/schema/EditTable.svelte +++ b/mathesar_ui/src/pages/schema/EditTable.svelte @@ -1,18 +1,18 @@ - + onUpdate({ name, description })} diff --git a/mathesar_ui/src/pages/schema/SchemaOverview.svelte b/mathesar_ui/src/pages/schema/SchemaOverview.svelte index ad9d8cd6ca..20d5094024 100644 --- a/mathesar_ui/src/pages/schema/SchemaOverview.svelte +++ b/mathesar_ui/src/pages/schema/SchemaOverview.svelte @@ -2,7 +2,7 @@ import { _ } from 'svelte-i18n'; import type { QueryInstance } from '@mathesar/api/rest/types/queries'; - import type { TableEntry } from '@mathesar/api/rest/types/tables'; + import type { Table } from '@mathesar/api/rest/types/tables'; import type { RequestStatus } from '@mathesar/api/rest/utils/requestUtils'; import type { Schema } from '@mathesar/api/rpc/schemas'; import type { Database } from '@mathesar/AppTypes'; @@ -23,7 +23,7 @@ import TableSkeleton from './TableSkeleton.svelte'; import TablesList from './TablesList.svelte'; - export let tablesMap: Map; + export let tablesMap: Map; export let explorationsMap: Map; export let tablesRequestStatus: RequestStatus; export let explorationsRequestStatus: RequestStatus; diff --git a/mathesar_ui/src/pages/schema/SchemaTables.svelte b/mathesar_ui/src/pages/schema/SchemaTables.svelte index c9f6316426..4937dad7d2 100644 --- a/mathesar_ui/src/pages/schema/SchemaTables.svelte +++ b/mathesar_ui/src/pages/schema/SchemaTables.svelte @@ -1,7 +1,7 @@ diff --git a/mathesar_ui/src/pages/schema/TablesList.svelte b/mathesar_ui/src/pages/schema/TablesList.svelte index 2969bb0b06..bbc766f9c1 100644 --- a/mathesar_ui/src/pages/schema/TablesList.svelte +++ b/mathesar_ui/src/pages/schema/TablesList.svelte @@ -1,7 +1,7 @@
- {#each tables as table (table.id)} + {#each tables as table (table.oid)} {:else} diff --git a/mathesar_ui/src/pages/table/TablePage.svelte b/mathesar_ui/src/pages/table/TablePage.svelte index a72515d1f5..ec4c1471ca 100644 --- a/mathesar_ui/src/pages/table/TablePage.svelte +++ b/mathesar_ui/src/pages/table/TablePage.svelte @@ -2,7 +2,7 @@ import { tick } from 'svelte'; import { router } from 'tinro'; - import type { TableEntry } from '@mathesar/api/rest/types/tables'; + import type { Table } from '@mathesar/api/rest/types/tables'; import { focusActiveCell } from '@mathesar/components/sheet/utils'; import LayoutWithHeader from '@mathesar/layouts/LayoutWithHeader.svelte'; import { makeSimplePageTitle } from '@mathesar/pages/pageTitleUtils'; @@ -27,7 +27,7 @@ ); setNewImperativeFilterControllerInContext(); - export let table: TableEntry; + export let table: Table; export let shareConsumer: ShareConsumer | undefined = undefined; let sheetElement: HTMLElement; @@ -36,7 +36,7 @@ $: ({ query } = $router); $: meta = Meta.fromSerialization(query[metaSerializationQueryKey] ?? ''); $: tabularData = new TabularData({ - id: table.id, + id: table.oid, abstractTypesMap, meta, table, diff --git a/mathesar_ui/src/routes/RecordPageRoute.svelte b/mathesar_ui/src/routes/RecordPageRoute.svelte index 82f16ba0f7..d5456249f2 100644 --- a/mathesar_ui/src/routes/RecordPageRoute.svelte +++ b/mathesar_ui/src/routes/RecordPageRoute.svelte @@ -1,5 +1,5 @@ diff --git a/mathesar_ui/src/systems/record-selector/recordSelectorUtils.ts b/mathesar_ui/src/systems/record-selector/recordSelectorUtils.ts index 3c897c4c7c..f316771211 100644 --- a/mathesar_ui/src/systems/record-selector/recordSelectorUtils.ts +++ b/mathesar_ui/src/systems/record-selector/recordSelectorUtils.ts @@ -1,4 +1,4 @@ -import type { TableEntry } from '@mathesar/api/rest/types/tables'; +import type { Table } from '@mathesar/api/rest/types/tables'; import type { Column } from '@mathesar/api/rest/types/tables/columns'; /** @@ -18,7 +18,7 @@ export function getColumnIdToFocusInitially({ table, columns, }: { - table: TableEntry | undefined; + table: Table | undefined; columns: Column[]; }): number | undefined { function getFromRecordSummaryTemplate() { diff --git a/mathesar_ui/src/systems/table-view/TableView.svelte b/mathesar_ui/src/systems/table-view/TableView.svelte index 833a2d7368..642282eea5 100644 --- a/mathesar_ui/src/systems/table-view/TableView.svelte +++ b/mathesar_ui/src/systems/table-view/TableView.svelte @@ -3,7 +3,7 @@ import type { ComponentProps } from 'svelte'; import { get } from 'svelte/store'; - import type { TableEntry } from '@mathesar/api/rest/types/tables'; + import type { Table } from '@mathesar/api/rest/types/tables'; import { ImmutableMap, Spinner } from '@mathesar/component-library'; import { Sheet } from '@mathesar/components/sheet'; import { SheetClipboardHandler } from '@mathesar/components/sheet/SheetClipboardHandler'; @@ -37,7 +37,7 @@ ); export let context: Context = 'page'; - export let table: Pick; + export let table: Pick; export let sheetElement: HTMLElement | undefined = undefined; let tableInspectorTab: ComponentProps['activeTabId'] = diff --git a/mathesar_ui/src/systems/table-view/actions-pane/ActionsPane.svelte b/mathesar_ui/src/systems/table-view/actions-pane/ActionsPane.svelte index a7277b088b..23a91eb430 100644 --- a/mathesar_ui/src/systems/table-view/actions-pane/ActionsPane.svelte +++ b/mathesar_ui/src/systems/table-view/actions-pane/ActionsPane.svelte @@ -1,7 +1,7 @@ import { _ } from 'svelte-i18n'; - import type { TableEntry } from '@mathesar/api/rest/types/tables'; + import type { Table } from '@mathesar/api/rest/types/tables'; import { type FilledFormValues, FormSubmit, @@ -57,7 +57,7 @@ ); $: baseColumn = requiredField(undefined); - $: targetTable = requiredField(undefined); + $: targetTable = requiredField(undefined); $: targetColumn = requiredField(undefined); $: namingStrategy = requiredField('auto'); $: constraintName = requiredField(undefined, [ @@ -78,7 +78,7 @@ $: targetTableStructure = $targetTable ? new TableStructure({ - id: $targetTable.id, + id: $targetTable.oid, abstractTypesMap: $currentDbAbstractTypes.data, }) : undefined; @@ -106,7 +106,7 @@ columns: [values.baseColumn.id], type: 'foreignkey', name: values.constraintName, - referent_table: values.targetTable.id, + referent_table: values.targetTable.oid, referent_columns: [values.targetColumn.id], }); // Why reset before close when the form is automatically reset during diff --git a/mathesar_ui/src/systems/table-view/header/Header.svelte b/mathesar_ui/src/systems/table-view/header/Header.svelte index 88e779d715..f4e30394e7 100644 --- a/mathesar_ui/src/systems/table-view/header/Header.svelte +++ b/mathesar_ui/src/systems/table-view/header/Header.svelte @@ -1,7 +1,7 @@ diff --git a/mathesar_ui/src/systems/table-view/link-table/LinkTypeOption.svelte b/mathesar_ui/src/systems/table-view/link-table/LinkTypeOption.svelte index f1a956740f..c6997204f5 100644 --- a/mathesar_ui/src/systems/table-view/link-table/LinkTypeOption.svelte +++ b/mathesar_ui/src/systems/table-view/link-table/LinkTypeOption.svelte @@ -1,7 +1,7 @@ diff --git a/mathesar_ui/src/systems/table-view/link-table/linkTableUtils.ts b/mathesar_ui/src/systems/table-view/link-table/linkTableUtils.ts index 285b598728..c03ad449ee 100644 --- a/mathesar_ui/src/systems/table-view/link-table/linkTableUtils.ts +++ b/mathesar_ui/src/systems/table-view/link-table/linkTableUtils.ts @@ -1,7 +1,7 @@ import { get } from 'svelte/store'; import { _ } from 'svelte-i18n'; -import type { TableEntry } from '@mathesar/api/rest/types/tables'; +import type { Table } from '@mathesar/api/rest/types/tables'; import { invalidIf } from '@mathesar/components/form'; import { getAvailableName } from '@mathesar/utils/db'; @@ -15,9 +15,9 @@ export function columnNameIsNotId() { } export function suggestMappingTableName( - baseTable: Pick, - targetTable: Pick | undefined, - allTables: Pick[], + baseTable: Pick, + targetTable: Pick | undefined, + allTables: Pick[], ): string { return targetTable ? getAvailableName( diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte index 9b8682c546..35b594c931 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte @@ -141,7 +141,7 @@ const extractedColumnIds = extractedColumns.map((c) => c.id); try { if ($targetType === 'existingTable') { - const targetTableId = $linkedTable?.table.id; + const targetTableId = $linkedTable?.table.oid; if (!targetTableId) { throw new Error($_('no_target_table_selected')); } diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/columnExtractionTypes.ts b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/columnExtractionTypes.ts index 0611fc54b4..58efb5dda3 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/columnExtractionTypes.ts +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/columnExtractionTypes.ts @@ -1,4 +1,4 @@ -import type { TableEntry } from '@mathesar/api/rest/types/tables'; +import type { Table } from '@mathesar/api/rest/types/tables'; import type { FkConstraint } from '@mathesar/api/rest/types/tables/constraints'; import type { ProcessedColumn } from '@mathesar/stores/table-data'; @@ -14,5 +14,5 @@ export interface LinkedTable { /** The columns specified in the FK constraint. */ columns: ProcessedColumn[]; /** The table to which the FK constraint points. */ - table: TableEntry; + table: Table; } diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/columnExtractionUtils.ts b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/columnExtractionUtils.ts index 40b4fb240e..3e9d492328 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/columnExtractionUtils.ts +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/columnExtractionUtils.ts @@ -1,7 +1,7 @@ import { get } from 'svelte/store'; import { _ } from 'svelte-i18n'; -import type { TableEntry } from '@mathesar/api/rest/types/tables'; +import type { Table } from '@mathesar/api/rest/types/tables'; import type { FkConstraint } from '@mathesar/api/rest/types/tables/constraints'; import { isDefinedNonNullable } from '@mathesar/component-library'; import { @@ -24,7 +24,7 @@ function getLinkedTable({ }: { fkConstraint: FkConstraint; columns: Map; - tables: Map; + tables: Map; }): LinkedTable | undefined { const table = tables.get(fkConstraint.referent_table); if (!table) { @@ -46,7 +46,7 @@ export function getLinkedTables({ }: { constraints: Constraint[]; columns: Map; - tables: Map; + tables: Map; }): LinkedTable[] { return constraints .map((c) => diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/FkRecordSummaryConfig.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/FkRecordSummaryConfig.svelte index 86afae45c2..68edf145e4 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/FkRecordSummaryConfig.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/FkRecordSummaryConfig.svelte @@ -1,15 +1,15 @@ ; +type TablesMap = Map; + +interface TablesData { + tablesMap: TablesMap; requestStatus: RequestStatus; } -const schemaTablesStoreMap: Map< - Schema['oid'], - Writable -> = new Map(); -const schemaTablesRequestMap: Map< +type TablesStore = Writable; + +const tablesStores: Map = new Map(); + +const tablesRequests: Map< Schema['oid'], - CancellablePromise> + CancellablePromise > = new Map(); -function sortedTableEntries(tableEntries: Table[]): Table[] { - return [...tableEntries].sort((a, b) => a.name.localeCompare(b.name)); +function sortTables(tables: Iterable
): Table[] { + return [...tables].sort((a, b) => a.name.localeCompare(b.name)); } -function setSchemaTablesStore( - schemaId: Schema['oid'], - tableEntries?: Table[], -): Writable { - const tables: DBTablesStoreData['data'] = new Map(); - if (tableEntries) { - sortedTableEntries(tableEntries).forEach((entry) => { - tables.set(entry.oid, entry); - }); +function setTablesStore( + schemaOid: Schema['oid'], + tables?: Table[], +): TablesStore { + const tablesMap: TablesMap = new Map(); + if (tables) { + sortTables(tables).forEach((t) => tablesMap.set(t.oid, t)); } - const storeValue: DBTablesStoreData = { - data: tables, + const storeValue: TablesData = { + tablesMap, requestStatus: { state: 'success' }, }; - let store = schemaTablesStoreMap.get(schemaId); + let store = tablesStores.get(schemaOid); if (!store) { store = writable(storeValue); - schemaTablesStoreMap.set(schemaId, store); + tablesStores.set(schemaOid, store); } else { store.set(storeValue); } return store; } -export function removeTablesInSchemaTablesStore(schemaId: Schema['oid']): void { - schemaTablesStoreMap.delete(schemaId); +export function removeTablesStore(schemaOid: Schema['oid']): void { + tablesStores.delete(schemaOid); } export async function refetchTablesForSchema( - schemaId: Schema['oid'], -): Promise { - const store = schemaTablesStoreMap.get(schemaId); + schemaOid: Schema['oid'], +): Promise { + const store = tablesStores.get(schemaOid); if (!store) { - console.error(`Tables store for schema: ${schemaId} not found.`); + console.error(`Tables store for schema: ${schemaOid} not found.`); return undefined; } @@ -110,16 +110,16 @@ export async function refetchTablesForSchema( requestStatus: { state: 'processing' }, })); - schemaTablesRequestMap.get(schemaId)?.cancel(); + tablesRequests.get(schemaOid)?.cancel(); const tablesRequest = getAPI>( - `/api/db/v0/tables/?schema=${schemaId}&limit=500`, + `/api/db/v0/tables/?schema=${schemaOid}&limit=500`, ); - schemaTablesRequestMap.set(schemaId, tablesRequest); + tablesRequests.set(schemaOid, tablesRequest); const response = await tablesRequest; const tableEntries = response.results || []; - const schemaTablesStore = setSchemaTablesStore(schemaId, tableEntries); + const schemaTablesStore = setTablesStore(schemaOid, tableEntries); return get(schemaTablesStore); } catch (err) { @@ -138,75 +138,63 @@ export async function refetchTablesForSchema( let preload = true; -export function getTablesStoreForSchema( - schemaId: Schema['oid'], -): Writable { - let store = schemaTablesStoreMap.get(schemaId); +function getTablesStore(schemaOid: Schema['oid']): TablesStore { + let store = tablesStores.get(schemaOid); if (!store) { store = writable({ requestStatus: { state: 'processing' }, - data: new Map(), + tablesMap: new Map(), }); - schemaTablesStoreMap.set(schemaId, store); - if (preload && commonData.current_schema === schemaId) { - store = setSchemaTablesStore(schemaId, commonData.tables ?? []); + tablesStores.set(schemaOid, store); + if (preload && commonData.current_schema === schemaOid) { + store = setTablesStore(schemaOid, commonData.tables ?? []); } else { - void refetchTablesForSchema(schemaId); + void refetchTablesForSchema(schemaOid); } preload = false; } else if (get(store).requestStatus.state === 'failure') { - void refetchTablesForSchema(schemaId); + void refetchTablesForSchema(schemaOid); } return store; } -/** - * TODO: Use a dedicated higher level Tables store and - * remove this function. - */ -function findSchemaStoreForTable(id: Table['oid']) { - return [...schemaTablesStoreMap.values()].find((entry) => - get(entry).data.has(id), - ); +function findSchemaStoreForTable( + tableOid: Table['oid'], +): TablesStore | undefined { + // TODO rewrite this function + throw new Error('Not implemented'); } -function findAndUpdateTableStore(id: Table['oid'], table: Table) { - findSchemaStoreForTable(id)?.update((tableStoreData) => { - const existingTableEntry = tableStoreData.data.get(id); - const updatedTableEntry = { - ...(existingTableEntry ?? {}), - ...table, - }; - tableStoreData.data.set(id, updatedTableEntry); - const tableEntryMap: DBTablesStoreData['data'] = new Map(); - sortedTableEntries([...tableStoreData.data.values()]).forEach((entry) => { - tableEntryMap.set(entry.oid, entry); +function findAndUpdateTableStore(tableOid: Table['oid'], newTable: Table) { + findSchemaStoreForTable(tableOid)?.update((tablesData) => { + const oldTable = tablesData.tablesMap.get(tableOid); + tablesData.tablesMap.set(tableOid, { ...(oldTable ?? {}), ...newTable }); + const tablesMap: TablesMap = new Map(); + sortTables([...tablesData.tablesMap.values()]).forEach((t) => { + tablesMap.set(t.oid, t); }); - return { - ...tableStoreData, - data: tableEntryMap, - }; + return { ...tablesData, tablesMap }; }); } export function deleteTable( database: Database, schema: Schema, - tableId: Table['oid'], + tableOid: Table['oid'], ): CancellablePromise
{ - const promise = deleteAPI
(`/api/db/v0/tables/${tableId}/`); + const promise = deleteAPI
(`/api/db/v0/tables/${tableOid}/`); return new CancellablePromise( (resolve, reject) => { - void promise.then((value) => { + void promise.then((table) => { addCountToSchemaNumTables(database, schema, -1); - schemaTablesStoreMap.get(schema.oid)?.update((tableStoreData) => { - tableStoreData.data.delete(tableId); + tablesStores.get(schema.oid)?.update((tableStoreData) => { + tableStoreData.tablesMap.delete(tableOid); return { ...tableStoreData, - data: new Map(tableStoreData.data), + tablesMap: new Map(tableStoreData.tablesMap), }; }); - return resolve(value); + return resolve(table); }, reject); }, () => { @@ -216,15 +204,18 @@ export function deleteTable( } export function updateTableMetaData( - id: number, + tableOid: number, updatedMetaData: AtLeastOne<{ name: string; description: string }>, ): CancellablePromise
{ - const promise = patchAPI
(`/api/db/v0/tables/${id}/`, updatedMetaData); + const promise = patchAPI
( + `/api/db/v0/tables/${tableOid}/`, + updatedMetaData, + ); return new CancellablePromise( (resolve, reject) => { - void promise.then((value) => { - findAndUpdateTableStore(id, value); - return resolve(value); + void promise.then((table) => { + findAndUpdateTableStore(tableOid, table); + return resolve(table); }, reject); }, () => { @@ -248,21 +239,18 @@ export function createTable( }); return new CancellablePromise( (resolve, reject) => { - void promise.then((value) => { + void promise.then((table) => { addCountToSchemaNumTables(database, schema, 1); - schemaTablesStoreMap.get(schema.oid)?.update((existing) => { - const tableEntryMap: DBTablesStoreData['data'] = new Map(); - sortedTableEntries([...existing.data.values(), value]).forEach( + tablesStores.get(schema.oid)?.update((existing) => { + const tablesMap: TablesMap = new Map(); + sortTables([...existing.tablesMap.values(), table]).forEach( (entry) => { - tableEntryMap.set(entry.oid, entry); + tablesMap.set(entry.oid, entry); }, ); - return { - ...existing, - data: tableEntryMap, - }; + return { ...existing, tablesMap }; }); - return resolve(value); + return resolve(table); }, reject); }, () => { @@ -272,18 +260,18 @@ export function createTable( } export function patchTable( - id: Table['oid'], + tableOid: Table['oid'], patch: { name?: Table['name']; import_verified?: Table['import_verified']; columns?: Table['columns']; }, ): CancellablePromise
{ - const promise = patchAPI
(`/api/db/v0/tables/${id}/`, patch); + const promise = patchAPI
(`/api/db/v0/tables/${tableOid}/`, patch); return new CancellablePromise( (resolve, reject) => { void promise.then((value) => { - findAndUpdateTableStore(id, value); + findAndUpdateTableStore(tableOid, value); return resolve(value); }, reject); }, @@ -306,8 +294,8 @@ export function patchTable( * 3. Move all api-call-only functions to /api. Only keep functions that * update the stores within /stores */ -export function getTable(id: Table['oid']): CancellablePromise
{ - return getAPI(`/api/db/v0/tables/${id}/`); +export function getTable(tableOid: Table['oid']): CancellablePromise
{ + return getAPI(`/api/db/v0/tables/${tableOid}/`); } export function splitTable({ @@ -330,11 +318,11 @@ export function splitTable({ } export function moveColumns( - tableId: number, + tableOid: number, idsOfColumnsToMove: number[], targetTableId: number, ): CancellablePromise { - return postAPI(`/api/db/v0/tables/${tableId}/move_columns/`, { + return postAPI(`/api/db/v0/tables/${tableOid}/move_columns/`, { move_columns: idsOfColumnsToMove, target_table: targetTableId, }); @@ -344,32 +332,32 @@ export function moveColumns( * Replace getTable with this function once the above mentioned changes are done. */ export function getTableFromStoreOrApi( - id: Table['oid'], + tableOid: Table['oid'], ): CancellablePromise
{ - const schemaStore = findSchemaStoreForTable(id); - if (schemaStore) { - const tableEntry = get(schemaStore).data.get(id); - if (tableEntry) { + const tablesStore = findSchemaStoreForTable(tableOid); + if (tablesStore) { + const table = get(tablesStore).tablesMap.get(tableOid); + if (table) { return new CancellablePromise((resolve) => { - resolve(tableEntry); + resolve(table); }); } } - const promise = getTable(id); + const promise = getTable(tableOid); return new CancellablePromise( (resolve, reject) => { void promise.then((table) => { - const store = schemaTablesStoreMap.get(table.schema); + const store = tablesStores.get(table.schema); if (store) { store.update((existing) => { const tableMap = new Map(); - const tables = [...existing.data.values(), table]; - sortedTableEntries(tables).forEach((t) => { + const tables = [...existing.tablesMap.values(), table]; + sortTables(tables).forEach((t) => { tableMap.set(t.oid, t); }); return { ...existing, - data: tableMap, + tablesMap: tableMap, }; }); } @@ -407,18 +395,18 @@ export function generateTablePreview(props: { return postAPI(`/api/db/v0/tables/${table.oid}/previews/`, { columns }); } -export const tables: Readable = derived( +export const tables: Readable = derived( currentSchemaId, ($currentSchemaId, set) => { let unsubscribe: Unsubscriber; if (!$currentSchemaId) { set({ - data: new Map(), + tablesMap: new Map(), requestStatus: { state: 'success' }, }); } else { - const store = getTablesStoreForSchema($currentSchemaId); + const store = getTablesStore($currentSchemaId); unsubscribe = store.subscribe((dbSchemasData) => { set(dbSchemasData); }); @@ -430,19 +418,18 @@ export const tables: Readable = derived( }, ); -export const importVerifiedTables: Readable = - derived( - tables, - ($tables) => - new Map( - [...$tables.data.values()] - .filter((table) => !isTableImportConfirmationRequired(table)) - .map((table) => [table.oid, table]), - ), - ); +export const importVerifiedTables: Readable = derived( + tables, + ($tables) => + new Map( + [...$tables.tablesMap.values()] + .filter((table) => !isTableImportConfirmationRequired(table)) + .map((table) => [table.oid, table]), + ), +); export const validateNewTableName = derived(tables, ($tables) => { - const names = new Set([...$tables.data.values()].map((t) => t.name)); + const names = new Set([...$tables.tablesMap.values()].map((t) => t.name)); return invalidIf( (name: string) => names.has(name), 'A table with that name already exists.', @@ -450,7 +437,7 @@ export const validateNewTableName = derived(tables, ($tables) => { }); export function getTableName(id: DBObjectEntry['id']): string | undefined { - return get(tables).data.get(id)?.name; + return get(tables).tablesMap.get(id)?.name; } export const currentTableId = writable(undefined); @@ -460,7 +447,7 @@ export const currentTable = derived( ([$currentTableId, $tables]) => $currentTableId === undefined ? undefined - : $tables.data.get($currentTableId), + : $tables.tablesMap.get($currentTableId), ); export function getJoinableTablesResult(tableId: number, maxDepth = 1) { diff --git a/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte b/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte index 449ac7ace3..585e5bcc20 100644 --- a/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte +++ b/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte @@ -43,7 +43,7 @@ $: ({ query, state, queryHasUnsavedChanges } = queryManager); $: currentTable = $query.base_table - ? $tablesDataStore.data.get($query.base_table) + ? $tablesDataStore.tablesMap.get($query.base_table) : undefined; $: isSaved = $query.isSaved(); $: hasNoColumns = $query.initial_columns.length === 0; diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte index 9c96febd74..05276014b0 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte @@ -88,7 +88,7 @@ const record = response.results[0]; const recordId = getPkValueInRecord(record, $columns); const previewData = response.preview_data ?? []; - const table = $tables.data.get(tableId); + const table = $tables.tablesMap.get(tableId); const template = table?.settings?.preview_settings?.template; if (!template) { throw new Error('No record summary template found in API response.'); diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte index f56e9c2442..9a9bf877f1 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte @@ -56,7 +56,7 @@ recordsData, processedColumns, } = tabularData); - $: table = $tables.data.get(tableId); + $: table = $tables.tablesMap.get(tableId); $: ({ recordSummaries, state: recordsDataState } = recordsData); $: recordsDataIsLoading = $recordsDataState === States.Loading; $: ({ constraints } = $constraintsDataStore); @@ -137,7 +137,7 @@ if (!record || recordId === undefined) { return; } - const tableEntry = $tables.data.get(tableId); + const tableEntry = $tables.tablesMap.get(tableId); const template = tableEntry?.settings?.preview_settings?.template ?? ''; const recordSummary = renderTransitiveRecordSummary({ template, diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte index 167133e63f..05c9c5c425 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte @@ -31,7 +31,7 @@ nestingLevel: controller.nestingLevel + 1, }); $: ({ tableId, purpose } = controller); - $: table = $tableId && $tables.data.get($tableId); + $: table = $tableId && $tables.tablesMap.get($tableId); $: tabularData = $tableId && table ? new TabularData({ diff --git a/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte b/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte index a4d6c92120..03fee0ab5a 100644 --- a/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte @@ -14,7 +14,7 @@ $: referentTable = constraint.type === 'foreignkey' - ? $tables.data.get(constraint.referent_table) + ? $tables.tablesMap.get(constraint.referent_table) : undefined; async function getReferentColumns(_constraint: Constraint) { diff --git a/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte b/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte index 3c6b7379bd..9c6f952ac8 100644 --- a/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte @@ -75,7 +75,7 @@ $: existingConstraintNames = new Set( $constraintsDataStore.constraints.map((c) => c.name), ); - $: tableName = $tables.data.get($tabularData.id)?.name ?? ''; + $: tableName = $tables.tablesMap.get($tabularData.id)?.name ?? ''; $: ({ processedColumns } = $tabularData); $: columnsInTable = Array.from($processedColumns.values()); $: nameValidationErrors = getNameValidationErrors( diff --git a/mathesar_ui/src/systems/table-view/header/header-cell/ColumnHeaderContextMenu.svelte b/mathesar_ui/src/systems/table-view/header/header-cell/ColumnHeaderContextMenu.svelte index 24ebf06e2d..0110e13aba 100644 --- a/mathesar_ui/src/systems/table-view/header/header-cell/ColumnHeaderContextMenu.svelte +++ b/mathesar_ui/src/systems/table-view/header/header-cell/ColumnHeaderContextMenu.svelte @@ -60,7 +60,9 @@ $: hasGrouping = $grouping.hasColumn(columnId); $: ({ linkFk } = processedColumn); - $: linkedTable = linkFk ? $tables.data.get(linkFk.referent_table) : undefined; + $: linkedTable = linkFk + ? $tables.tablesMap.get(linkFk.referent_table) + : undefined; $: linkedTableHref = linkedTable ? $storeToGetTablePageUrl({ tableId: linkedTable.oid }) : undefined; diff --git a/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte b/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte index 33b6977c52..dcd0fef696 100644 --- a/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte +++ b/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte @@ -61,7 +61,7 @@ // =========================================================================== $: singularBaseTableName = makeSingular(base.name); $: importVerifiedTables = [...$importVerifiedTablesStore.values()]; - $: allTables = [...$tablesDataStore.data.values()]; + $: allTables = [...$tablesDataStore.tablesMap.values()]; $: ({ columnsDataStore } = $tabularData); $: baseColumns = columnsDataStore.columns; diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte index fd9af53489..7815072839 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte @@ -145,7 +145,7 @@ {@const referentTable = referentTableId === undefined ? undefined - : $tables.data.get(referentTableId)} + : $tables.tablesMap.get(referentTableId)} {#if referentTable !== undefined}
diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte index 35b594c931..f1395b917c 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte @@ -84,7 +84,7 @@ $: linkedTables = getLinkedTables({ constraints, columns: $processedColumns, - tables: $tablesDataStore.data, + tables: $tablesDataStore.tablesMap, }); $: selectedColumnsHelpText = (() => { if ($targetType === 'existingTable') { diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte index 332abba58f..2092698daf 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte @@ -37,7 +37,7 @@ $currentSchema.oid, { oid: $tabularData.id, - name: $tables.data.get($tabularData.id)?.name ?? '', + name: $tables.tablesMap.get($tabularData.id)?.name ?? '', }, ) : ''; diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte index afe973e18f..752b257977 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte @@ -15,7 +15,7 @@
{$_('description')} onUpdate({ description })} isLongText {disabled} diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte index f78e0debe5..9311372cd7 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte @@ -19,7 +19,7 @@
{$_('name')} onUpdate({ name })} getValidationErrors={getNameValidationErrors} {disabled} From 76057f5f7fb9b437f33fc5c8de27cca2878633ad Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 19 Jul 2024 14:16:00 -0400 Subject: [PATCH 0459/1141] Rename some stores exported from tables.ts --- .../src/components/EditTableHOC.svelte | 9 ++------- .../breadcrumb/EntitySelector.svelte | 8 ++------ .../preview/ImportPreviewContent.svelte | 4 ++-- mathesar_ui/src/pages/record/Widgets.svelte | 4 ++-- .../src/pages/schema/ExplorationItem.svelte | 2 +- .../src/pages/schema/SchemaOverview.svelte | 2 +- .../src/pages/schema/SchemaPage.svelte | 2 +- .../src/routes/SharedTableRoute.svelte | 4 ++-- mathesar_ui/src/routes/TableRoute.svelte | 4 ++-- mathesar_ui/src/stores/tables.ts | 20 +++++++++++-------- .../systems/data-explorer/ActionsPane.svelte | 2 +- .../RecordSelectorContent.svelte | 4 ++-- .../RecordSelectorTable.svelte | 6 +++--- .../RecordSelectorWindow.svelte | 4 ++-- .../ForeignKeyConstraintDetails.svelte | 4 ++-- .../constraints/NewUniqueConstraint.svelte | 4 ++-- .../ColumnHeaderContextMenu.svelte | 4 ++-- .../link-table/LinkTableForm.svelte | 5 ++--- .../table-inspector/column/ColumnMode.svelte | 4 ++-- .../column/ColumnTypeSpecifierTag.svelte | 4 ++-- .../ExtractColumnsModal.svelte | 2 +- .../table-inspector/table/TableActions.svelte | 8 ++++++-- .../table/TableDescription.svelte | 5 +++-- .../table-inspector/table/TableName.svelte | 5 +++-- 24 files changed, 60 insertions(+), 60 deletions(-) diff --git a/mathesar_ui/src/components/EditTableHOC.svelte b/mathesar_ui/src/components/EditTableHOC.svelte index 6bfc59393a..2e399ab2e0 100644 --- a/mathesar_ui/src/components/EditTableHOC.svelte +++ b/mathesar_ui/src/components/EditTableHOC.svelte @@ -4,7 +4,7 @@ import { currentSchemaId } from '@mathesar/stores/schemas'; import { refetchTablesForSchema, - tables, + currentTables, updateTableMetaData, } from '@mathesar/stores/tables'; import type { AtLeastOne } from '@mathesar/typeUtils'; @@ -12,12 +12,7 @@ export let tableId: number; function schemaContainsTableName(name: string): boolean { - const allTables = [...$tables.tablesMap.values()]; - const tablesUsingName = allTables.filter( - (current) => current.name === name, - ); - - return tablesUsingName.length > 0 && tablesUsingName[0].oid !== tableId; + return $currentTables.some((t) => t.name === name && t.oid !== tableId); } function getNameValidationErrors(name: string): string[] { diff --git a/mathesar_ui/src/components/breadcrumb/EntitySelector.svelte b/mathesar_ui/src/components/breadcrumb/EntitySelector.svelte index 57e8e394f7..10b3855a7a 100644 --- a/mathesar_ui/src/components/breadcrumb/EntitySelector.svelte +++ b/mathesar_ui/src/components/breadcrumb/EntitySelector.svelte @@ -9,10 +9,7 @@ import { iconTable } from '@mathesar/icons'; import { getExplorationPageUrl } from '@mathesar/routes/urls'; import { queries as queriesStore } from '@mathesar/stores/queries'; - import { - currentTableId, - tables as tablesStore, - } from '@mathesar/stores/tables'; + import { currentTableId, currentTables } from '@mathesar/stores/tables'; import { getLinkForTableItem } from '@mathesar/utils/tables'; import BreadcrumbSelector from './BreadcrumbSelector.svelte'; @@ -63,11 +60,10 @@ }; } - $: tables = [...$tablesStore.tablesMap.values()]; $: queries = [...$queriesStore.data.values()]; $: selectorData = new Map([ - [$_('tables'), tables.map(makeTableBreadcrumbSelectorItem)], + [$_('tables'), $currentTables.map(makeTableBreadcrumbSelectorItem)], [$_('explorations'), queries.map(makeQueryBreadcrumbSelectorItem)], ]); diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte index 51dc936274..ea2138deeb 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte @@ -28,7 +28,7 @@ generateTablePreview, getTypeSuggestionsForTable, patchTable, - tables, + currentTables, } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; import { @@ -69,7 +69,7 @@ let columns: Column[] = []; let columnPropertiesMap = buildColumnPropertiesMap([]); - $: otherTableNames = [...$tables.tablesMap.values()] + $: otherTableNames = $currentTables .filter((t) => t.oid !== table.oid) .map((t) => t.name); $: customizedTableName = requiredField(table.name, [ diff --git a/mathesar_ui/src/pages/record/Widgets.svelte b/mathesar_ui/src/pages/record/Widgets.svelte index 8d75a77a0e..21f23fd65f 100644 --- a/mathesar_ui/src/pages/record/Widgets.svelte +++ b/mathesar_ui/src/pages/record/Widgets.svelte @@ -8,7 +8,7 @@ import NameWithIcon from '@mathesar/components/NameWithIcon.svelte'; import { RichText } from '@mathesar/components/rich-text'; import { iconRecord } from '@mathesar/icons'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; import { Help, isDefinedNonNullable } from '@mathesar-component-library'; import TableWidget from './TableWidget.svelte'; @@ -25,7 +25,7 @@ ); function buildWidgetInput(joinableTable: JoinableTable) { - const table = $tables.tablesMap.get(joinableTable.target); + const table = $currentTablesData.tablesMap.get(joinableTable.target); if (!table) return undefined; const id = joinableTable.jp_path[0].slice(-1)[0]; const name = columnNameMap.get(id) ?? `(${$_('unknown_column')})`; diff --git a/mathesar_ui/src/pages/schema/ExplorationItem.svelte b/mathesar_ui/src/pages/schema/ExplorationItem.svelte index 9076144df9..aa62e26d65 100644 --- a/mathesar_ui/src/pages/schema/ExplorationItem.svelte +++ b/mathesar_ui/src/pages/schema/ExplorationItem.svelte @@ -7,7 +7,7 @@ import TableName from '@mathesar/components/TableName.svelte'; import { iconExploration } from '@mathesar/icons'; import { getExplorationPageUrl } from '@mathesar/routes/urls'; - import { tables as tablesStore } from '@mathesar/stores/tables'; + import { currentTablesData as tablesStore } from '@mathesar/stores/tables'; import { Icon } from '@mathesar-component-library'; export let exploration: QueryInstance; diff --git a/mathesar_ui/src/pages/schema/SchemaOverview.svelte b/mathesar_ui/src/pages/schema/SchemaOverview.svelte index 9f41c1ceeb..8b5707c7dd 100644 --- a/mathesar_ui/src/pages/schema/SchemaOverview.svelte +++ b/mathesar_ui/src/pages/schema/SchemaOverview.svelte @@ -23,7 +23,7 @@ import TableSkeleton from './TableSkeleton.svelte'; import TablesList from './TablesList.svelte'; - export let tablesMap: Map; + export let tablesMap: Map; export let explorationsMap: Map; export let tablesRequestStatus: RequestStatus; export let explorationsRequestStatus: RequestStatus; diff --git a/mathesar_ui/src/pages/schema/SchemaPage.svelte b/mathesar_ui/src/pages/schema/SchemaPage.svelte index 8b73e92710..f327b12610 100644 --- a/mathesar_ui/src/pages/schema/SchemaPage.svelte +++ b/mathesar_ui/src/pages/schema/SchemaPage.svelte @@ -16,7 +16,7 @@ import { queries } from '@mathesar/stores/queries'; import { importVerifiedTables as importVerifiedTablesStore, - tables as tablesStore, + currentTablesData as tablesStore, } from '@mathesar/stores/tables'; import { getUserProfileStoreFromContext } from '@mathesar/stores/userProfile'; import { logEvent } from '@mathesar/utils/telemetry'; diff --git a/mathesar_ui/src/routes/SharedTableRoute.svelte b/mathesar_ui/src/routes/SharedTableRoute.svelte index 0d768185a5..4a62e73e67 100644 --- a/mathesar_ui/src/routes/SharedTableRoute.svelte +++ b/mathesar_ui/src/routes/SharedTableRoute.svelte @@ -5,7 +5,7 @@ import ErrorPage from '@mathesar/pages/ErrorPage.svelte'; import TablePage from '@mathesar/pages/table/TablePage.svelte'; - import { currentTableId, tables } from '@mathesar/stores/tables'; + import { currentTableId, currentTablesData } from '@mathesar/stores/tables'; import { preloadRouteData } from '@mathesar/utils/preloadData'; import { ShareConsumer } from '@mathesar/utils/shares'; @@ -17,7 +17,7 @@ $: tableId = routeSpecificData?.table_id ?? undefined; $: $currentTableId = tableId ?? undefined; - $: table = tableId ? $tables.tablesMap.get(tableId) : undefined; + $: table = tableId ? $currentTablesData.tablesMap.get(tableId) : undefined; $: shareConsumer = new ShareConsumer({ slug, }); diff --git a/mathesar_ui/src/routes/TableRoute.svelte b/mathesar_ui/src/routes/TableRoute.svelte index f243530cd6..0dcf247238 100644 --- a/mathesar_ui/src/routes/TableRoute.svelte +++ b/mathesar_ui/src/routes/TableRoute.svelte @@ -8,7 +8,7 @@ import AppendBreadcrumb from '@mathesar/components/breadcrumb/AppendBreadcrumb.svelte'; import ErrorPage from '@mathesar/pages/ErrorPage.svelte'; import TablePage from '@mathesar/pages/table/TablePage.svelte'; - import { currentTableId, tables } from '@mathesar/stores/tables'; + import { currentTableId, currentTablesData } from '@mathesar/stores/tables'; import RecordPageRoute from './RecordPageRoute.svelte'; @@ -17,7 +17,7 @@ export let tableId: number; $: $currentTableId = tableId; - $: table = $tables.tablesMap.get(tableId); + $: table = $currentTablesData.tablesMap.get(tableId); function handleUnmount() { $currentTableId = undefined; diff --git a/mathesar_ui/src/stores/tables.ts b/mathesar_ui/src/stores/tables.ts index 7dd4f6f62c..8ee392ad53 100644 --- a/mathesar_ui/src/stores/tables.ts +++ b/mathesar_ui/src/stores/tables.ts @@ -395,7 +395,7 @@ export function generateTablePreview(props: { return postAPI(`/api/db/v0/tables/${table.oid}/previews/`, { columns }); } -export const tables: Readable = derived( +export const currentTablesData: Readable = derived( currentSchemaId, ($currentSchemaId, set) => { let unsubscribe: Unsubscriber; @@ -418,18 +418,22 @@ export const tables: Readable = derived( }, ); +export const currentTables = derived(currentTablesData, (tablesData) => + sortTables(tablesData.tablesMap.values()), +); + export const importVerifiedTables: Readable = derived( - tables, - ($tables) => + currentTablesData, + (tablesData) => new Map( - [...$tables.tablesMap.values()] + [...tablesData.tablesMap.values()] .filter((table) => !isTableImportConfirmationRequired(table)) .map((table) => [table.oid, table]), ), ); -export const validateNewTableName = derived(tables, ($tables) => { - const names = new Set([...$tables.tablesMap.values()].map((t) => t.name)); +export const validateNewTableName = derived(currentTablesData, (tablesData) => { + const names = new Set([...tablesData.tablesMap.values()].map((t) => t.name)); return invalidIf( (name: string) => names.has(name), 'A table with that name already exists.', @@ -437,13 +441,13 @@ export const validateNewTableName = derived(tables, ($tables) => { }); export function getTableName(id: DBObjectEntry['id']): string | undefined { - return get(tables).tablesMap.get(id)?.name; + return get(currentTablesData).tablesMap.get(id)?.name; } export const currentTableId = writable(undefined); export const currentTable = derived( - [currentTableId, tables], + [currentTableId, currentTablesData], ([$currentTableId, $tables]) => $currentTableId === undefined ? undefined diff --git a/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte b/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte index 585e5bcc20..955f9586c4 100644 --- a/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte +++ b/mathesar_ui/src/systems/data-explorer/ActionsPane.svelte @@ -16,7 +16,7 @@ } from '@mathesar/icons'; import { modal } from '@mathesar/stores/modal'; import { queries } from '@mathesar/stores/queries'; - import { tables as tablesDataStore } from '@mathesar/stores/tables'; + import { currentTablesData as tablesDataStore } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; import { Button, diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte index 05276014b0..4607a5c238 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte @@ -15,7 +15,7 @@ renderTransitiveRecordSummary, } from '@mathesar/stores/table-data/record-summaries/recordSummaryUtils'; import { getPkValueInRecord } from '@mathesar/stores/table-data/records'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; import { getUserProfileStoreFromContext } from '@mathesar/stores/userProfile'; import { getErrorMessage } from '@mathesar/utils/errors'; @@ -88,7 +88,7 @@ const record = response.results[0]; const recordId = getPkValueInRecord(record, $columns); const previewData = response.preview_data ?? []; - const table = $tables.tablesMap.get(tableId); + const table = $currentTablesData.tablesMap.get(tableId); const template = table?.settings?.preview_settings?.template; if (!template) { throw new Error('No record summary template found in API response.'); diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte index 9a9bf877f1..e9f8986b65 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte @@ -15,7 +15,7 @@ renderTransitiveRecordSummary, } from '@mathesar/stores/table-data/record-summaries/recordSummaryUtils'; import { getPkValueInRecord } from '@mathesar/stores/table-data/records'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; import overflowObserver, { makeOverflowDetails, } from '@mathesar/utils/overflowObserver'; @@ -56,7 +56,7 @@ recordsData, processedColumns, } = tabularData); - $: table = $tables.tablesMap.get(tableId); + $: table = $currentTablesData.tablesMap.get(tableId); $: ({ recordSummaries, state: recordsDataState } = recordsData); $: recordsDataIsLoading = $recordsDataState === States.Loading; $: ({ constraints } = $constraintsDataStore); @@ -137,7 +137,7 @@ if (!record || recordId === undefined) { return; } - const tableEntry = $tables.tablesMap.get(tableId); + const tableEntry = $currentTablesData.tablesMap.get(tableId); const template = tableEntry?.settings?.preview_settings?.template ?? ''; const recordSummary = renderTransitiveRecordSummary({ template, diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte index 05c9c5c425..0b115103c0 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte @@ -6,7 +6,7 @@ import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; import { Meta, TabularData } from '@mathesar/stores/table-data'; import { getTableName } from '@mathesar/stores/tables'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; import Pagination from '@mathesar/utils/Pagination'; import { Window, portal } from '@mathesar-component-library'; @@ -31,7 +31,7 @@ nestingLevel: controller.nestingLevel + 1, }); $: ({ tableId, purpose } = controller); - $: table = $tableId && $tables.tablesMap.get($tableId); + $: table = $tableId && $currentTablesData.tablesMap.get($tableId); $: tabularData = $tableId && table ? new TabularData({ diff --git a/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte b/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte index 03fee0ab5a..bf0566c542 100644 --- a/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte @@ -8,13 +8,13 @@ import ColumnName from '@mathesar/components/column/ColumnName.svelte'; import TableName from '@mathesar/components/TableName.svelte'; import type { Constraint } from '@mathesar/stores/table-data'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; export let constraint: Constraint; $: referentTable = constraint.type === 'foreignkey' - ? $tables.tablesMap.get(constraint.referent_table) + ? $currentTablesData.tablesMap.get(constraint.referent_table) : undefined; async function getReferentColumns(_constraint: Constraint) { diff --git a/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte b/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte index 9c6f952ac8..07a369b0e0 100644 --- a/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte @@ -8,7 +8,7 @@ type ProcessedColumn, getTabularDataStoreFromContext, } from '@mathesar/stores/table-data'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; import { getColumnConstraintTypeByColumnId } from '@mathesar/utils/columnUtils'; import { getAvailableName } from '@mathesar/utils/db'; @@ -75,7 +75,7 @@ $: existingConstraintNames = new Set( $constraintsDataStore.constraints.map((c) => c.name), ); - $: tableName = $tables.tablesMap.get($tabularData.id)?.name ?? ''; + $: tableName = $currentTablesData.tablesMap.get($tabularData.id)?.name ?? ''; $: ({ processedColumns } = $tabularData); $: columnsInTable = Array.from($processedColumns.values()); $: nameValidationErrors = getNameValidationErrors( diff --git a/mathesar_ui/src/systems/table-view/header/header-cell/ColumnHeaderContextMenu.svelte b/mathesar_ui/src/systems/table-view/header/header-cell/ColumnHeaderContextMenu.svelte index 0110e13aba..afb73a4c21 100644 --- a/mathesar_ui/src/systems/table-view/header/header-cell/ColumnHeaderContextMenu.svelte +++ b/mathesar_ui/src/systems/table-view/header/header-cell/ColumnHeaderContextMenu.svelte @@ -23,7 +23,7 @@ type ProcessedColumn, getTabularDataStoreFromContext, } from '@mathesar/stores/table-data'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; import { getUserProfileStoreFromContext } from '@mathesar/stores/userProfile'; import { ButtonMenuItem, LinkMenuItem } from '@mathesar-component-library'; @@ -61,7 +61,7 @@ $: ({ linkFk } = processedColumn); $: linkedTable = linkFk - ? $tables.tablesMap.get(linkFk.referent_table) + ? $currentTablesData.tablesMap.get(linkFk.referent_table) : undefined; $: linkedTableHref = linkedTable ? $storeToGetTablePageUrl({ tableId: linkedTable.oid }) diff --git a/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte b/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte index dcd0fef696..7c3e457b68 100644 --- a/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte +++ b/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte @@ -26,7 +26,7 @@ import { importVerifiedTables as importVerifiedTablesStore, refetchTablesForSchema, - tables as tablesDataStore, + currentTables, validateNewTableName, } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; @@ -61,7 +61,6 @@ // =========================================================================== $: singularBaseTableName = makeSingular(base.name); $: importVerifiedTables = [...$importVerifiedTablesStore.values()]; - $: allTables = [...$tablesDataStore.tablesMap.values()]; $: ({ columnsDataStore } = $tabularData); $: baseColumns = columnsDataStore.columns; @@ -93,7 +92,7 @@ [columnNameIsAvailable($targetColumns)], ); $: mappingTableName = requiredField( - suggestMappingTableName(base, target, allTables), + suggestMappingTableName(base, target, $currentTables), [$validateNewTableName], ); $: columnNameMappingToBase = (() => { diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte index 7815072839..b254a43c5d 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte @@ -4,7 +4,7 @@ import { currentDatabase } from '@mathesar/stores/databases'; import { currentSchema } from '@mathesar/stores/schemas'; import { getTabularDataStoreFromContext } from '@mathesar/stores/table-data'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; import { getUserProfileStoreFromContext } from '@mathesar/stores/userProfile'; import FkRecordSummaryConfig from '@mathesar/systems/table-view/table-inspector/record-summary/FkRecordSummaryConfig.svelte'; import { Collapsible } from '@mathesar-component-library'; @@ -145,7 +145,7 @@ {@const referentTable = referentTableId === undefined ? undefined - : $tables.tablesMap.get(referentTableId)} + : $currentTablesData.tablesMap.get(referentTableId)} {#if referentTable !== undefined} diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte index f1395b917c..49f8299144 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte @@ -24,7 +24,7 @@ getTableFromStoreOrApi, moveColumns, splitTable, - tables as tablesDataStore, + currentTablesData as tablesDataStore, validateNewTableName, } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte index 2092698daf..75b222ad5a 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableActions.svelte @@ -8,7 +8,11 @@ import { currentDatabase } from '@mathesar/stores/databases'; import { currentSchema } from '@mathesar/stores/schemas'; import { getTabularDataStoreFromContext } from '@mathesar/stores/table-data'; - import { currentTable, deleteTable, tables } from '@mathesar/stores/tables'; + import { + currentTable, + deleteTable, + currentTablesData, + } from '@mathesar/stores/tables'; import { constructDataExplorerUrlToSummarizeFromGroup, createDataExplorerUrlToExploreATable, @@ -37,7 +41,7 @@ $currentSchema.oid, { oid: $tabularData.id, - name: $tables.tablesMap.get($tabularData.id)?.name ?? '', + name: $currentTablesData.tablesMap.get($tabularData.id)?.name ?? '', }, ) : ''; diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte index 752b257977..80d6f20d14 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte @@ -4,7 +4,7 @@ import EditableTextWithActions from '@mathesar/components/EditableTextWithActions.svelte'; import EditTableHOC from '@mathesar/components/EditTableHOC.svelte'; import { getTabularDataStoreFromContext } from '@mathesar/stores/table-data'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; const tabularData = getTabularDataStoreFromContext(); @@ -15,7 +15,8 @@
{$_('description')} onUpdate({ description })} isLongText {disabled} diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte index 9311372cd7..2a0d4d5b27 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte @@ -4,7 +4,7 @@ import EditableTextWithActions from '@mathesar/components/EditableTextWithActions.svelte'; import EditTableHOC from '@mathesar/components/EditTableHOC.svelte'; import { getTabularDataStoreFromContext } from '@mathesar/stores/table-data'; - import { tables } from '@mathesar/stores/tables'; + import { currentTablesData } from '@mathesar/stores/tables'; const tabularData = getTabularDataStoreFromContext(); @@ -19,7 +19,8 @@
{$_('name')} onUpdate({ name })} getValidationErrors={getNameValidationErrors} {disabled} From b5170e30c18997785e2f0c69f57dfd2cbefa99ac Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 19 Jul 2024 14:16:00 -0400 Subject: [PATCH 0460/1141] Add connection params to tables functions --- mathesar_ui/src/api/rpc/index.ts | 2 + .../src/components/EditTableHOC.svelte | 14 +- .../src/pages/database/DatabaseDetails.svelte | 2 +- .../preview/ImportPreviewContent.svelte | 44 ++- .../src/pages/schema/SchemaOverview.svelte | 2 +- mathesar_ui/src/stores/tables.ts | 254 +++++++++--------- .../src/systems/data-explorer/QueryManager.ts | 16 +- .../RecordSelectorWindow.svelte | 12 +- .../systems/table-view/header/Header.svelte | 11 +- .../link-table/LinkTableForm.svelte | 7 +- .../record-summary/RecordSummaryConfig.svelte | 5 +- 11 files changed, 200 insertions(+), 169 deletions(-) diff --git a/mathesar_ui/src/api/rpc/index.ts b/mathesar_ui/src/api/rpc/index.ts index e9a3b3c531..ed58604e71 100644 --- a/mathesar_ui/src/api/rpc/index.ts +++ b/mathesar_ui/src/api/rpc/index.ts @@ -4,6 +4,7 @@ import { buildRpcApi } from '@mathesar/packages/json-rpc-client-builder'; import { connections } from './connections'; import { schemas } from './schemas'; +import { tables } from './tables'; /** Mathesar's JSON-RPC API */ export const api = buildRpcApi({ @@ -12,5 +13,6 @@ export const api = buildRpcApi({ methodTree: { connections, schemas, + tables, }, }); diff --git a/mathesar_ui/src/components/EditTableHOC.svelte b/mathesar_ui/src/components/EditTableHOC.svelte index 2e399ab2e0..1c08470b15 100644 --- a/mathesar_ui/src/components/EditTableHOC.svelte +++ b/mathesar_ui/src/components/EditTableHOC.svelte @@ -1,11 +1,9 @@ diff --git a/mathesar_ui/src/pages/database/DatabaseDetails.svelte b/mathesar_ui/src/pages/database/DatabaseDetails.svelte index 4e2a8e6d02..0bfab64b20 100644 --- a/mathesar_ui/src/pages/database/DatabaseDetails.svelte +++ b/mathesar_ui/src/pages/database/DatabaseDetails.svelte @@ -102,7 +102,7 @@ onProceed: async () => { await deleteSchemaAPI(database.id, schema.oid); // TODO: Create common util to handle data clearing & sync between stores - removeTablesStore(schema.oid); + removeTablesStore(database, schema); }, }); } diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte index ea2138deeb..44b9a56d7c 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte @@ -2,6 +2,11 @@ import { _ } from 'svelte-i18n'; import { router } from 'tinro'; + import { + CancellablePromise, + CancelOrProceedButtonPair, + Spinner, + } from '@mathesar-component-library'; import { columnsApi } from '@mathesar/api/rest/columns'; import type { DataFile } from '@mathesar/api/rest/types/dataFiles'; import type { Column } from '@mathesar/api/rest/types/tables/columns'; @@ -24,21 +29,13 @@ } from '@mathesar/routes/urls'; import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; import AsyncStore from '@mathesar/stores/AsyncStore'; - import { - generateTablePreview, - getTypeSuggestionsForTable, - patchTable, - currentTables, - } from '@mathesar/stores/tables'; + import { currentTables, patchTable } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; - import { - CancelOrProceedButtonPair, - Spinner, - } from '@mathesar-component-library'; import ColumnNamingStrategyInput from '../column-names/ColumnNamingStrategyInput.svelte'; import ColumnTypeInferenceInput from '../inference/ColumnTypeInferenceInput.svelte'; + import { getAPI, postAPI } from '@mathesar/api/rest/utils/requestUtils'; import ErrorInfo from './ErrorInfo.svelte'; import ImportPreviewLayout from './ImportPreviewLayout.svelte'; import { @@ -54,6 +51,31 @@ /** Set via back-end */ const TRUNCATION_LIMIT = 20; + /** + * Note the optimizing query parameter. It asserts that the table will not have + * columns with default values (probably because it is currently being imported + * and a table produced by importing will not have column defaults). It + * follows, that this function cannot be used where columns might have defaults. + */ + function getTypeSuggestionsForTable( + id: Table['oid'], + ): CancellablePromise> { + const optimizingQueryParam = 'columns_might_have_defaults=false'; + return getAPI( + `/api/db/v0/tables/${id}/type_suggestions/?${optimizingQueryParam}`, + ); + } + + function generateTablePreview(props: { + table: Pick; + columns: Table['columns']; + }): CancellablePromise<{ + records: Record[]; + }> { + const { columns, table } = props; + return postAPI(`/api/db/v0/tables/${table.oid}/previews/`, { columns }); + } + const columnsFetch = new AsyncStore(columnsApi.list); const previewRequest = new AsyncStore(generateTablePreview); const typeSuggestionsRequest = new AsyncStore(getTypeSuggestionsForTable); @@ -152,7 +174,7 @@ async function finishImport() { try { - await patchTable(table.oid, { + await patchTable(database, table.oid, { name: $customizedTableName, import_verified: true, columns: finalizeColumns(columns, columnPropertiesMap), diff --git a/mathesar_ui/src/pages/schema/SchemaOverview.svelte b/mathesar_ui/src/pages/schema/SchemaOverview.svelte index 8b5707c7dd..e6ada4b4b3 100644 --- a/mathesar_ui/src/pages/schema/SchemaOverview.svelte +++ b/mathesar_ui/src/pages/schema/SchemaOverview.svelte @@ -61,7 +61,7 @@
{ - await refetchTablesForSchema(schema.oid); + await refetchTablesForSchema(database, schema); }} label={$_('retry')} icon={iconRefresh} diff --git a/mathesar_ui/src/stores/tables.ts b/mathesar_ui/src/stores/tables.ts index 8ee392ad53..fe09a100cd 100644 --- a/mathesar_ui/src/stores/tables.ts +++ b/mathesar_ui/src/stores/tables.ts @@ -1,30 +1,29 @@ /** * @file * - * TODO This file **badly** needs to be refactored, cleaned up, and made to - * function more consistently with the rest of the codebase. + * TODO This file should ideally be refactored, cleaned up, and made to function + * more consistently with the rest of the codebase. * - * - For values of type `Writable`, we seem to be using using - * names like `schemaStore`, `tableStore`, `tablesStore`, `schemaTablesStore` - * almost interchangeably which is a readability nightmare. - * - * - Tables need to be sorted before being stored, but that sorting happens in - * many different places. I suggest having a derived store that does the - * sorting. + * Also, tables need to be sorted before being stored, but that sorting happens + * in many different places. I suggest having a derived store that does the + * sorting. */ -import type { Readable, Unsubscriber, Writable } from 'svelte/store'; -import { derived, get, writable } from 'svelte/store'; +import type { Readable, Writable } from 'svelte/store'; +import { derived, get, readable, writable } from 'svelte/store'; +import { + CancellablePromise, + collapse, + type RecursivePartial, +} from '@mathesar-component-library'; +import type { DBObjectEntry } from '@mathesar/AppTypes'; import type { JoinableTablesResult } from '@mathesar/api/rest/types/tables/joinable_tables'; import type { SplitTableRequest, SplitTableResponse, } from '@mathesar/api/rest/types/tables/split_table'; -import type { - PaginatedResponse, - RequestStatus, -} from '@mathesar/api/rest/utils/requestUtils'; +import type { RequestStatus } from '@mathesar/api/rest/utils/requestUtils'; import { deleteAPI, getAPI, @@ -33,17 +32,16 @@ import { } from '@mathesar/api/rest/utils/requestUtils'; import type { Schema } from '@mathesar/api/rpc/schemas'; import type { Table } from '@mathesar/api/rpc/tables'; -import type { DBObjectEntry, Database } from '@mathesar/AppTypes'; import { invalidIf } from '@mathesar/components/form'; import type { AtLeastOne } from '@mathesar/typeUtils'; import { preloadCommonData } from '@mathesar/utils/preloadData'; import { isTableImportConfirmationRequired } from '@mathesar/utils/tables'; -import { - CancellablePromise, - type RecursivePartial, -} from '@mathesar-component-library'; +import type { Connection } from '@mathesar/api/rpc/connections'; +import { TupleMap } from '@mathesar/packages/tuple-map'; +import { connectionsStore } from './databases'; import { addCountToSchemaNumTables, currentSchemaId } from './schemas'; +import { api } from '@mathesar/api/rpc'; const commonData = preloadCommonData(); @@ -54,21 +52,33 @@ interface TablesData { requestStatus: RequestStatus; } +function makeEmptyTablesData(): TablesData { + return { + tablesMap: new Map(), + requestStatus: { state: 'success' }, + }; +} + type TablesStore = Writable; -const tablesStores: Map = new Map(); +/** Maps [connectionId, schemaOid] to TablesStore */ +const tablesStores = new TupleMap< + [Connection['id'], Schema['oid']], + TablesStore +>(); -const tablesRequests: Map< - Schema['oid'], +const tablesRequests = new TupleMap< + [Connection['id'], Schema['oid']], CancellablePromise -> = new Map(); +>(); function sortTables(tables: Iterable
): Table[] { return [...tables].sort((a, b) => a.name.localeCompare(b.name)); } function setTablesStore( - schemaOid: Schema['oid'], + connection: Pick, + schema: Pick, tables?: Table[], ): TablesStore { const tablesMap: TablesMap = new Map(); @@ -81,26 +91,33 @@ function setTablesStore( requestStatus: { state: 'success' }, }; - let store = tablesStores.get(schemaOid); + let store = tablesStores.get([connection.id, schema.oid]); if (!store) { store = writable(storeValue); - tablesStores.set(schemaOid, store); + tablesStores.set([connection.id, schema.oid], store); } else { store.set(storeValue); } return store; } -export function removeTablesStore(schemaOid: Schema['oid']): void { - tablesStores.delete(schemaOid); +export function removeTablesStore( + connection: Pick, + schema: Pick, +): void { + tablesStores.delete([connection.id, schema.oid]); } export async function refetchTablesForSchema( - schemaOid: Schema['oid'], + connection: Pick, + schema: Pick, ): Promise { - const store = tablesStores.get(schemaOid); + const store = tablesStores.get([connection.id, schema.oid]); if (!store) { - console.error(`Tables store for schema: ${schemaOid} not found.`); + // TODO: why are we logging an error here? I would expect that we'd either + // throw or ignore. If there's a reason for this logging, please add a code + // comment explaining why. + console.error(`Tables store not found.`); return undefined; } @@ -110,16 +127,15 @@ export async function refetchTablesForSchema( requestStatus: { state: 'processing' }, })); - tablesRequests.get(schemaOid)?.cancel(); + tablesRequests.get([connection.id, schema.oid])?.cancel(); - const tablesRequest = getAPI>( - `/api/db/v0/tables/?schema=${schemaOid}&limit=500`, + const tablesRequest = getAPI( + `/api/db/v0/tables/?schema=${schema.oid}&limit=500`, ); - tablesRequests.set(schemaOid, tablesRequest); - const response = await tablesRequest; - const tableEntries = response.results || []; + tablesRequests.set([connection.id, schema.oid], tablesRequest); + const tableEntries = await tablesRequest; - const schemaTablesStore = setTablesStore(schemaOid, tableEntries); + const schemaTablesStore = setTablesStore(connection, schema, tableEntries); return get(schemaTablesStore); } catch (err) { @@ -138,22 +154,26 @@ export async function refetchTablesForSchema( let preload = true; -function getTablesStore(schemaOid: Schema['oid']): TablesStore { - let store = tablesStores.get(schemaOid); +function getTablesStore( + connection: Pick, + schema: Pick, +): TablesStore { + let store = tablesStores.get([connection.id, schema.oid]); if (!store) { store = writable({ requestStatus: { state: 'processing' }, tablesMap: new Map(), }); - tablesStores.set(schemaOid, store); - if (preload && commonData.current_schema === schemaOid) { - store = setTablesStore(schemaOid, commonData.tables ?? []); + tablesStores.set([connection.id, schema.oid], store); + // TODO_3651: add condition for current connection as well as current schema + if (preload && commonData.current_schema === schema.oid) { + store = setTablesStore(connection, schema, commonData.tables ?? []); } else { - void refetchTablesForSchema(schemaOid); + void refetchTablesForSchema(connection, schema); } preload = false; } else if (get(store).requestStatus.state === 'failure') { - void refetchTablesForSchema(schemaOid); + void refetchTablesForSchema(connection, schema); } return store; } @@ -161,7 +181,7 @@ function getTablesStore(schemaOid: Schema['oid']): TablesStore { function findSchemaStoreForTable( tableOid: Table['oid'], ): TablesStore | undefined { - // TODO rewrite this function + // TODO_3651 rewrite this function throw new Error('Not implemented'); } @@ -178,7 +198,7 @@ function findAndUpdateTableStore(tableOid: Table['oid'], newTable: Table) { } export function deleteTable( - database: Database, + connection: Connection, schema: Schema, tableOid: Table['oid'], ): CancellablePromise
{ @@ -186,14 +206,16 @@ export function deleteTable( return new CancellablePromise( (resolve, reject) => { void promise.then((table) => { - addCountToSchemaNumTables(database, schema, -1); - tablesStores.get(schema.oid)?.update((tableStoreData) => { - tableStoreData.tablesMap.delete(tableOid); - return { - ...tableStoreData, - tablesMap: new Map(tableStoreData.tablesMap), - }; - }); + addCountToSchemaNumTables(connection, schema, -1); + tablesStores + .get([connection.id, schema.oid]) + ?.update((tableStoreData) => { + tableStoreData.tablesMap.delete(tableOid); + return { + ...tableStoreData, + tablesMap: new Map(tableStoreData.tablesMap), + }; + }); return resolve(table); }, reject); }, @@ -203,7 +225,10 @@ export function deleteTable( ); } +// TODO_3651: This is not actually updating metadata. Merge this function with +// patchTable below. Separate actual metadata into a different function export function updateTableMetaData( + connection: Pick, tableOid: number, updatedMetaData: AtLeastOne<{ name: string; description: string }>, ): CancellablePromise
{ @@ -225,7 +250,7 @@ export function updateTableMetaData( } export function createTable( - database: Database, + connection: Connection, schema: Schema, tableArgs: { name?: string; @@ -240,8 +265,8 @@ export function createTable( return new CancellablePromise( (resolve, reject) => { void promise.then((table) => { - addCountToSchemaNumTables(database, schema, 1); - tablesStores.get(schema.oid)?.update((existing) => { + addCountToSchemaNumTables(connection, schema, 1); + tablesStores.get([connection.id, schema.oid])?.update((existing) => { const tablesMap: TablesMap = new Map(); sortTables([...existing.tablesMap.values(), table]).forEach( (entry) => { @@ -260,6 +285,7 @@ export function createTable( } export function patchTable( + connection: Pick, tableOid: Table['oid'], patch: { name?: Table['name']; @@ -281,23 +307,6 @@ export function patchTable( ); } -/** - * NOTE: The getTable function currently does not get data from the store. - * We need to keep it that way for the time-being, because the components - * that call this function expect latest data from the db, while the store - * contains stale information. - * - * TODO: - * 1. Keep stores upto-date when user performs any action to related db objects. - * 2. Find a sync mechanism to keep the frontend stores upto-date when - * data in db changes. - * 3. Move all api-call-only functions to /api. Only keep functions that - * update the stores within /stores - */ -export function getTable(tableOid: Table['oid']): CancellablePromise
{ - return getAPI(`/api/db/v0/tables/${tableOid}/`); -} - export function splitTable({ id, idsOfColumnsToExtract, @@ -328,10 +337,8 @@ export function moveColumns( }); } -/** - * Replace getTable with this function once the above mentioned changes are done. - */ export function getTableFromStoreOrApi( + connection: Pick, tableOid: Table['oid'], ): CancellablePromise
{ const tablesStore = findSchemaStoreForTable(tableOid); @@ -343,11 +350,16 @@ export function getTableFromStoreOrApi( }); } } - const promise = getTable(tableOid); + const promise = api.tables + .get({ + database_id: connection.id, + table_oid: tableOid, + }) + .run(); return new CancellablePromise( (resolve, reject) => { void promise.then((table) => { - const store = tablesStores.get(table.schema); + const store = tablesStores.get([connection.id, table.schema]); if (store) { store.update((existing) => { const tableMap = new Map(); @@ -370,52 +382,19 @@ export function getTableFromStoreOrApi( ); } -/** - * Note the optimizing query parameter. It asserts that the table will not have - * columns with default values (probably because it is currently being imported - * and a table produced by importing will not have column defaults). It - * follows, that this function cannot be used where columns might have defaults. - */ -export function getTypeSuggestionsForTable( - id: Table['oid'], -): CancellablePromise> { - const optimizingQueryParam = 'columns_might_have_defaults=false'; - return getAPI( - `/api/db/v0/tables/${id}/type_suggestions/?${optimizingQueryParam}`, - ); -} - -export function generateTablePreview(props: { - table: Pick; - columns: Table['columns']; -}): CancellablePromise<{ - records: Record[]; -}> { - const { columns, table } = props; - return postAPI(`/api/db/v0/tables/${table.oid}/previews/`, { columns }); -} - -export const currentTablesData: Readable = derived( - currentSchemaId, - ($currentSchemaId, set) => { - let unsubscribe: Unsubscriber; - - if (!$currentSchemaId) { - set({ - tablesMap: new Map(), - requestStatus: { state: 'success' }, - }); - } else { - const store = getTablesStore($currentSchemaId); - unsubscribe = store.subscribe((dbSchemasData) => { - set(dbSchemasData); - }); - } +export const currentTablesData = collapse( + derived( + [connectionsStore.currentConnection, currentSchemaId], + ([connection, schemaOid]) => + !connection || !schemaOid + ? readable(makeEmptyTablesData()) + : getTablesStore(connection, { oid: schemaOid }), + ), +); - return () => { - unsubscribe?.(); - }; - }, +export const currentTablesMap = derived( + currentTablesData, + (tablesData) => tablesData.tablesMap, ); export const currentTables = derived(currentTablesData, (tablesData) => @@ -440,10 +419,6 @@ export const validateNewTableName = derived(currentTablesData, (tablesData) => { ); }); -export function getTableName(id: DBObjectEntry['id']): string | undefined { - return get(currentTablesData).tablesMap.get(id)?.name; -} - export const currentTableId = writable(undefined); export const currentTable = derived( @@ -462,31 +437,42 @@ export function getJoinableTablesResult(tableId: number, maxDepth = 1) { type TableSettings = Table['settings']; -export async function saveTableSettings( +async function saveTableSettings( + connection: Pick, table: Pick, settings: RecursivePartial, ): Promise { const url = `/api/db/v0/tables/${table.oid}/settings/${table.settings.id}/`; await patchAPI(url, settings); - await refetchTablesForSchema(table.schema); + await refetchTablesForSchema(connection, { oid: table.schema }); } export function saveRecordSummaryTemplate( + connection: Pick, table: Pick, previewSettings: TableSettings['preview_settings'], ): Promise { const { customized } = previewSettings; - return saveTableSettings(table, { + return saveTableSettings(connection, table, { preview_settings: customized ? previewSettings : { customized }, }); } export function saveColumnOrder( + connection: Pick, table: Pick, columnOrder: TableSettings['column_order'], ): Promise { - return saveTableSettings(table, { + return saveTableSettings(connection, table, { // Using the Set constructor to remove potential duplicates column_order: [...new Set(columnOrder)], }); } + +export async function refetchTablesForCurrentSchema() { + const connection = get(connectionsStore.currentConnection); + const schemaOid = get(currentSchemaId); + if (connection && schemaOid) { + await refetchTablesForSchema(connection, { oid: schemaOid }); + } +} diff --git a/mathesar_ui/src/systems/data-explorer/QueryManager.ts b/mathesar_ui/src/systems/data-explorer/QueryManager.ts index 6a5aaece66..af169c114e 100644 --- a/mathesar_ui/src/systems/data-explorer/QueryManager.ts +++ b/mathesar_ui/src/systems/data-explorer/QueryManager.ts @@ -12,7 +12,6 @@ import { getAPI } from '@mathesar/api/rest/utils/requestUtils'; import type { Table } from '@mathesar/api/rpc/tables'; import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; import { createQuery, putQuery } from '@mathesar/stores/queries'; -import { getTable } from '@mathesar/stores/tables'; import CacheManager from '@mathesar/utils/CacheManager'; import type { CancellablePromise } from '@mathesar-component-library'; @@ -27,6 +26,8 @@ import { getColumnInformationMap, getTablesThatReferenceBaseTable, } from './utils'; +import { api } from '@mathesar/api/rpc'; +import { connectionsStore } from '@mathesar/stores/databases'; export default class QueryManager extends QueryRunner { private undoRedoManager: QueryUndoRedoManager; @@ -135,7 +136,18 @@ export default class QueryManager extends QueryRunner { inputColumnsFetchState: { state: 'processing' }, })); - this.baseTableFetchPromise = getTable(baseTableId); + const currentConnection = get(connectionsStore.currentConnection); + if (!currentConnection) { + throw new Error('No current connection selected.'); + } + + this.baseTableFetchPromise = api.tables + .get({ + database_id: currentConnection.id, + table_oid: baseTableId, + }) + .run(); + this.joinableColumnsfetchPromise = getAPI( `/api/db/v0/tables/${baseTableId}/joinable_tables/`, ); diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte index 0b115103c0..2471426b66 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte @@ -5,10 +5,9 @@ import TableName from '@mathesar/components/TableName.svelte'; import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; import { Meta, TabularData } from '@mathesar/stores/table-data'; - import { getTableName } from '@mathesar/stores/tables'; - import { currentTablesData } from '@mathesar/stores/tables'; + import { currentTablesMap } from '@mathesar/stores/tables'; import Pagination from '@mathesar/utils/Pagination'; - import { Window, portal } from '@mathesar-component-library'; + import { Window, defined, portal } from '@mathesar-component-library'; import RecordSelectorContent from './RecordSelectorContent.svelte'; import { RecordSelectorController } from './RecordSelectorController'; @@ -31,7 +30,7 @@ nestingLevel: controller.nestingLevel + 1, }); $: ({ tableId, purpose } = controller); - $: table = $tableId && $currentTablesData.tablesMap.get($tableId); + $: table = defined($tableId, (id) => $currentTablesMap.get(id)); $: tabularData = $tableId && table ? new TabularData({ @@ -42,7 +41,6 @@ table, }) : undefined; - $: tableName = $tableId ? getTableName($tableId) : undefined; $: nestedSelectorIsOpen = nestedController.isOpen; $: marginBottom = $nestedSelectorIsOpen ? `calc(${nestedSelectorVerticalOffset} - ${contentHeight}px)` @@ -96,8 +94,8 @@ : $_('open_table_record')} let:slotName > - {#if slotName === 'tableName' && tableName} - + {#if slotName === 'tableName' && table} + {/if} diff --git a/mathesar_ui/src/systems/table-view/header/Header.svelte b/mathesar_ui/src/systems/table-view/header/Header.svelte index 706280a516..9ad8424603 100644 --- a/mathesar_ui/src/systems/table-view/header/Header.svelte +++ b/mathesar_ui/src/systems/table-view/header/Header.svelte @@ -22,6 +22,7 @@ import ColumnHeaderContextMenu from './header-cell/ColumnHeaderContextMenu.svelte'; import HeaderCell from './header-cell/HeaderCell.svelte'; import NewColumnCell from './new-column-cell/NewColumnCell.svelte'; + import { connectionsStore } from '@mathesar/stores/databases'; const tabularData = getTabularDataStoreFromContext(); @@ -32,6 +33,7 @@ $: columnOrder = columnOrder ?? []; $: columnOrderString = columnOrder.map(String); $: ({ selection, processedColumns } = $tabularData); + $: ({ currentConnectionId } = connectionsStore); let locationOfFirstDraggedColumn: number | undefined = undefined; let selectedColumnIdsOrdered: string[] = []; @@ -61,6 +63,9 @@ } function dropColumn(columnDroppedOn?: ProcessedColumn) { + const connectionId = $currentConnectionId; + if (!connectionId) throw new Error('No current connection ID'); + // Early exit if a column is dropped in the same place. // Should only be done for single column if non-continuous selection is allowed. if ( @@ -87,7 +92,11 @@ newColumnOrder.splice(0, 0, ...selectedColumnIdsOrdered); } - void saveColumnOrder(table, newColumnOrder.map(Number)); + void saveColumnOrder( + { id: connectionId }, + table, + newColumnOrder.map(Number), + ); // Reset drag information locationOfFirstDraggedColumn = undefined; diff --git a/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte b/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte index 7c3e457b68..bf2326e8d1 100644 --- a/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte +++ b/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte @@ -18,14 +18,13 @@ import { RichText } from '@mathesar/components/rich-text'; import SelectTable from '@mathesar/components/SelectTable.svelte'; import { iconTableLink } from '@mathesar/icons'; - import { currentSchemaId } from '@mathesar/stores/schemas'; import { ColumnsDataStore, getTabularDataStoreFromContext, } from '@mathesar/stores/table-data'; import { importVerifiedTables as importVerifiedTablesStore, - refetchTablesForSchema, + refetchTablesForCurrentSchema, currentTables, validateNewTableName, } from '@mathesar/stores/tables'; @@ -198,8 +197,8 @@ } async function reFetchOtherThingsThatChanged() { - if ($linkType === 'manyToMany' && $currentSchemaId !== undefined) { - await refetchTablesForSchema($currentSchemaId); + if ($linkType === 'manyToMany') { + await refetchTablesForCurrentSchema(); return; } const tableWithNewColumn = $linkType === 'oneToMany' ? target : base; diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/RecordSummaryConfig.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/RecordSummaryConfig.svelte index ac6ce4b4b2..13608f4e95 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/RecordSummaryConfig.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/RecordSummaryConfig.svelte @@ -73,7 +73,10 @@ async function save() { try { - await saveRecordSummaryTemplate(table, $form.values); + if (!database) { + throw new Error('Current database not found'); + } + await saveRecordSummaryTemplate(database, table, $form.values); } catch (e) { toast.error(`${$_('unable_to_save_changes')} ${getErrorMessage(e)}`); } From 8d2a524917958b05703c4d386711f3ff0450b744 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 19 Jul 2024 14:16:01 -0400 Subject: [PATCH 0461/1141] Minor cleanup --- mathesar_ui/src/stores/tables.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/mathesar_ui/src/stores/tables.ts b/mathesar_ui/src/stores/tables.ts index fe09a100cd..ba0a9f5d7a 100644 --- a/mathesar_ui/src/stores/tables.ts +++ b/mathesar_ui/src/stores/tables.ts @@ -17,7 +17,6 @@ import { collapse, type RecursivePartial, } from '@mathesar-component-library'; -import type { DBObjectEntry } from '@mathesar/AppTypes'; import type { JoinableTablesResult } from '@mathesar/api/rest/types/tables/joinable_tables'; import type { SplitTableRequest, @@ -435,22 +434,20 @@ export function getJoinableTablesResult(tableId: number, maxDepth = 1) { ); } -type TableSettings = Table['settings']; - async function saveTableSettings( connection: Pick, table: Pick, - settings: RecursivePartial, + settings: RecursivePartial, ): Promise { const url = `/api/db/v0/tables/${table.oid}/settings/${table.settings.id}/`; - await patchAPI(url, settings); + await patchAPI(url, settings); await refetchTablesForSchema(connection, { oid: table.schema }); } export function saveRecordSummaryTemplate( connection: Pick, table: Pick, - previewSettings: TableSettings['preview_settings'], + previewSettings: Table['settings']['preview_settings'], ): Promise { const { customized } = previewSettings; return saveTableSettings(connection, table, { @@ -461,7 +458,7 @@ export function saveRecordSummaryTemplate( export function saveColumnOrder( connection: Pick, table: Pick, - columnOrder: TableSettings['column_order'], + columnOrder: Table['settings']['column_order'], ): Promise { return saveTableSettings(connection, table, { // Using the Set constructor to remove potential duplicates From 8be0d60aaee08bdc0f8ea267417bca64b8bc5c85 Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 19 Jul 2024 14:16:01 -0400 Subject: [PATCH 0462/1141] Update tables API types with new metadata fields --- mathesar_ui/src/api/rpc/tables.ts | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/mathesar_ui/src/api/rpc/tables.ts b/mathesar_ui/src/api/rpc/tables.ts index 2d45f10cc3..1684d3a2ab 100644 --- a/mathesar_ui/src/api/rpc/tables.ts +++ b/mathesar_ui/src/api/rpc/tables.ts @@ -1,6 +1,6 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; -export interface Table { +export interface RawTable { oid: number; name: string; /** The OID of the schema containing the table */ @@ -8,8 +8,27 @@ export interface Table { description: string | null; } +interface TableMetadata { + import_verified: boolean | null; + column_order: number[] | null; + record_summary_customized: boolean | null; + record_summary_template: string | null; +} + +export interface Table extends RawTable { + metadata: TableMetadata; +} + export const tables = { list: rpcMethodTypeContainer< + { + database_id: number; + schema_oid: number; + }, + RawTable[] + >(), + + list_with_metadata: rpcMethodTypeContainer< { database_id: number; schema_oid: number; @@ -22,7 +41,7 @@ export const tables = { database_id: number; table_oid: number; }, - Table + RawTable >(), /** Returns the oid of the table created */ From 6ff2b633f53b5f3b30fabd60259f7affe2dfbb1f Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Fri, 19 Jul 2024 14:16:01 -0400 Subject: [PATCH 0463/1141] Adjust code to match Table type with metadata --- mathesar_ui/src/api/rpc/tables.ts | 8 ++ .../src/components/EditTableHOC.svelte | 38 ----- mathesar_ui/src/components/TableName.svelte | 7 +- .../preview/ImportPreviewContent.svelte | 27 ++-- .../import/preview/ImportPreviewPage.svelte | 26 ++-- .../import/preview/importPreviewPageUtils.ts | 2 +- mathesar_ui/src/pages/record/RecordStore.ts | 12 +- mathesar_ui/src/pages/schema/EditTable.svelte | 53 ++++--- mathesar_ui/src/stores/databases.ts | 14 ++ mathesar_ui/src/stores/tables.ts | 131 +++++++----------- .../src/systems/data-explorer/QueryManager.ts | 2 +- .../src/systems/data-explorer/utils.ts | 70 ++++++---- .../RecordSelectorContent.svelte | 9 +- .../RecordSelectorTable.svelte | 10 +- .../record-selector/recordSelectorUtils.ts | 10 +- .../src/systems/table-view/TableView.svelte | 5 +- .../systems/table-view/header/Header.svelte | 43 +++--- .../ExtractColumnsModal.svelte | 8 +- .../record-summary/RecordSummaryConfig.svelte | 14 +- .../table/TableDescription.svelte | 33 +++-- .../table-inspector/table/TableName.svelte | 42 +++--- mathesar_ui/src/utils/tables.ts | 66 ++++----- 22 files changed, 336 insertions(+), 294 deletions(-) delete mode 100644 mathesar_ui/src/components/EditTableHOC.svelte diff --git a/mathesar_ui/src/api/rpc/tables.ts b/mathesar_ui/src/api/rpc/tables.ts index 1684d3a2ab..2b9b66d2cd 100644 --- a/mathesar_ui/src/api/rpc/tables.ts +++ b/mathesar_ui/src/api/rpc/tables.ts @@ -44,6 +44,14 @@ export const tables = { RawTable >(), + get_with_metadata: rpcMethodTypeContainer< + { + database_id: number; + table_oid: number; + }, + Table + >(), + /** Returns the oid of the table created */ add: rpcMethodTypeContainer< { diff --git a/mathesar_ui/src/components/EditTableHOC.svelte b/mathesar_ui/src/components/EditTableHOC.svelte deleted file mode 100644 index 1c08470b15..0000000000 --- a/mathesar_ui/src/components/EditTableHOC.svelte +++ /dev/null @@ -1,38 +0,0 @@ - - - diff --git a/mathesar_ui/src/components/TableName.svelte b/mathesar_ui/src/components/TableName.svelte index cbf6258239..72f64baee3 100644 --- a/mathesar_ui/src/components/TableName.svelte +++ b/mathesar_ui/src/components/TableName.svelte @@ -8,11 +8,8 @@ import NameWithIcon from './NameWithIcon.svelte'; interface $$Props extends Omit, 'icon'> { - table: { - name: Table['name']; - data_files?: Table['data_files']; - import_verified?: Table['import_verified']; - }; + table: Pick & + Parameters[0]; isLoading?: boolean; } diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte index 44b9a56d7c..f0c32b7fe8 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte @@ -29,7 +29,7 @@ } from '@mathesar/routes/urls'; import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; import AsyncStore from '@mathesar/stores/AsyncStore'; - import { currentTables, patchTable } from '@mathesar/stores/tables'; + import { currentTables } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; import ColumnNamingStrategyInput from '../column-names/ColumnNamingStrategyInput.svelte'; @@ -68,7 +68,7 @@ function generateTablePreview(props: { table: Pick; - columns: Table['columns']; + columns: Column[]; }): CancellablePromise<{ records: Record[]; }> { @@ -173,16 +173,19 @@ } async function finishImport() { - try { - await patchTable(database, table.oid, { - name: $customizedTableName, - import_verified: true, - columns: finalizeColumns(columns, columnPropertiesMap), - }); - router.goto(getTablePageUrl(database.id, schema.oid, table.oid), true); - } catch (err) { - toast.fromError(err); - } + // TODO reimplement patching tables columns with RPC API. + throw new Error('Not implemented'); + + // try { + // await patchTable(database, table.oid, { + // name: $customizedTableName, + // import_verified: true, + // columns: finalizeColumns(columns, columnPropertiesMap), + // }); + // router.goto(getTablePageUrl(database.id, schema.oid, table.oid), true); + // } catch (err) { + // toast.fromError(err); + // } } diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewPage.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewPage.svelte index 775b6d0712..3b8641bd19 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewPage.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewPage.svelte @@ -15,6 +15,7 @@ import ImportPreviewContent from './ImportPreviewContent.svelte'; import ImportPreviewLayout from './ImportPreviewLayout.svelte'; + import { currentConnection } from '@mathesar/stores/databases'; const tableFetch = new AsyncStore(getTableFromStoreOrApi); const dataFileFetch = new AsyncStore(dataFilesApi.get); @@ -29,20 +30,29 @@ } $: void (async () => { - const table = (await tableFetch.run(tableId)).resolvedValue; + const table = ( + await tableFetch.run({ + connection: $currentConnection, + tableOid: tableId, + }) + ).resolvedValue; if (!table) { return; } - if (table.import_verified) { + if (table.metadata.import_verified) { redirectToTablePage(); return; } - const firstDataFileId = table.data_files?.[0]; - if (firstDataFileId === undefined) { - redirectToTablePage(); - return; - } - await dataFileFetch.run(firstDataFileId); + + // TODO_BETA: re-implement fetching and storing of `table.data_files` + // metadata from RPC API or similar. + throw new Error('Not implemented'); + // const firstDataFileId = table.data_files?.[0]; + // if (firstDataFileId === undefined) { + // redirectToTablePage(); + // return; + // } + // await dataFileFetch.run(firstDataFileId); })(); $: error = $tableFetch.error ?? $dataFileFetch.error; diff --git a/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts b/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts index c9aa18e830..e76f4bbd62 100644 --- a/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts +++ b/mathesar_ui/src/pages/import/preview/importPreviewPageUtils.ts @@ -101,7 +101,7 @@ export function buildColumnPropertiesMap( export function finalizeColumns( columns: Column[], columnPropertiesMap: ColumnPropertiesMap, -): Table['columns'] { +) { return columns .filter((column) => columnPropertiesMap[column.id]?.selected) .map((column) => ({ diff --git a/mathesar_ui/src/pages/record/RecordStore.ts b/mathesar_ui/src/pages/record/RecordStore.ts index b49f2d3bb6..188a0a522d 100644 --- a/mathesar_ui/src/pages/record/RecordStore.ts +++ b/mathesar_ui/src/pages/record/RecordStore.ts @@ -36,7 +36,17 @@ export default class RecordStore { this.table = table; this.recordPk = recordPk; this.url = `/api/db/v0/tables/${this.table.oid}/records/${this.recordPk}/`; - const { template } = this.table.settings.preview_settings; + // TODO_RS_TEMPLATE + // + // We need to handle the case where no record summary template is set. + // Previously it was the responsibility of the service layer to _always_ + // return a record summary template, even by deriving one on the fly to send + // if necessary. With the changes for beta, it will be the responsibility of + // the client to handle the case where no template is set. We need to wait + // until after the service layer changes are made before we can implement + // this here. + const template = + this.table.metadata.record_summary_template ?? 'TODO_RS_TEMPLATE'; this.summary = derived( [this.fieldValues, this.recordSummaries], ([fields, fkSummaryData]) => diff --git a/mathesar_ui/src/pages/schema/EditTable.svelte b/mathesar_ui/src/pages/schema/EditTable.svelte index 39ba72b6bb..fc3d80d5ca 100644 --- a/mathesar_ui/src/pages/schema/EditTable.svelte +++ b/mathesar_ui/src/pages/schema/EditTable.svelte @@ -1,31 +1,46 @@ - - onUpdate({ name, description })} - {getNameValidationErrors} - getInitialName={() => table.name ?? ''} - getInitialDescription={() => table.description ?? ''} - > - - - {#if slotName === 'tableName'} - {initialName} - {/if} - - - - + table.name ?? ''} + getInitialDescription={() => table.description ?? ''} +> + + + {#if slotName === 'tableName'} + {initialName} + {/if} + + + diff --git a/mathesar_ui/src/stores/databases.ts b/mathesar_ui/src/stores/databases.ts index 24c98cf5d2..3a59fa0936 100644 --- a/mathesar_ui/src/stores/databases.ts +++ b/mathesar_ui/src/stores/databases.ts @@ -129,5 +129,19 @@ class ConnectionsStore { export const connectionsStore: MakeWritablePropertiesReadable = new ConnectionsStore(); +/** + * @throws an error when used in a context where no current connection exists. + * This behavior sacrifices some stability for the sake of developer ergonomics. + * This sacrifice seems acceptable given that such a large part of the + * application depends on the existence of one and only one connection. + */ +export const currentConnection = derived( + connectionsStore.currentConnection, + (c) => { + if (!c) throw new Error('No current connection'); + return c; + }, +); + /** @deprecated Use connectionsStore.currentConnection instead */ export const currentDatabase = connectionsStore.currentConnection; diff --git a/mathesar_ui/src/stores/tables.ts b/mathesar_ui/src/stores/tables.ts index ba0a9f5d7a..8878269ac9 100644 --- a/mathesar_ui/src/stores/tables.ts +++ b/mathesar_ui/src/stores/tables.ts @@ -32,7 +32,6 @@ import { import type { Schema } from '@mathesar/api/rpc/schemas'; import type { Table } from '@mathesar/api/rpc/tables'; import { invalidIf } from '@mathesar/components/form'; -import type { AtLeastOne } from '@mathesar/typeUtils'; import { preloadCommonData } from '@mathesar/utils/preloadData'; import { isTableImportConfirmationRequired } from '@mathesar/utils/tables'; @@ -41,6 +40,8 @@ import { TupleMap } from '@mathesar/packages/tuple-map'; import { connectionsStore } from './databases'; import { addCountToSchemaNumTables, currentSchemaId } from './schemas'; import { api } from '@mathesar/api/rpc'; +import { execPipe, filter, map } from 'iter-tools'; +import { _ } from 'svelte-i18n'; const commonData = preloadCommonData(); @@ -224,28 +225,12 @@ export function deleteTable( ); } -// TODO_3651: This is not actually updating metadata. Merge this function with -// patchTable below. Separate actual metadata into a different function -export function updateTableMetaData( +export async function updateTable( connection: Pick, - tableOid: number, - updatedMetaData: AtLeastOne<{ name: string; description: string }>, -): CancellablePromise
{ - const promise = patchAPI
( - `/api/db/v0/tables/${tableOid}/`, - updatedMetaData, - ); - return new CancellablePromise( - (resolve, reject) => { - void promise.then((table) => { - findAndUpdateTableStore(tableOid, table); - return resolve(table); - }, reject); - }, - () => { - promise.cancel(); - }, - ); + table: RecursivePartial
& { oid: Table['oid'] }, +) { + // TODO_3651 + throw new Error('Not implemented'); } export function createTable( @@ -283,29 +268,6 @@ export function createTable( ); } -export function patchTable( - connection: Pick, - tableOid: Table['oid'], - patch: { - name?: Table['name']; - import_verified?: Table['import_verified']; - columns?: Table['columns']; - }, -): CancellablePromise
{ - const promise = patchAPI
(`/api/db/v0/tables/${tableOid}/`, patch); - return new CancellablePromise( - (resolve, reject) => { - void promise.then((value) => { - findAndUpdateTableStore(tableOid, value); - return resolve(value); - }, reject); - }, - () => { - promise.cancel(); - }, - ); -} - export function splitTable({ id, idsOfColumnsToExtract, @@ -336,10 +298,13 @@ export function moveColumns( }); } -export function getTableFromStoreOrApi( - connection: Pick, - tableOid: Table['oid'], -): CancellablePromise
{ +export function getTableFromStoreOrApi({ + connection, + tableOid, +}: { + connection: Pick; + tableOid: Table['oid']; +}): CancellablePromise
{ const tablesStore = findSchemaStoreForTable(tableOid); if (tablesStore) { const table = get(tablesStore).tablesMap.get(tableOid); @@ -350,7 +315,7 @@ export function getTableFromStoreOrApi( } } const promise = api.tables - .get({ + .get_with_metadata({ database_id: connection.id, table_oid: tableOid, }) @@ -434,38 +399,6 @@ export function getJoinableTablesResult(tableId: number, maxDepth = 1) { ); } -async function saveTableSettings( - connection: Pick, - table: Pick, - settings: RecursivePartial, -): Promise { - const url = `/api/db/v0/tables/${table.oid}/settings/${table.settings.id}/`; - await patchAPI(url, settings); - await refetchTablesForSchema(connection, { oid: table.schema }); -} - -export function saveRecordSummaryTemplate( - connection: Pick, - table: Pick, - previewSettings: Table['settings']['preview_settings'], -): Promise { - const { customized } = previewSettings; - return saveTableSettings(connection, table, { - preview_settings: customized ? previewSettings : { customized }, - }); -} - -export function saveColumnOrder( - connection: Pick, - table: Pick, - columnOrder: Table['settings']['column_order'], -): Promise { - return saveTableSettings(connection, table, { - // Using the Set constructor to remove potential duplicates - column_order: [...new Set(columnOrder)], - }); -} - export async function refetchTablesForCurrentSchema() { const connection = get(connectionsStore.currentConnection); const schemaOid = get(currentSchemaId); @@ -473,3 +406,37 @@ export async function refetchTablesForCurrentSchema() { await refetchTablesForSchema(connection, { oid: schemaOid }); } } + +export function factoryToGetTableNameValidationErrors( + connection: Pick, + table: Table, +): Readable<(n: string) => string[]> { + const tablesStore = tablesStores.get([connection.id, table.schema]); + if (!tablesStore) throw new Error('Tables store not found'); + + const otherTableNames = derived( + tablesStore, + (d) => + new Set( + execPipe( + d.tablesMap.values(), + filter((t) => t.oid !== table.oid), + map((t) => t.name), + ), + ), + ); + + return derived([otherTableNames, _], ([$otherTableNames, $_]) => { + function getNameValidationErrors(name: string): string[] { + if (!name.trim()) { + return [$_('table_name_cannot_be_empty')]; + } + if ($otherTableNames.has(name)) { + return [$_('table_with_name_already_exists')]; + } + return []; + } + + return getNameValidationErrors; + }); +} diff --git a/mathesar_ui/src/systems/data-explorer/QueryManager.ts b/mathesar_ui/src/systems/data-explorer/QueryManager.ts index af169c114e..37209feaa2 100644 --- a/mathesar_ui/src/systems/data-explorer/QueryManager.ts +++ b/mathesar_ui/src/systems/data-explorer/QueryManager.ts @@ -142,7 +142,7 @@ export default class QueryManager extends QueryRunner { } this.baseTableFetchPromise = api.tables - .get({ + .get_with_metadata({ database_id: currentConnection.id, table_oid: baseTableId, }) diff --git a/mathesar_ui/src/systems/data-explorer/utils.ts b/mathesar_ui/src/systems/data-explorer/utils.ts index 5899b87f68..7a3c77f1fd 100644 --- a/mathesar_ui/src/systems/data-explorer/utils.ts +++ b/mathesar_ui/src/systems/data-explorer/utils.ts @@ -181,19 +181,23 @@ export function getLinkFromColumn( export function getColumnInformationMap( result: JoinableTablesResult, - baseTable: Pick, + baseTable: Pick, ): InputColumnsStoreSubstance['inputColumnInformationMap'] { const map: InputColumnsStoreSubstance['inputColumnInformationMap'] = new Map(); - baseTable.columns.forEach((column) => { - map.set(column.id, { - id: column.id, - name: column.name, - type: column.type, - tableId: baseTable.oid, - tableName: baseTable.name, - }); - }); + + // TODO_BETA: figure out how to deal with the fact that our `Table` type no + // longer has a `columns` field. + + // baseTable.columns.forEach((column) => { + // map.set(column.id, { + // id: column.id, + // name: column.name, + // type: column.type, + // tableId: baseTable.oid, + // tableName: baseTable.name, + // }); + // }); Object.keys(result.tables).forEach((tableIdKey) => { const tableId = parseInt(tableIdKey, 10); const table = result.tables[tableId]; @@ -213,26 +217,32 @@ export function getColumnInformationMap( export function getBaseTableColumnsWithLinks( result: JoinableTablesResult, - baseTable: Pick, + baseTable: Pick, ): Map { - const columnMapEntries: [ColumnWithLink['id'], ColumnWithLink][] = - baseTable.columns.map((column) => [ - column.id, - { - id: column.id, - name: column.name, - type: column.type, - tableName: baseTable.name, - linksTo: getLinkFromColumn(result, column.id, 1), - producesMultipleResults: false, - }, - ]); + // TODO_BETA: figure out how to deal with the fact that our `Table` type no + // longer has a `columns` field. + + // const columnMapEntries: [ColumnWithLink['id'], ColumnWithLink][] = + // baseTable.columns.map((column) => [ + // column.id, + // { + // id: column.id, + // name: column.name, + // type: column.type, + // tableName: baseTable.name, + // linksTo: getLinkFromColumn(result, column.id, 1), + // producesMultipleResults: false, + // }, + // ]); + + const columnMapEntries: [ColumnWithLink['id'], ColumnWithLink][] = []; + return new Map(columnMapEntries.sort(compareColumnByLinks)); } export function getTablesThatReferenceBaseTable( result: JoinableTablesResult, - baseTable: Pick, + baseTable: Pick, ): ReferencedByTable[] { const referenceLinks = result.joinable_tables.filter( (entry) => entry.depth === 1 && entry.fk_path[0][1] === true, @@ -243,10 +253,16 @@ export function getTablesThatReferenceBaseTable( const tableId = reference.target; const table = result.tables[tableId]; const baseTableColumnId = reference.jp_path[0][0]; - const baseTableColumn = baseTable.columns.find( - (column) => column.id === baseTableColumnId, - ); const referenceTableColumnId = reference.jp_path[0][1]; + + // TODO_BETA: figure out how to deal with the fact that our `Table` type no + // longer has a `columns` field. + + // const baseTableColumn = baseTable.columns.find( + // (column) => column.id === baseTableColumnId, + // ); + const baseTableColumn = undefined; + if (!baseTableColumn) { return; } diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte index 4607a5c238..5bf3143624 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte @@ -89,9 +89,14 @@ const recordId = getPkValueInRecord(record, $columns); const previewData = response.preview_data ?? []; const table = $currentTablesData.tablesMap.get(tableId); - const template = table?.settings?.preview_settings?.template; + const template = table?.metadata?.record_summary_template; + // TODO_RS_TEMPLATE + // + // We need to change the logic here to account for the fact that sometimes + // the record summary template actually _will_ be missing. We need to + // handle this on the client. if (!template) { - throw new Error('No record summary template found in API response.'); + throw new Error('TODO_RS_TEMPLATE'); } const recordSummary = renderTransitiveRecordSummary({ inputData: buildInputData(record), diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte index e9f8986b65..cf9a36368d 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte @@ -138,7 +138,15 @@ return; } const tableEntry = $currentTablesData.tablesMap.get(tableId); - const template = tableEntry?.settings?.preview_settings?.template ?? ''; + const template = table?.metadata?.record_summary_template; + if (!template) { + throw new Error('TODO_RS_TEMPLATE'); + // TODO_RS_TEMPLATE + // + // We need to change the logic here to account for the fact that sometimes + // the record summary template actually _will_ be missing. We need to + // handle this on the client. + } const recordSummary = renderTransitiveRecordSummary({ template, inputData: buildInputData(record), diff --git a/mathesar_ui/src/systems/record-selector/recordSelectorUtils.ts b/mathesar_ui/src/systems/record-selector/recordSelectorUtils.ts index f39c9972b7..8b2d4cc4e4 100644 --- a/mathesar_ui/src/systems/record-selector/recordSelectorUtils.ts +++ b/mathesar_ui/src/systems/record-selector/recordSelectorUtils.ts @@ -25,7 +25,15 @@ export function getColumnIdToFocusInitially({ if (!table) { return undefined; } - const { template } = table.settings.preview_settings; + const template = table?.metadata?.record_summary_template; + if (!template) { + throw new Error('TODO_RS_TEMPLATE'); + // TODO_RS_TEMPLATE + // + // We need to change the logic here to account for the fact that sometimes + // the record summary template actually _will_ be missing. We need to + // handle this on the client. + } const match = template.match(/\{\d+\}/)?.[0] ?? undefined; if (!match) { return undefined; diff --git a/mathesar_ui/src/systems/table-view/TableView.svelte b/mathesar_ui/src/systems/table-view/TableView.svelte index 5885287a5f..81030731f5 100644 --- a/mathesar_ui/src/systems/table-view/TableView.svelte +++ b/mathesar_ui/src/systems/table-view/TableView.svelte @@ -37,7 +37,7 @@ ); export let context: Context = 'page'; - export let table: Pick; + export let table: Table; export let sheetElement: HTMLElement | undefined = undefined; let tableInspectorTab: ComponentProps['activeTabId'] = @@ -62,8 +62,7 @@ }); $: ({ horizontalScrollOffset, scrollOffset, isTableInspectorVisible } = display); - $: ({ settings } = table); - $: ({ column_order: columnOrder } = settings); + $: columnOrder = table.metadata.column_order ?? []; $: hasNewColumnButton = allowsDdlOperations; /** * These are separate variables for readability and also to keep the door open diff --git a/mathesar_ui/src/systems/table-view/header/Header.svelte b/mathesar_ui/src/systems/table-view/header/Header.svelte index 9ad8424603..9a2042a9bd 100644 --- a/mathesar_ui/src/systems/table-view/header/Header.svelte +++ b/mathesar_ui/src/systems/table-view/header/Header.svelte @@ -16,45 +16,42 @@ ID_ROW_CONTROL_COLUMN, getTabularDataStoreFromContext, } from '@mathesar/stores/table-data'; - import { saveColumnOrder } from '@mathesar/stores/tables'; + import { updateTable } from '@mathesar/stores/tables'; import { Draggable, Droppable } from './drag-and-drop'; import ColumnHeaderContextMenu from './header-cell/ColumnHeaderContextMenu.svelte'; import HeaderCell from './header-cell/HeaderCell.svelte'; import NewColumnCell from './new-column-cell/NewColumnCell.svelte'; - import { connectionsStore } from '@mathesar/stores/databases'; + import { currentConnection } from '@mathesar/stores/databases'; const tabularData = getTabularDataStoreFromContext(); export let hasNewColumnButton = false; export let columnOrder: number[]; - export let table: Pick; + export let table: Pick; $: columnOrder = columnOrder ?? []; - $: columnOrderString = columnOrder.map(String); $: ({ selection, processedColumns } = $tabularData); - $: ({ currentConnectionId } = connectionsStore); let locationOfFirstDraggedColumn: number | undefined = undefined; - let selectedColumnIdsOrdered: string[] = []; - let newColumnOrder: string[] = []; + let selectedColumnIdsOrdered: number[] = []; + let newColumnOrder: number[] = []; function dragColumn() { // Keep only IDs for which the column exists for (const columnId of $processedColumns.keys()) { - const columnIdString = columnId.toString(); - columnOrderString = [...new Set(columnOrderString)]; - if (!columnOrderString.includes(columnIdString)) { - columnOrderString = [...columnOrderString, columnIdString]; + columnOrder = [...new Set(columnOrder)]; + if (!columnOrder.includes(columnId)) { + columnOrder = [...columnOrder, columnId]; } } - columnOrderString = columnOrderString; + columnOrder = columnOrder; // Remove selected column IDs and keep their order - for (const id of columnOrderString) { - if ($selection.columnIds.has(id)) { + for (const id of columnOrder) { + if ($selection.columnIds.has(String(id))) { selectedColumnIdsOrdered.push(id); if (!locationOfFirstDraggedColumn) { - locationOfFirstDraggedColumn = columnOrderString.indexOf(id); + locationOfFirstDraggedColumn = columnOrder.indexOf(id); } } else { newColumnOrder.push(id); @@ -63,9 +60,6 @@ } function dropColumn(columnDroppedOn?: ProcessedColumn) { - const connectionId = $currentConnectionId; - if (!connectionId) throw new Error('No current connection ID'); - // Early exit if a column is dropped in the same place. // Should only be done for single column if non-continuous selection is allowed. if ( @@ -83,7 +77,7 @@ // if that column is to the right, else insert it before if (columnDroppedOn) { newColumnOrder.splice( - columnOrderString.indexOf(columnDroppedOn.id.toString()), + columnOrder.indexOf(columnDroppedOn.id), 0, ...selectedColumnIdsOrdered, ); @@ -92,11 +86,10 @@ newColumnOrder.splice(0, 0, ...selectedColumnIdsOrdered); } - void saveColumnOrder( - { id: connectionId }, - table, - newColumnOrder.map(Number), - ); + void updateTable($currentConnection, { + oid: table.oid, + metadata: { column_order: newColumnOrder }, + }); // Reset drag information locationOfFirstDraggedColumn = undefined; @@ -127,7 +120,7 @@ on:drop={() => dropColumn(processedColumn)} on:dragover={(e) => e.preventDefault()} {locationOfFirstDraggedColumn} - columnLocation={columnOrderString.indexOf(columnId.toString())} + columnLocation={columnOrder.indexOf(columnId)} {isSelected} > diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte index 49f8299144..bff522a810 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte @@ -43,6 +43,7 @@ import type { ExtractColumnsModalController } from './ExtractColumnsModalController'; import SelectLinkedTable from './SelectLinkedTable.svelte'; import SuccessToastContent from './SuccessToastContent.svelte'; + import { currentConnection } from '@mathesar/stores/databases'; const tabularData = getTabularDataStoreFromContext(); @@ -164,7 +165,12 @@ extractedTableName: newTableName, newFkColumnName: $newFkColumnName, }); - followUps.push(getTableFromStoreOrApi(response.extracted_table)); + followUps.push( + getTableFromStoreOrApi({ + connection: $currentConnection, + tableOid: response.extracted_table, + }), + ); followUps.push( $tabularData.refreshAfterColumnExtraction( extractedColumnIds, diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/RecordSummaryConfig.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/RecordSummaryConfig.svelte index 13608f4e95..77d3ef30a4 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/RecordSummaryConfig.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/RecordSummaryConfig.svelte @@ -17,7 +17,7 @@ import { currentSchema } from '@mathesar/stores/schemas'; import type { RecordRow, TabularData } from '@mathesar/stores/table-data'; import { renderRecordSummaryForRow } from '@mathesar/stores/table-data/record-summaries/recordSummaryUtils'; - import { saveRecordSummaryTemplate } from '@mathesar/stores/tables'; + import { updateTable } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; import { getUserProfileStoreFromContext } from '@mathesar/stores/userProfile'; import { getErrorMessage } from '@mathesar/utils/errors'; @@ -41,8 +41,8 @@ $: ({ columns } = columnsDataStore); $: ({ savedRecords, recordSummaries } = recordsData); $: firstRow = $savedRecords[0] as RecordRow | undefined; - $: initialCustomized = table.settings.preview_settings.customized ?? false; - $: initialTemplate = table.settings.preview_settings.template ?? ''; + $: initialCustomized = table.metadata.record_summary_customized ?? false; + $: initialTemplate = table.metadata.record_summary_template ?? ''; $: customized = requiredField(initialCustomized); $: customizedDisabled = customized.disabled; $: template = optionalField(initialTemplate, [hasColumnReferences($columns)]); @@ -76,7 +76,13 @@ if (!database) { throw new Error('Current database not found'); } - await saveRecordSummaryTemplate(database, table, $form.values); + await updateTable(database, { + oid: table.oid, + metadata: { + record_summary_customized: $customized, + record_summary_template: $template, + }, + }); } catch (e) { toast.error(`${$_('unable_to_save_changes')} ${getErrorMessage(e)}`); } diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte index 80d6f20d14..55873eb052 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte @@ -2,27 +2,32 @@ import { _ } from 'svelte-i18n'; import EditableTextWithActions from '@mathesar/components/EditableTextWithActions.svelte'; - import EditTableHOC from '@mathesar/components/EditTableHOC.svelte'; import { getTabularDataStoreFromContext } from '@mathesar/stores/table-data'; - import { currentTablesData } from '@mathesar/stores/tables'; + import { currentTablesData, updateTable } from '@mathesar/stores/tables'; + import { currentConnection } from '@mathesar/stores/databases'; const tabularData = getTabularDataStoreFromContext(); export let disabled = false; + + async function handleSave(description: string) { + updateTable($currentConnection, { + oid: $tabularData.table.oid, + description, + }); + } - -
- {$_('description')} - onUpdate({ description })} - isLongText - {disabled} - /> -
-
+
+ {$_('description')} + +
diff --git a/mathesar_ui/src/systems/connections/AddConnectionModal.svelte b/mathesar_ui/src/systems/connections/AddConnectionModal.svelte deleted file mode 100644 index dc52dfada4..0000000000 --- a/mathesar_ui/src/systems/connections/AddConnectionModal.svelte +++ /dev/null @@ -1,33 +0,0 @@ - - - - - diff --git a/mathesar_ui/src/systems/connections/DeleteConnectionModal.svelte b/mathesar_ui/src/systems/connections/DeleteConnectionModal.svelte deleted file mode 100644 index d158ddf381..0000000000 --- a/mathesar_ui/src/systems/connections/DeleteConnectionModal.svelte +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - {#if slotName === 'connectionName'} - {connection.nickname} - {/if} - - - -

{$_('action_cannot_be_undone')}

-

- {$_('delete_connection_info')} -

-

- {$_('delete_connection_db_delete_info')} -

- -

- - - -

- - - - - -
diff --git a/mathesar_ui/src/systems/connections/EditConnectionModal.svelte b/mathesar_ui/src/systems/connections/EditConnectionModal.svelte deleted file mode 100644 index 375ac95020..0000000000 --- a/mathesar_ui/src/systems/connections/EditConnectionModal.svelte +++ /dev/null @@ -1,198 +0,0 @@ - - - form.reset()}> - - - {#if slotName === 'connectionName'} - {connection.nickname} - {/if} - - - -
- -
- - -
-
- -
-
- -
-
-
- - -
- - -
- - - - - - - - - - - - {#if $changePassword} - - {/if} -
- -
- form.reset()} - onProceed={save} - /> -
- - - diff --git a/mathesar_ui/src/systems/connections/GeneralConnection.svelte b/mathesar_ui/src/systems/connections/GeneralConnection.svelte deleted file mode 100644 index 1b6ae557b2..0000000000 --- a/mathesar_ui/src/systems/connections/GeneralConnection.svelte +++ /dev/null @@ -1,43 +0,0 @@ - - - - - {#if generalConnection.type === 'user_database'} - {generalConnection.connection.nickname} - {:else} - ({$_('internal_database')}) - {/if} - - - {user}@{host}:{port}/{database} - - - - diff --git a/mathesar_ui/src/systems/connections/generalConnections.ts b/mathesar_ui/src/systems/connections/generalConnections.ts deleted file mode 100644 index b33e2d7767..0000000000 --- a/mathesar_ui/src/systems/connections/generalConnections.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { map } from 'iter-tools'; -import { type Readable, derived } from 'svelte/store'; - -import type { - Connection, - ConnectionReference, -} from '@mathesar/api/rest/connections'; -import { connectionsStore } from '@mathesar/stores/databases'; -import { - type CommonData, - preloadCommonData, -} from '@mathesar/utils/preloadData'; - -interface UserDatabaseConnection { - type: 'user_database'; - connection: Connection; -} -interface InternalDatabaseConnection { - type: 'internal_database'; - connection: CommonData['internal_db_connection']; -} - -export type GeneralConnection = - | UserDatabaseConnection - | InternalDatabaseConnection; - -function isUserDatabaseConnection( - connection: GeneralConnection, -): connection is UserDatabaseConnection { - return connection.type === 'user_database'; -} - -function getCommonDataGeneralConnections(): GeneralConnection[] { - const commonData = preloadCommonData(); - if (!commonData) return []; - return [ - { - type: 'internal_database', - connection: commonData.internal_db_connection, - }, - ]; -} - -export const generalConnections: Readable = derived( - connectionsStore.connections, - (connections) => [ - ...getCommonDataGeneralConnections(), - ...map( - ([, connection]) => ({ type: 'user_database' as const, connection }), - connections, - ), - ], -); - -export function pickDefaultGeneralConnection(connections: GeneralConnection[]) { - if (connections.length === 0) return undefined; - const internalConnection = connections.find( - (connection) => connection.type === 'internal_database', - ); - if (internalConnection) return internalConnection; - return connections - .filter(isUserDatabaseConnection) - .reduce((a, b) => (a.connection.id > b.connection.id ? a : b)); -} - -export function getUsername({ connection }: GeneralConnection): string { - return 'user' in connection ? connection.user : connection.username; -} - -export function getConnectionReference( - connection: GeneralConnection, -): ConnectionReference { - return connection.type === 'internal_database' - ? { connection_type: 'internal_database' } - : { connection_type: 'user_database', id: connection.connection.id }; -} From b32870003c5bd646c551e92f5819ee1c6ea354ee Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 25 Jul 2024 20:26:11 +0530 Subject: [PATCH 0511/1141] Fix linting errors --- mathesar_ui/.eslintrc.cjs | 1 - mathesar_ui/src/pages/database/DatabaseDetails.svelte | 1 + mathesar_ui/src/pages/home/DatabaseRow.svelte | 3 ++- mathesar_ui/src/pages/home/HomePage.svelte | 3 ++- mathesar_ui/src/stores/databases.ts | 3 ++- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/mathesar_ui/.eslintrc.cjs b/mathesar_ui/.eslintrc.cjs index f2212d4005..6b30ababfa 100644 --- a/mathesar_ui/.eslintrc.cjs +++ b/mathesar_ui/.eslintrc.cjs @@ -189,7 +189,6 @@ module.exports = { ts: 'never', }, ], - 'import/no-unresolved': 'off', 'import/prefer-default-export': 'off', 'no-void': 'off', 'no-underscore-dangle': 'off', diff --git a/mathesar_ui/src/pages/database/DatabaseDetails.svelte b/mathesar_ui/src/pages/database/DatabaseDetails.svelte index c1a31ea295..05dbf0b25e 100644 --- a/mathesar_ui/src/pages/database/DatabaseDetails.svelte +++ b/mathesar_ui/src/pages/database/DatabaseDetails.svelte @@ -1,5 +1,6 @@ + +
+
+ + + + + + + + + + +
diff --git a/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte b/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte new file mode 100644 index 0000000000..833e63678c --- /dev/null +++ b/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte @@ -0,0 +1,52 @@ + + +
+ installationSchemaLabels[o]} + getCheckboxHelp={(o) => installationSchemaHelp[o]} + getCheckboxDisabled={(o) => o === 'internal'} + /> +
+ + diff --git a/mathesar_ui/src/systems/databases/create-database/createDatabaseUtils.ts b/mathesar_ui/src/systems/databases/create-database/createDatabaseUtils.ts new file mode 100644 index 0000000000..e9bbef1ec8 --- /dev/null +++ b/mathesar_ui/src/systems/databases/create-database/createDatabaseUtils.ts @@ -0,0 +1,12 @@ +import { + sampleDataOptions, + type SampleDataSchemaIdentifier, +} from '@mathesar/api/rpc/database_setup'; + +export type InstallationSchema = SampleDataSchemaIdentifier | 'internal'; + +export function getSampleSchemasFromInstallationSchemas( + installationSchemas: InstallationSchema[], +): SampleDataSchemaIdentifier[] { + return sampleDataOptions.filter((o) => installationSchemas.includes(o)); +} From 0a16b44ed4d79d5c20d616db5e2b344ab30e21e0 Mon Sep 17 00:00:00 2001 From: pavish Date: Mon, 29 Jul 2024 19:27:09 +0530 Subject: [PATCH 0517/1141] Implement component to connect existing database --- .../ConnectExistingDatabase.svelte | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 mathesar_ui/src/systems/databases/create-database/ConnectExistingDatabase.svelte diff --git a/mathesar_ui/src/systems/databases/create-database/ConnectExistingDatabase.svelte b/mathesar_ui/src/systems/databases/create-database/ConnectExistingDatabase.svelte new file mode 100644 index 0000000000..51687b0591 --- /dev/null +++ b/mathesar_ui/src/systems/databases/create-database/ConnectExistingDatabase.svelte @@ -0,0 +1,91 @@ + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
From 716e9812e3985f06007102fc4fa39968fc83f01c Mon Sep 17 00:00:00 2001 From: pavish Date: Mon, 29 Jul 2024 19:27:25 +0530 Subject: [PATCH 0518/1141] Implement connect database modal --- mathesar_ui/src/i18n/languages/en/dict.json | 7 +- mathesar_ui/src/pages/WelcomePage.svelte | 2 +- mathesar_ui/src/pages/home/DatabaseRow.svelte | 44 +-------- mathesar_ui/src/pages/home/HomePage.svelte | 14 +-- .../DatabasesEmptyState.svelte | 0 .../ConnectDatabaseModal.svelte | 91 +++++++++++++++++++ .../create-database/ConnectOption.svelte | 52 +++++++++++ .../{connections => databases}/index.ts | 2 +- 8 files changed, 158 insertions(+), 54 deletions(-) rename mathesar_ui/src/systems/{connections => databases}/DatabasesEmptyState.svelte (100%) create mode 100644 mathesar_ui/src/systems/databases/create-database/ConnectDatabaseModal.svelte create mode 100644 mathesar_ui/src/systems/databases/create-database/ConnectOption.svelte rename mathesar_ui/src/systems/{connections => databases}/index.ts (71%) diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index cb35003739..14af6872a9 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -79,12 +79,12 @@ "confirm_and_create_table": "Confirm & create table", "confirm_delete_table": "To confirm the deletion of the [tableName] table, please enter the table name into the input field below.", "confirm_password": "Confirm Password", + "connect_existing_database": "Connect to an Existing Database", "connect_with_community_help": "Connect with Mathesar users and contributors in our community chat. Share ideas, get help, and contribute to discussions.", "connection_added_successfully": "The connection has been successfully added.", "connection_deleted_successfully": "The connection has been successfully deleted", "connection_name": "Connection Name", "connection_nickname": "Connection nickname", - "connection_nickname_help": "This nickname will appear in Mathesar's navigation and must be unique across all connections. You can change it later.", "connection_updated_successfully": "The connection has been successfully updated", "connections": "Connections", "connections_matching_search": "{count, plural, one {{count} connection matches [searchValue]} other {{count} connections match [searchValue]}}", @@ -107,9 +107,11 @@ "count_records_deleted_successfully": "{count, plural, one {{count} record deleted successfully!} other {{count} records deleted successfully!}}", "count_tables": "{count, plural, one {{count} table} other {{count} tables}}", "create_a_table_by_importing": "Create a table by importing your data", + "create_database_mathesar_internal_server": "Create a new PostgreSQL database on Mathesar's internal server", "create_db_if_not_exists": "Create this database if it does not already exist", "create_exploration_empty_state_help": "Use Data Explorer to analyze and share your data. Explorations are based on tables in your schema, to get started choose a table and start adding columns and transformations.", "create_link": "Create Link", + "create_new_database": "Create a New Database", "create_new_pg_user": "Create a new PostgreSQL user", "create_new_schema": "Create New Schema", "create_record_from_search": "Create Record From Search Criteria", @@ -241,6 +243,7 @@ "host_name": "Host name", "hours": "Hours", "how_do_you_want_to_create_table": "How do you want to create your table?", + "how_would_you_like_to_connect_db": "How would you like to connect your database?", "id_is_reserved_column": "The name \"id\" is reserved for the primary key column that will be created when creating the table.", "if_upgrade_fails_help": "If the upgrade fails, the update status screen will still show that an upgrade is available, and you will need to refer to our [documentationLink](documentation) for further troubleshooting.", "if_upgrade_succeeds_help": "If the upgrade succeeds, you will see that you're running the latest version.", @@ -313,7 +316,6 @@ "new_link_wont_work_once_regenerated": "Once you regenerate a new link, the old link will no longer work.", "new_password": "New Password", "new_pg_user_privileges_help": "The user will be granted CONNECT and CREATE privileges on the database.", - "new_postgresql_database_connection": "New PostgreSQL Database Connection", "new_record": "New Record", "new_records_reposition_refresh": "New records will be repositioned on refresh", "new_table": "New Table", @@ -391,6 +393,7 @@ "processing_data": "Processing Data", "prompt_new_password_next_login": "Resetting the password will prompt the user to change their password on their next login.", "properties": "Properties", + "provide_details_connect_existing_database": "Provide the details to connect an existing PostgreSQL database to Mathesar", "provide_url_to_file": "Provide a URL to the file", "public_schema_info": "Every PostgreSQL database includes the \"public\" schema. This protected schema can be read by anybody who accesses the database.", "read_release_notes": "Read the [releaseNotesLink](release notes) to see if this release requires any special upgrade instructions.", diff --git a/mathesar_ui/src/pages/WelcomePage.svelte b/mathesar_ui/src/pages/WelcomePage.svelte index 1334c4ff23..3c52398d90 100644 --- a/mathesar_ui/src/pages/WelcomePage.svelte +++ b/mathesar_ui/src/pages/WelcomePage.svelte @@ -5,7 +5,7 @@ import LayoutWithHeader from '@mathesar/layouts/LayoutWithHeader.svelte'; import { getDocsLink, getWikiLink } from '@mathesar/routes/urls'; import { getUserProfileStoreFromContext } from '@mathesar/stores/userProfile'; - import { DatabasesEmptyState } from '@mathesar/systems/connections'; + import { DatabasesEmptyState } from '@mathesar/systems/databases'; import { AnchorButton, Icon } from '@mathesar-component-library'; const userProfileStore = getUserProfileStoreFromContext(); diff --git a/mathesar_ui/src/pages/home/DatabaseRow.svelte b/mathesar_ui/src/pages/home/DatabaseRow.svelte index eaba9e3419..133faa3d34 100644 --- a/mathesar_ui/src/pages/home/DatabaseRow.svelte +++ b/mathesar_ui/src/pages/home/DatabaseRow.svelte @@ -2,19 +2,9 @@ import { _ } from 'svelte-i18n'; import type { Database } from '@mathesar/api/rpc/databases'; - import { iconDeleteMajor, iconEdit } from '@mathesar/icons'; import { iconConnection } from '@mathesar/icons'; import { getDatabasePageUrl } from '@mathesar/routes/urls'; - import { modal } from '@mathesar/stores/modal'; - import { getUserProfileStoreFromContext } from '@mathesar/stores/userProfile'; - import { Button, Icon } from '@mathesar-component-library'; - - const userProfileStore = getUserProfileStoreFromContext(); - $: userProfile = $userProfileStore; - $: isSuperUser = userProfile?.isSuperUser; - - const editConnectionModalController = modal.spawnModalController(); - const deleteConnectionModalController = modal.spawnModalController(); + import { Icon } from '@mathesar-component-library'; export let database: Database; @@ -26,44 +16,12 @@ {database.name} - {#if isSuperUser} -
- {/if} - - diff --git a/mathesar_ui/src/pages/home/HomePage.svelte b/mathesar_ui/src/pages/home/HomePage.svelte index bd49315a32..43f29d31dc 100644 --- a/mathesar_ui/src/pages/home/HomePage.svelte +++ b/mathesar_ui/src/pages/home/HomePage.svelte @@ -10,12 +10,15 @@ import { databasesStore } from '@mathesar/stores/databases'; import { modal } from '@mathesar/stores/modal'; import { getUserProfileStoreFromContext } from '@mathesar/stores/userProfile'; - import { DatabasesEmptyState } from '@mathesar/systems/connections'; + import { + ConnectDatabaseModal, + DatabasesEmptyState, + } from '@mathesar/systems/databases'; import { Button, Icon } from '@mathesar-component-library'; import DatabaseRow from './DatabaseRow.svelte'; - const addConnectionModalController = modal.spawnModalController(); + const connectDbModalController = modal.spawnModalController(); const userProfileStore = getUserProfileStoreFromContext(); $: userProfile = $userProfileStore; @@ -70,7 +73,7 @@ {#if isSuperUser} - {#if isSuperUser} - - {/if} @@ -119,7 +119,7 @@ - + diff --git a/mathesar_ui/src/systems/databases/create-database/ConnectOption.svelte b/mathesar_ui/src/systems/databases/create-database/ConnectOption.svelte new file mode 100644 index 0000000000..729f9238b1 --- /dev/null +++ b/mathesar_ui/src/systems/databases/create-database/ConnectOption.svelte @@ -0,0 +1,52 @@ + + + + + diff --git a/mathesar_ui/src/systems/connections/index.ts b/mathesar_ui/src/systems/databases/index.ts similarity index 71% rename from mathesar_ui/src/systems/connections/index.ts rename to mathesar_ui/src/systems/databases/index.ts index b2b1f7a755..137d13a306 100644 --- a/mathesar_ui/src/systems/connections/index.ts +++ b/mathesar_ui/src/systems/databases/index.ts @@ -1,4 +1,4 @@ export { default as DatabasesEmptyState } from './DatabasesEmptyState.svelte'; -// export { default as AddConnectionModal } from './AddConnectionModal.svelte'; +export { default as ConnectDatabaseModal } from './create-database/ConnectDatabaseModal.svelte'; // export { default as EditConnectionModal } from './EditConnectionModal.svelte'; // export { default as DeleteConnectionModal } from './DeleteConnectionModal.svelte'; From d3ad0ccd6f81da7d30443c7d3392be8e2c17ec7b Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Mon, 29 Jul 2024 22:17:52 -0400 Subject: [PATCH 0519/1141] Transition frontend to use RPC API for columns.list --- mathesar_ui/src/api/rest/columns.ts | 11 - mathesar_ui/src/api/rest/types/queries.ts | 2 +- .../src/api/rest/types/tables/columns.ts | 162 ------------ .../api/rest/types/tables/joinable_tables.ts | 12 +- mathesar_ui/src/api/rpc/columns.ts | 241 ++++++++++++++++++ mathesar_ui/src/api/rpc/index.ts | 2 + .../number-input/number-formatter/options.ts | 6 +- .../AbstractTypeControl.svelte | 10 +- .../AbstractTypeSelector.svelte | 10 +- .../DurationConfiguration.svelte | 2 +- .../components/abstract-type-control/utils.ts | 16 +- .../cell-fabric/data-types/boolean.ts | 41 +-- .../data-types/components/typeDefinitions.ts | 2 +- .../components/cell-fabric/data-types/date.ts | 21 +- .../cell-fabric/data-types/datetime.ts | 24 +- .../cell-fabric/data-types/duration.ts | 20 +- .../cell-fabric/data-types/money.ts | 51 ++-- .../cell-fabric/data-types/number.ts | 50 ++-- .../cell-fabric/data-types/string.ts | 12 +- .../components/cell-fabric/data-types/time.ts | 22 +- .../src/components/cell-fabric/utils.ts | 2 +- .../packages/json-rpc-client-builder/index.ts | 1 + .../json-rpc-client-builder/requests.ts | 15 ++ .../preview/ImportPreviewContent.svelte | 15 +- .../import/preview/ImportPreviewSheet.svelte | 2 +- .../pages/import/preview/PreviewColumn.svelte | 2 +- .../import/preview/importPreviewPageUtils.ts | 2 +- .../src/pages/record/RecordPage.svelte | 4 +- .../src/pages/record/TableWidget.svelte | 7 +- mathesar_ui/src/pages/table/TablePage.svelte | 7 +- .../abstract-types/type-configs/boolean.ts | 45 ++-- .../type-configs/comboTypes/arrayFactory.ts | 20 +- .../abstract-types/type-configs/date.ts | 27 +- .../abstract-types/type-configs/datetime.ts | 26 +- .../abstract-types/type-configs/duration.ts | 23 +- .../abstract-types/type-configs/money.ts | 54 ++-- .../abstract-types/type-configs/number.ts | 41 ++- .../abstract-types/type-configs/text.ts | 4 +- .../abstract-types/type-configs/time.ts | 26 +- .../abstract-types/type-configs/utils.ts | 5 +- .../src/stores/abstract-types/types.ts | 2 +- .../src/stores/table-data/TableStructure.ts | 20 +- mathesar_ui/src/stores/table-data/columns.ts | 140 +++++----- .../src/stores/table-data/constraints.ts | 2 +- .../src/stores/table-data/constraintsUtils.ts | 2 +- .../src/stores/table-data/processedColumns.ts | 2 +- mathesar_ui/src/stores/table-data/records.ts | 2 +- .../src/stores/table-data/tabularData.ts | 23 +- mathesar_ui/src/stores/table-data/utils.ts | 2 +- mathesar_ui/src/stores/tables.ts | 7 +- .../data-explorer/urlSerializationUtils.ts | 2 +- .../src/systems/data-explorer/utils.ts | 20 +- .../RecordSelectorContent.svelte | 13 +- .../RecordSelectorController.ts | 2 +- .../RecordSelectorTable.svelte | 10 +- .../RecordSelectorWindow.svelte | 3 +- .../record-selector/recordSelectorUtils.ts | 2 +- .../src/systems/table-view/Body.svelte | 5 +- .../actions-pane/ActionsPane.svelte | 6 +- .../ForeignKeyConstraintDetails.svelte | 2 +- .../constraints/NewFkConstraint.svelte | 6 +- .../constraints/NewUniqueConstraint.svelte | 3 +- .../new-column-cell/NewColumnCell.svelte | 7 +- .../link-table/LinkTableForm.svelte | 5 +- .../column/ColumnFormatting.svelte | 8 +- .../table-inspector/column/ColumnType.svelte | 4 +- .../column/SetDefaultValue.svelte | 12 +- .../ExtractColumnsModal.svelte | 8 +- .../record-summary/AppendColumn.svelte | 2 +- .../FkRecordSummaryConfig.svelte | 5 +- .../record-summary/TemplateInput.svelte | 2 +- .../recordSummaryTemplateUtils.ts | 2 +- .../table-inspector/table/TableActions.svelte | 21 +- .../table/TableDescription.svelte | 12 +- .../table-inspector/table/TableName.svelte | 8 +- .../table/links/TableLinks.svelte | 3 +- .../utils/date-time/DateTimeSpecification.ts | 5 +- .../src/utils/duration/DurationFormatter.ts | 2 +- .../utils/duration/DurationSpecification.ts | 25 +- mathesar_ui/src/utils/objectUtils.ts | 14 + 80 files changed, 763 insertions(+), 703 deletions(-) delete mode 100644 mathesar_ui/src/api/rest/columns.ts delete mode 100644 mathesar_ui/src/api/rest/types/tables/columns.ts create mode 100644 mathesar_ui/src/api/rpc/columns.ts create mode 100644 mathesar_ui/src/utils/objectUtils.ts diff --git a/mathesar_ui/src/api/rest/columns.ts b/mathesar_ui/src/api/rest/columns.ts deleted file mode 100644 index 64e1baf5a9..0000000000 --- a/mathesar_ui/src/api/rest/columns.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { Column } from './types/tables/columns'; -import { type PaginatedResponse, getAPI } from './utils/requestUtils'; - -function list(tableId: number) { - const url = `/api/db/v0/tables/${tableId}/columns/?limit=500`; - return getAPI>(url); -} - -export const columnsApi = { - list, -}; diff --git a/mathesar_ui/src/api/rest/types/queries.ts b/mathesar_ui/src/api/rest/types/queries.ts index 6517eef56a..6b5e72cea6 100644 --- a/mathesar_ui/src/api/rest/types/queries.ts +++ b/mathesar_ui/src/api/rest/types/queries.ts @@ -1,6 +1,6 @@ -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { JpPath } from '@mathesar/api/rest/types/tables/joinable_tables'; import type { PaginatedResponse } from '@mathesar/api/rest/utils/requestUtils'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { Schema } from '@mathesar/api/rpc/schemas'; export type QueryColumnAlias = string; diff --git a/mathesar_ui/src/api/rest/types/tables/columns.ts b/mathesar_ui/src/api/rest/types/tables/columns.ts deleted file mode 100644 index 763ba287fe..0000000000 --- a/mathesar_ui/src/api/rest/types/tables/columns.ts +++ /dev/null @@ -1,162 +0,0 @@ -import type { DbType } from '@mathesar/AppTypes'; - -/** - * | value | example locale | example format | - * | --------- | -------------- | -------------- | - * | 'english' | 'en' | '-123,456.7' | - * | 'german' | 'de' | '-123.456,7' | - * | 'french' | 'fr' | '-123 456,7' | - * | 'hindi' | 'hi' | '-1,23,456.7' | - * | 'swiss' | 'de-CH' | '-123'456.7' | - */ -export type NumberFormat = 'english' | 'german' | 'french' | 'hindi' | 'swiss'; - -/** - * This common for both Number and Money types - */ -interface FormattedNumberDisplayOptions { - /** When `null`, the browser's locale will be used. */ - number_format: NumberFormat | null; - - /** - * - "true": display grouping separators even if the locale prefers otherwise. - * - "false": do not display grouping separators. - */ - use_grouping: 'true' | 'false'; - - minimum_fraction_digits: number | null; - maximum_fraction_digits: number | null; -} - -export interface NumberDisplayOptions - extends Record, - FormattedNumberDisplayOptions {} - -/** - * See the [Postgres docs][1] for an explanation of `scale` and `precision`. - * - * [1]: https://www.postgresql.org/docs/current/datatype-numeric.html - */ -export interface NumberTypeOptions { - scale: number; - precision: number; -} - -export interface MoneyDisplayOptions extends FormattedNumberDisplayOptions { - /** - * e.g. "$", "€", "NZD", etc. - */ - currency_symbol: string; - - /** - * | value | formatting pattern | - * | ---------------- | -------------------------------------------- | - * | 'after-minus' | {minus_sign}{currency_symbol}{number} | - * | 'end-with-space' | {minus_sign}{number}{space}{currency_symbol} | - */ - currency_symbol_location: 'after-minus' | 'end-with-space'; - - /** - * PLANNED FOR FUTURE IMPLEMENTATION POST-ALPHA. - */ - // use_accounting_notation: boolean; -} - -export interface TextTypeOptions extends Record { - length: number | null; -} - -export interface BooleanDisplayOptions extends Record { - input: 'checkbox' | 'dropdown' | null; - custom_labels: { - TRUE: string; - FALSE: string; - } | null; -} - -export type DurationUnit = 'd' | 'h' | 'm' | 's' | 'ms'; - -export interface DurationDisplayOptions extends Record { - min: DurationUnit | null; - max: DurationUnit | null; - show_units: boolean | null; -} - -export type DateFormat = 'none' | 'us' | 'eu' | 'friendly' | 'iso'; - -export interface DateDisplayOptions extends Record { - format: DateFormat | null; -} - -export type TimeFormat = '24hr' | '12hr' | '24hrLong' | '12hrLong'; - -export interface TimeDisplayOptions extends Record { - format: TimeFormat | null; -} - -export interface TimeStampDisplayOptions extends Record { - date_format: DateDisplayOptions['format']; - time_format: TimeDisplayOptions['format']; -} - -export interface BaseColumn { - id: number; - name: string; - description: string | null; - type: DbType; - index: number; - nullable: boolean; - primary_key: boolean; - valid_target_types: DbType[]; - default: { - is_dynamic: boolean; - value: unknown; - } | null; -} - -/** - * TODO: - * - * Once we have all column types defined like `NumberColumn` is defined, then - * convert the `Column` type to a discriminated union of all possible specific - * column types. - */ -export interface Column extends BaseColumn { - type_options: Record | null; - display_options: Record | null; -} - -export interface MathesarMoneyColumn extends Column { - type: 'MATHESAR_TYPES.MATHESAR_MONEY'; - type_options: Partial | null; - display_options: Partial | null; -} - -export interface PostgresMoneyColumn extends Column { - type: 'MONEY'; - type_options: null; - display_options: Partial | null; -} - -export type MoneyColumn = MathesarMoneyColumn | PostgresMoneyColumn; - -// TODO: Remove specification of DB types here -export interface NumberColumn extends Column { - type: - | 'BIGINT' - | 'BIGSERIAL' - | 'DECIMAL' - | 'DOUBLE PRECISION' - | 'INTEGER' - | 'NUMERIC' - | 'REAL' - | 'SERIAL' - | 'SMALLINT' - | 'SMALLSERIAL'; - type_options: Partial | null; - display_options: Partial | null; -} - -export interface ArrayTypeOptions extends Record { - item_type: string; -} diff --git a/mathesar_ui/src/api/rest/types/tables/joinable_tables.ts b/mathesar_ui/src/api/rest/types/tables/joinable_tables.ts index 8b84483b2e..c53a5742f7 100644 --- a/mathesar_ui/src/api/rest/types/tables/joinable_tables.ts +++ b/mathesar_ui/src/api/rest/types/tables/joinable_tables.ts @@ -6,11 +6,11 @@ import type { Table } from '@mathesar/api/rpc/tables'; -import type { Column } from './columns'; - type ForeignKeyId = number; type IsLinkReversed = boolean; -export type JpPath = [Column['id'], Column['id']][]; + +/** [attnum, attnum][] */ +export type JpPath = [number, number][]; export interface JoinableTable { target: Table['oid']; // baseTableId @@ -26,14 +26,14 @@ export interface JoinableTablesResult { string, // tableId { name: Table['name']; - columns: Column['id'][]; + columns: number[]; } >; columns: Record< string, // columnId { - name: Column['name']; - type: Column['type']; + name: string; + type: string; } >; } diff --git a/mathesar_ui/src/api/rpc/columns.ts b/mathesar_ui/src/api/rpc/columns.ts new file mode 100644 index 0000000000..e4a3838ce6 --- /dev/null +++ b/mathesar_ui/src/api/rpc/columns.ts @@ -0,0 +1,241 @@ +import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; + +export type BooleanInputType = 'checkbox' | 'dropdown'; + +/** + * | value | example locale | example format | + * | --------- | -------------- | -------------- | + * | 'english' | 'en' | '-123,456.7' | + * | 'german' | 'de' | '-123.456,7' | + * | 'french' | 'fr' | '-123 456,7' | + * | 'hindi' | 'hi' | '-1,23,456.7' | + * | 'swiss' | 'de-CH' | '-123'456.7' | + */ +export type NumberFormat = 'english' | 'german' | 'french' | 'hindi' | 'swiss'; + +/** + * Corresponds to the Intl.NumberFormat options `useGrouping`. + * + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/NumberFormat/NumberFormat#usegrouping + * + * - `"always"` - Display grouping separators even if the locale prefers + * otherwise. + * - `"auto"` - Display grouping separators based on the locale preference, + * which may also be dependent on the currency. + * - `"never"` - Do not display grouping separators. + */ +export type NumberGrouping = 'always' | 'auto' | 'never'; + +/** + * | value | formatting pattern | + * | ---------------- | -------------------------------------------- | + * | 'after-minus' | {minus_sign}{currency_symbol}{number} | + * | 'end-with-space' | {minus_sign}{number}{space}{currency_symbol} | + */ +export type CurrencyLocation = 'after-minus' | 'end-with-space'; + +export const allDurationUnits = ['d', 'h', 'm', 's', 'ms'] as const; + +export type DurationUnit = (typeof allDurationUnits)[number]; + +export type DateFormat = 'none' | 'us' | 'eu' | 'friendly' | 'iso'; + +export type TimeFormat = '24hr' | '12hr' | '24hrLong' | '12hrLong'; + +/** + * See the [Postgres docs][1] for an explanation of `scale` and `precision`. + * + * [1]: https://www.postgresql.org/docs/current/datatype-numeric.html + */ +interface ColumnTypeOptions { + /** + * For numeric types, the number of significant digits. For date/time types, + * the number of fractional digits. + */ + precision?: number | null; + + /** For numeric types, the number of fractional digits. */ + scale?: number | null; + + /** Which time fields are stored. See Postgres docs. */ + fields?: string | null; + + /** The maximum length of a character-type field. */ + length?: number | null; + + /** The member type for arrays. */ + item_type?: string | null; +} + +/** + * The column display options values, typed as we need in order to render them + * in the UI. + */ +export interface RequiredColumnDisplayOptions { + /** The type of input used for boolean values */ + bool_input: BooleanInputType; + + /** The text to display for a boolean `true` value */ + bool_true: string; + + /** The text to display for a boolean `false` value */ + bool_false: string; + + /** The minimum number of fraction digits to display for a number */ + num_min_frac_digits: number; + + /** The maximum number of fraction digits to display for a number */ + num_max_frac_digits: number; + + /** When `null`, the browser's locale will be used. */ + num_format: NumberFormat | null; + + /** + * - `null`: display grouping separators if the locale prefers it. + * - `true`: display grouping separators. + * - `false`: do not display grouping separators. + */ + num_grouping: NumberGrouping; + + /** The currency symbol to show for a money type e.g. "$", "€", "NZD", etc. */ + mon_currency_symbol: string; + + mon_currency_location: CurrencyLocation; + + time_format: TimeFormat; + + date_format: DateFormat; + + duration_min: DurationUnit; + + duration_max: DurationUnit; + + duration_show_units: boolean; +} + +/** The column display options values, types as we get them from the API. */ +type ColumnDisplayOptions = { + [K in keyof RequiredColumnDisplayOptions]?: + | RequiredColumnDisplayOptions[K] + | null; +}; + +export const defaultColumnDisplayOptions: RequiredColumnDisplayOptions = { + bool_input: 'checkbox', + bool_true: 'true', + bool_false: 'false', + num_min_frac_digits: 0, + num_max_frac_digits: 20, + num_format: null, + num_grouping: 'auto', + mon_currency_symbol: '$', + mon_currency_location: 'after-minus', + time_format: '24hr', + date_format: 'none', + duration_min: 's', + duration_max: 'm', + duration_show_units: true, +}; + +/** + * Gets a display option value from a column, if present. Otherwise returns the + * default value for that display option. + */ +export function getColumnDisplayOption< + Option extends keyof RequiredColumnDisplayOptions, +>(column: Pick, opt: Option) { + return column.display_options?.[opt] ?? defaultColumnDisplayOptions[opt]; +} + +interface ColumnDefault { + value: string; + is_dynamic: boolean; +} + +/** The raw column data, from the user database only */ +interface RawColumn { + /** The PostgreSQL attnum of the column */ + id: number; + name: string; + description: string | null; + /** The PostgreSQL data type */ + type: string; + type_options: ColumnTypeOptions | null; + nullable: boolean; + primary_key: boolean; + default: ColumnDefault | null; + has_dependents: boolean; + valid_target_types: string[]; +} + +/** + * The raw column data from the user database combined with Mathesar's metadata + */ +export interface Column extends RawColumn { + display_options: ColumnDisplayOptions | null; +} + +export interface ColumnCreationSpec { + name?: string; + type?: string; + description?: string; + type_options?: ColumnTypeOptions; + nullable?: boolean; + default?: ColumnDefault; +} + +export interface ColumnPatchSpec { + id: number; + name?: string; + type?: string | null; + description?: string | null; + type_options?: ColumnTypeOptions | null; + nullable?: boolean; + default?: ColumnDefault | null; +} + +export const columns = { + list: rpcMethodTypeContainer< + { + database_id: number; + table_oid: number; + }, + RawColumn[] + >(), + + list_with_metadata: rpcMethodTypeContainer< + { + database_id: number; + table_oid: number; + }, + Column[] + >(), + + /** Returns an array of the attnums of the newly-added columns */ + add: rpcMethodTypeContainer< + { + database_id: number; + table_oid: number; + column_data_list: ColumnCreationSpec[]; + }, + number[] + >(), + + patch: rpcMethodTypeContainer< + { + database_id: number; + table_oid: number; + column_data_list: ColumnPatchSpec[]; + }, + void + >(), + + delete: rpcMethodTypeContainer< + { + database_id: number; + table_oid: number; + column_attnums: number[]; + }, + void + >(), +}; diff --git a/mathesar_ui/src/api/rpc/index.ts b/mathesar_ui/src/api/rpc/index.ts index a3cefb1626..4a0a5a9697 100644 --- a/mathesar_ui/src/api/rpc/index.ts +++ b/mathesar_ui/src/api/rpc/index.ts @@ -2,6 +2,7 @@ import Cookies from 'js-cookie'; import { buildRpcApi } from '@mathesar/packages/json-rpc-client-builder'; +import { columns } from './columns'; import { configured_roles } from './configured_roles'; import { database_setup } from './database_setup'; import { databases } from './databases'; @@ -20,5 +21,6 @@ export const api = buildRpcApi({ schemas, servers, tables, + columns, }, }); diff --git a/mathesar_ui/src/component-library/number-input/number-formatter/options.ts b/mathesar_ui/src/component-library/number-input/number-formatter/options.ts index 2be88e9fea..04ba0d54cb 100644 --- a/mathesar_ui/src/component-library/number-input/number-formatter/options.ts +++ b/mathesar_ui/src/component-library/number-input/number-formatter/options.ts @@ -7,14 +7,10 @@ export interface Options { /** * Corresponds to the options of the [Intl.NumberFormat][1] API. * - * The MDN docs say that "true" and "false" are accepted as strings, but in my - * testing with Firefox and Chromium, I noticed that those values need to be - * passed as booleans to work correctly. - * * [1]: * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/NumberFormat/NumberFormat */ - useGrouping: boolean; + useGrouping: 'always' | 'auto' | 'min2' | true | false; minimumFractionDigits: number; maximumFractionDigits: number; forceTrailingDecimal: boolean; diff --git a/mathesar_ui/src/components/abstract-type-control/AbstractTypeControl.svelte b/mathesar_ui/src/components/abstract-type-control/AbstractTypeControl.svelte index 5049f2185f..93301afd3e 100644 --- a/mathesar_ui/src/components/abstract-type-control/AbstractTypeControl.svelte +++ b/mathesar_ui/src/components/abstract-type-control/AbstractTypeControl.svelte @@ -4,6 +4,7 @@ import type { RequestStatus } from '@mathesar/api/rest/utils/requestUtils'; import { toast } from '@mathesar/stores/toast'; + import { objectsAreDeeplyEqual } from '@mathesar/utils/objectUtils'; import { CancelOrProceedButtonPair, createValidationContext, @@ -13,10 +14,9 @@ import AbstractTypeDBOptions from './AbstractTypeDBOptions.svelte'; import AbstractTypeSelector from './AbstractTypeSelector.svelte'; - import { - type ColumnTypeOptionsSaveArgs, - type ColumnWithAbstractType, - hasTypeOptionsChanged, + import type { + ColumnTypeOptionsSaveArgs, + ColumnWithAbstractType, } from './utils'; const dispatch = createEventDispatcher(); @@ -36,7 +36,7 @@ $: actionButtonsVisible = selectedAbstractType !== column.abstractType || selectedDbType !== column.type || - hasTypeOptionsChanged(column.type_options ?? {}, typeOptions ?? {}); + !objectsAreDeeplyEqual(column.type_options, typeOptions); let typeChangeState: RequestStatus; diff --git a/mathesar_ui/src/components/abstract-type-control/AbstractTypeSelector.svelte b/mathesar_ui/src/components/abstract-type-control/AbstractTypeSelector.svelte index 9984c0632a..1bd153be53 100644 --- a/mathesar_ui/src/components/abstract-type-control/AbstractTypeSelector.svelte +++ b/mathesar_ui/src/components/abstract-type-control/AbstractTypeSelector.svelte @@ -26,7 +26,15 @@ $: allowedTypeConversions = getAllowedAbstractTypesForDbTypeAndItsTargetTypes( column.type, - column.valid_target_types ?? [], + + // TODO_BETA + // + // We need to find another way to get the valid target types for a column + // since the RPC API no longer returns this. + + // column.valid_target_types ?? [], + [], + $currentDbAbstractTypes.data, ).filter((item) => !['jsonlist', 'map'].includes(item.identifier)); diff --git a/mathesar_ui/src/components/abstract-type-control/config-components/DurationConfiguration.svelte b/mathesar_ui/src/components/abstract-type-control/config-components/DurationConfiguration.svelte index 2771c8178b..0ffe9f0e46 100644 --- a/mathesar_ui/src/components/abstract-type-control/config-components/DurationConfiguration.svelte +++ b/mathesar_ui/src/components/abstract-type-control/config-components/DurationConfiguration.svelte @@ -2,7 +2,7 @@ import type { Writable } from 'svelte/store'; import { _ } from 'svelte-i18n'; - import type { DurationUnit } from '@mathesar/api/rest/types/tables/columns'; + import type { DurationUnit } from '@mathesar/api/rpc/columns'; import { RichText } from '@mathesar/components/rich-text'; import { DurationSpecification } from '@mathesar/utils/duration'; import type { DurationConfig } from '@mathesar/utils/duration/types'; diff --git a/mathesar_ui/src/components/abstract-type-control/utils.ts b/mathesar_ui/src/components/abstract-type-control/utils.ts index 822b90faa5..84d003c891 100644 --- a/mathesar_ui/src/components/abstract-type-control/utils.ts +++ b/mathesar_ui/src/components/abstract-type-control/utils.ts @@ -1,6 +1,6 @@ import { readable } from 'svelte/store'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { DbType } from '@mathesar/AppTypes'; import type { AbstractType, @@ -93,17 +93,3 @@ export function constructDisplayForm( displayFormValues, }; } - -export function hasTypeOptionsChanged( - previousTypeOptions: NonNullable, - currentTypeOptions: NonNullable, -): boolean { - for (const key in currentTypeOptions) { - if (Object.hasOwn(currentTypeOptions, key)) { - if (currentTypeOptions[key] !== previousTypeOptions[key]) { - return true; - } - } - } - return false; -} diff --git a/mathesar_ui/src/components/cell-fabric/data-types/boolean.ts b/mathesar_ui/src/components/cell-fabric/data-types/boolean.ts index 00a3435bce..112f966071 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/boolean.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/boolean.ts @@ -1,4 +1,4 @@ -import type { BooleanDisplayOptions } from '@mathesar/api/rest/types/tables/columns'; +import { type Column, getColumnDisplayOption } from '@mathesar/api/rpc/columns'; import { Select, isDefinedNonNullable } from '@mathesar-component-library'; import type { ComponentAndProps, @@ -11,37 +11,38 @@ import type { CheckBoxCellExternalProps, SingleSelectCellExternalProps, } from './components/typeDefinitions'; -import type { CellColumnLike, CellComponentFactory } from './typeDefinitions'; - -export interface BooleanLikeColumn extends CellColumnLike { - display_options: Partial | null; -} +import type { CellComponentFactory } from './typeDefinitions'; type Props = | CheckBoxCellExternalProps | SingleSelectCellExternalProps; -function getLabels( - displayOptions?: BooleanLikeColumn['display_options'], -): [string, string] { - const customLabels = displayOptions?.custom_labels ?? undefined; - return [customLabels?.TRUE ?? 'true', customLabels?.FALSE ?? 'false']; +interface BooleanLabels { + true: string; + false: string; +} + +function getLabels(column: Column): BooleanLabels { + return { + true: getColumnDisplayOption(column, 'bool_true'), + false: getColumnDisplayOption(column, 'bool_false'), + }; } function getFormattedValue( - labels: [string, string], + labels: BooleanLabels, value?: boolean | null, ): string { if (isDefinedNonNullable(value)) { - return value ? labels[0] : labels[1]; + return value ? labels.true : labels.false; } return ''; } function getProps( - column: BooleanLikeColumn, + column: Column, ): SingleSelectCellExternalProps { - const labels = getLabels(column.display_options); + const labels = getLabels(column); return { options: [null, true, false], getLabel: (value?: boolean | null) => getFormattedValue(labels, value), @@ -50,9 +51,9 @@ function getProps( const booleanType: CellComponentFactory = { initialInputValue: null, - get: (column: BooleanLikeColumn): ComponentAndProps => { + get: (column: Column): ComponentAndProps => { const displayOptions = column.display_options ?? undefined; - if (displayOptions && displayOptions.input === 'dropdown') { + if (displayOptions && displayOptions.bool_input === 'dropdown') { return { component: SingleSelectCell, props: getProps(column), @@ -61,13 +62,13 @@ const booleanType: CellComponentFactory = { return { component: CheckboxCell, props: {} }; }, getInput: ( - column: BooleanLikeColumn, + column: Column, ): ComponentAndProps> => ({ component: Select, props: getProps(column), }), - getDisplayFormatter(column: BooleanLikeColumn) { - const labels = getLabels(column.display_options); + getDisplayFormatter(column: Column) { + const labels = getLabels(column); return (value: unknown) => { if (value === null || value === undefined || typeof value === 'boolean') { return getFormattedValue(labels, value); diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/typeDefinitions.ts b/mathesar_ui/src/components/cell-fabric/data-types/components/typeDefinitions.ts index 9897637c48..2c07cdac98 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/typeDefinitions.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/typeDefinitions.ts @@ -1,5 +1,5 @@ -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { FkConstraint } from '@mathesar/api/rest/types/tables/constraints'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { DBObjectEntry } from '@mathesar/AppTypes'; import type { DateTimeFormatter } from '@mathesar/utils/date-time/types'; import type { diff --git a/mathesar_ui/src/components/cell-fabric/data-types/date.ts b/mathesar_ui/src/components/cell-fabric/data-types/date.ts index a754b40b06..66c130800b 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/date.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/date.ts @@ -1,4 +1,4 @@ -import type { DateDisplayOptions } from '@mathesar/api/rest/types/tables/columns'; +import { type Column, getColumnDisplayOption } from '@mathesar/api/rpc/columns'; import { DateTimeFormatter, DateTimeSpecification, @@ -9,15 +9,10 @@ import type { ComponentAndProps } from '@mathesar-component-library/types'; import DateTimeCell from './components/date-time/DateTimeCell.svelte'; import DateTimeInput from './components/date-time/DateTimeInput.svelte'; import type { DateTimeCellExternalProps } from './components/typeDefinitions'; -import type { CellColumnLike, CellComponentFactory } from './typeDefinitions'; +import type { CellComponentFactory } from './typeDefinitions'; -export interface DateLikeColumn extends CellColumnLike { - display_options: Partial | null; -} - -function getProps(column: DateLikeColumn): DateTimeCellExternalProps { - const displayOptions = column.display_options ?? {}; - const format = displayOptions.format ?? 'none'; +function getProps(column: Column): DateTimeCellExternalProps { + const format = getColumnDisplayOption(column, 'date_format'); const specification = new DateTimeSpecification({ type: 'date', dateFormat: format, @@ -39,14 +34,12 @@ function getProps(column: DateLikeColumn): DateTimeCellExternalProps { } const stringType: CellComponentFactory = { - get: ( - column: DateLikeColumn, - ): ComponentAndProps => ({ + get: (column: Column): ComponentAndProps => ({ component: DateTimeCell, props: getProps(column), }), getInput: ( - column: DateLikeColumn, + column: Column, ): ComponentAndProps< Omit > => ({ @@ -56,7 +49,7 @@ const stringType: CellComponentFactory = { allowRelativePresets: true, }, }), - getDisplayFormatter(column: DateLikeColumn) { + getDisplayFormatter(column: Column) { return (v) => getProps(column).formatForDisplay(String(v)); }, }; diff --git a/mathesar_ui/src/components/cell-fabric/data-types/datetime.ts b/mathesar_ui/src/components/cell-fabric/data-types/datetime.ts index d214e45060..a5ef36949c 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/datetime.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/datetime.ts @@ -1,4 +1,4 @@ -import type { TimeStampDisplayOptions } from '@mathesar/api/rest/types/tables/columns'; +import { type Column, getColumnDisplayOption } from '@mathesar/api/rpc/columns'; import { DateTimeFormatter, DateTimeSpecification, @@ -9,19 +9,14 @@ import type { ComponentAndProps } from '@mathesar-component-library/types'; import DateTimeCell from './components/date-time/DateTimeCell.svelte'; import DateTimeInput from './components/date-time/DateTimeInput.svelte'; import type { DateTimeCellExternalProps } from './components/typeDefinitions'; -import type { CellColumnLike, CellComponentFactory } from './typeDefinitions'; - -export interface DateLikeColumn extends CellColumnLike { - display_options: Partial | null; -} +import type { CellComponentFactory } from './typeDefinitions'; function getProps( - column: DateLikeColumn, + column: Column, supportTimeZone: boolean, ): DateTimeCellExternalProps { - const displayOptions = column.display_options ?? {}; - const dateFormat = displayOptions.date_format ?? 'none'; - const timeFormat = displayOptions.time_format ?? '24hr'; + const dateFormat = getColumnDisplayOption(column, 'date_format'); + const timeFormat = getColumnDisplayOption(column, 'time_format'); const specification = new DateTimeSpecification({ type: supportTimeZone ? 'timestampWithTZ' : 'timestamp', dateFormat, @@ -47,14 +42,14 @@ function getProps( const datetimeType: CellComponentFactory = { get: ( - column: DateLikeColumn, + column: Column, config?: { supportTimeZone?: boolean }, ): ComponentAndProps => ({ component: DateTimeCell, props: getProps(column, config?.supportTimeZone ?? false), }), getInput: ( - column: DateLikeColumn, + column: Column, config?: { supportTimeZone?: boolean }, ): ComponentAndProps< Omit @@ -65,10 +60,7 @@ const datetimeType: CellComponentFactory = { allowRelativePresets: true, }, }), - getDisplayFormatter( - column: DateLikeColumn, - config?: { supportTimeZone?: boolean }, - ) { + getDisplayFormatter(column: Column, config?: { supportTimeZone?: boolean }) { return (v) => getProps(column, config?.supportTimeZone ?? false).formatForDisplay( String(v), diff --git a/mathesar_ui/src/components/cell-fabric/data-types/duration.ts b/mathesar_ui/src/components/cell-fabric/data-types/duration.ts index 5f7af38409..2864958f0c 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/duration.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/duration.ts @@ -1,4 +1,4 @@ -import type { DurationDisplayOptions } from '@mathesar/api/rest/types/tables/columns'; +import type { Column } from '@mathesar/api/rpc/columns'; import { DurationFormatter, DurationSpecification, @@ -14,16 +14,12 @@ import type { import FormattedInputCell from './components/formatted-input/FormattedInputCell.svelte'; import type { FormattedInputCellExternalProps } from './components/typeDefinitions'; -import type { CellColumnLike, CellComponentFactory } from './typeDefinitions'; +import type { CellComponentFactory } from './typeDefinitions'; -export interface DurationLikeColumn extends CellColumnLike { - display_options: Partial | null; -} - -function getProps(column: DurationLikeColumn): FormattedInputCellExternalProps { +function getProps(column: Column): FormattedInputCellExternalProps { const defaults = DurationSpecification.getDefaults(); - const max = column.display_options?.max ?? defaults.max; - const min = column.display_options?.min ?? defaults.min; + const max = column.display_options?.duration_max ?? defaults.max; + const min = column.display_options?.duration_min ?? defaults.min; const durationSpecification = new DurationSpecification({ max, min }); const formatter = new DurationFormatter(durationSpecification); return { @@ -42,18 +38,18 @@ function getProps(column: DurationLikeColumn): FormattedInputCellExternalProps { const durationType: CellComponentFactory = { get: ( - column: DurationLikeColumn, + column: Column, ): ComponentAndProps => ({ component: FormattedInputCell, props: getProps(column), }), getInput: ( - column: DurationLikeColumn, + column: Column, ): ComponentAndProps> => ({ component: FormattedInput, props: getProps(column), }), - getDisplayFormatter(column: DurationLikeColumn) { + getDisplayFormatter(column: Column) { return (v) => getProps(column).formatForDisplay(String(v)); }, }; diff --git a/mathesar_ui/src/components/cell-fabric/data-types/money.ts b/mathesar_ui/src/components/cell-fabric/data-types/money.ts index 028f50ac60..8666d4fc49 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/money.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/money.ts @@ -1,7 +1,8 @@ -import type { - MoneyColumn, - NumberFormat, -} from '@mathesar/api/rest/types/tables/columns'; +import { + type Column, + type NumberFormat, + getColumnDisplayOption, +} from '@mathesar/api/rpc/columns'; import { StringifiedNumberFormatter, isDefinedNonNullable, @@ -14,10 +15,6 @@ import type { MoneyCellExternalProps } from './components/typeDefinitions'; import { getUseGrouping } from './number'; import type { CellComponentFactory } from './typeDefinitions'; -// Values to use if for some reason we don't get them from the API. -const FALLBACK_CURRENCY_SYMBOL = '$'; -const FALLBACK_CURRENCY_SYMBOL_LOCATION = 'after-minus'; - // prettier-ignore const localeMap = new Map([ ['english' , 'en' ], @@ -27,30 +24,36 @@ const localeMap = new Map([ ['swiss' , 'de-CH' ], ]); -function moneyColumnIsInteger(column: MoneyColumn): boolean { +function ColumnIsInteger(column: Column): boolean { return (column.type_options?.scale ?? Infinity) === 0; } function getFormatterOptions( - column: MoneyColumn, + column: Column, ): MoneyCellExternalProps['formatterOptions'] { - const displayOptions = column.display_options; - const format = displayOptions?.number_format ?? null; + const format = getColumnDisplayOption(column, 'num_format'); return { locale: (format && localeMap.get(format)) ?? undefined, - useGrouping: getUseGrouping(displayOptions?.use_grouping ?? 'true'), - allowFloat: !moneyColumnIsInteger(column), + useGrouping: getUseGrouping(column), + allowFloat: !ColumnIsInteger(column), allowNegative: true, - minimumFractionDigits: displayOptions?.minimum_fraction_digits ?? undefined, - maximumFractionDigits: displayOptions?.maximum_fraction_digits ?? undefined, - currencySymbol: displayOptions?.currency_symbol ?? FALLBACK_CURRENCY_SYMBOL, - currencySymbolLocation: - displayOptions?.currency_symbol_location ?? - FALLBACK_CURRENCY_SYMBOL_LOCATION, + minimumFractionDigits: getColumnDisplayOption( + column, + 'num_min_frac_digits', + ), + maximumFractionDigits: getColumnDisplayOption( + column, + 'num_max_frac_digits', + ), + currencySymbol: getColumnDisplayOption(column, 'mon_currency_symbol'), + currencySymbolLocation: getColumnDisplayOption( + column, + 'mon_currency_location', + ), }; } -function getProps(column: MoneyColumn): MoneyCellExternalProps { +function getProps(column: Column): MoneyCellExternalProps { const formatterOptions = getFormatterOptions(column); const displayFormatter = new StringifiedNumberFormatter(formatterOptions); const insertCurrencySymbol = (() => { @@ -78,7 +81,7 @@ function getProps(column: MoneyColumn): MoneyCellExternalProps { } const moneyType: CellComponentFactory = { - get(column: MoneyColumn): ComponentAndProps { + get(column: Column): ComponentAndProps { return { component: MoneyCell, props: getProps(column), @@ -86,7 +89,7 @@ const moneyType: CellComponentFactory = { }, getInput( - column: MoneyColumn, + column: Column, ): ComponentAndProps { return { component: MoneyCellInput, @@ -97,7 +100,7 @@ const moneyType: CellComponentFactory = { }; }, - getDisplayFormatter(column: MoneyColumn) { + getDisplayFormatter(column: Column) { return (v) => getProps(column).formatForDisplay(String(v)); }, }; diff --git a/mathesar_ui/src/components/cell-fabric/data-types/number.ts b/mathesar_ui/src/components/cell-fabric/data-types/number.ts index eff54ce072..d1d93eadd0 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/number.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/number.ts @@ -1,10 +1,11 @@ -import type { - NumberColumn, - NumberDisplayOptions, - NumberFormat, -} from '@mathesar/api/rest/types/tables/columns'; +import { + type Column, + type NumberFormat, + getColumnDisplayOption, +} from '@mathesar/api/rpc/columns'; import { StringifiedNumberFormatter, + assertExhaustive, isDefinedNonNullable, } from '@mathesar-component-library'; import type { ComponentAndProps } from '@mathesar-component-library/types'; @@ -34,7 +35,7 @@ interface Config extends Record { } function getAllowFloat( - column: NumberColumn, + column: Column, floatAllowanceStrategy?: FloatAllowanceStrategy, ): boolean { if (floatAllowanceStrategy === 'scale-based') { @@ -47,29 +48,33 @@ function getAllowFloat( } export function getUseGrouping( - apiUseGrouping: NumberDisplayOptions['use_grouping'], + column: Column, ): NumberCellExternalProps['formatterOptions']['useGrouping'] { - switch (apiUseGrouping) { - case 'true': - return true; - case 'false': - default: + const grouping = getColumnDisplayOption(column, 'num_grouping'); + switch (grouping) { + case 'always': + return 'always'; + case 'auto': + return 'auto'; + case 'never': return false; + default: + return assertExhaustive(grouping); } } function getFormatterOptions( - column: NumberColumn, + column: Column, config?: Config, ): NumberCellExternalProps['formatterOptions'] { const displayOptions = column.display_options; - const format = displayOptions?.number_format ?? null; + const format = displayOptions?.num_format ?? null; const locale = (format && localeMap.get(format)) ?? undefined; - const useGrouping = getUseGrouping(displayOptions?.use_grouping ?? 'false'); + const useGrouping = getUseGrouping(column); const allowFloat = getAllowFloat(column, config?.floatAllowanceStrategy); const allowNegative = true; const minimumFractionDigits = - displayOptions?.minimum_fraction_digits ?? undefined; + displayOptions?.num_min_frac_digits ?? undefined; return { locale, allowFloat, @@ -79,14 +84,11 @@ function getFormatterOptions( }; } -function getProps( - column: NumberColumn, - config?: Config, -): NumberCellExternalProps { +function getProps(column: Column, config?: Config): NumberCellExternalProps { const basicFormatterOptions = getFormatterOptions(column, config); const displayOptions = column.display_options; const maximumFractionDigits = - displayOptions?.maximum_fraction_digits ?? undefined; + displayOptions?.num_max_frac_digits ?? undefined; const formatterOptions = { ...basicFormatterOptions, // We only want to apply `maximumFractionDigits` during display. We don't @@ -109,7 +111,7 @@ function getProps( const numberType: CellComponentFactory = { get( - column: NumberColumn, + column: Column, config?: Config, ): ComponentAndProps { return { @@ -119,7 +121,7 @@ const numberType: CellComponentFactory = { }, getInput( - column: NumberColumn, + column: Column, config?: Config, ): ComponentAndProps { return { @@ -128,7 +130,7 @@ const numberType: CellComponentFactory = { }; }, - getDisplayFormatter(column: NumberColumn, config?: Config) { + getDisplayFormatter(column: Column, config?: Config) { return (v) => getProps(column, config).formatForDisplay(String(v)); }, }; diff --git a/mathesar_ui/src/components/cell-fabric/data-types/string.ts b/mathesar_ui/src/components/cell-fabric/data-types/string.ts index 79ab7c20cd..3711fa67c8 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/string.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/string.ts @@ -1,4 +1,4 @@ -import type { TextTypeOptions } from '@mathesar/api/rest/types/tables/columns'; +import type { Column } from '@mathesar/api/rpc/columns'; import GrowableTextArea from '@mathesar/components/GrowableTextArea.svelte'; import { TextInput, optionalNonNullable } from '@mathesar-component-library'; import type { @@ -12,16 +12,12 @@ import type { TextAreaCellExternalProps, TextBoxCellExternalProps, } from './components/typeDefinitions'; -import type { CellColumnLike, CellComponentFactory } from './typeDefinitions'; - -export interface StringLikeColumn extends CellColumnLike { - type_options: Partial | null; -} +import type { CellComponentFactory } from './typeDefinitions'; const stringType: CellComponentFactory = { initialInputValue: '', get: ( - column: StringLikeColumn, + column: Column, config?: { multiLine?: boolean }, ): ComponentAndProps< TextBoxCellExternalProps | TextAreaCellExternalProps @@ -31,7 +27,7 @@ const stringType: CellComponentFactory = { return { component, props: typeOptions }; }, getInput: ( - column: StringLikeColumn, + column: Column, config?: { multiLine?: boolean }, ): ComponentAndProps => { const component = config?.multiLine ? GrowableTextArea : TextInput; diff --git a/mathesar_ui/src/components/cell-fabric/data-types/time.ts b/mathesar_ui/src/components/cell-fabric/data-types/time.ts index a6c1ed331f..f4301ceae0 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/time.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/time.ts @@ -1,4 +1,4 @@ -import type { TimeDisplayOptions } from '@mathesar/api/rest/types/tables/columns'; +import { type Column, getColumnDisplayOption } from '@mathesar/api/rpc/columns'; import { DateTimeFormatter, DateTimeSpecification, @@ -9,18 +9,13 @@ import type { ComponentAndProps } from '@mathesar-component-library/types'; import DateTimeCell from './components/date-time/DateTimeCell.svelte'; import DateTimeInput from './components/date-time/DateTimeInput.svelte'; import type { DateTimeCellExternalProps } from './components/typeDefinitions'; -import type { CellColumnLike, CellComponentFactory } from './typeDefinitions'; - -export interface TimeLikeColumn extends CellColumnLike { - display_options: Partial | null; -} +import type { CellComponentFactory } from './typeDefinitions'; function getProps( - column: TimeLikeColumn, + column: Column, supportTimeZone: boolean, ): DateTimeCellExternalProps { - const displayOptions = column.display_options ?? {}; - const format = displayOptions.format ?? '24hr'; + const format = getColumnDisplayOption(column, 'time_format'); const specification = new DateTimeSpecification({ type: supportTimeZone ? 'timeWithTZ' : 'time', timeFormat: format, @@ -45,14 +40,14 @@ function getProps( const timeType: CellComponentFactory = { get: ( - column: TimeLikeColumn, + column: Column, config?: { supportTimeZone?: boolean }, ): ComponentAndProps => ({ component: DateTimeCell, props: getProps(column, config?.supportTimeZone ?? false), }), getInput: ( - column: TimeLikeColumn, + column: Column, config?: { supportTimeZone?: boolean }, ): ComponentAndProps< Omit @@ -63,10 +58,7 @@ const timeType: CellComponentFactory = { allowRelativePresets: true, }, }), - getDisplayFormatter( - column: TimeLikeColumn, - config?: { supportTimeZone?: boolean }, - ) { + getDisplayFormatter(column: Column, config?: { supportTimeZone?: boolean }) { const supportTimeZone = config?.supportTimeZone ?? false; return (v) => getProps(column, supportTimeZone).formatForDisplay(String(v)); }, diff --git a/mathesar_ui/src/components/cell-fabric/utils.ts b/mathesar_ui/src/components/cell-fabric/utils.ts index a97e119ab0..ddcff2edf7 100644 --- a/mathesar_ui/src/components/cell-fabric/utils.ts +++ b/mathesar_ui/src/components/cell-fabric/utils.ts @@ -1,4 +1,4 @@ -import type { Column } from '@mathesar/api/rest/types/tables/columns'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { Table } from '@mathesar/api/rpc/tables'; import type { CellInfo } from '@mathesar/stores/abstract-types/types'; import type { RecordSummariesForSheet } from '@mathesar/stores/table-data/record-summaries/recordSummaryUtils'; diff --git a/mathesar_ui/src/packages/json-rpc-client-builder/index.ts b/mathesar_ui/src/packages/json-rpc-client-builder/index.ts index 066c9206d9..d8eb45d709 100644 --- a/mathesar_ui/src/packages/json-rpc-client-builder/index.ts +++ b/mathesar_ui/src/packages/json-rpc-client-builder/index.ts @@ -2,6 +2,7 @@ export { type RpcError } from './RpcError'; export { buildRpcApi, rpcMethodTypeContainer } from './builder'; export { batchSend, + runner, type RpcBatchResponse, type RpcRequest, type RpcResponse, diff --git a/mathesar_ui/src/packages/json-rpc-client-builder/requests.ts b/mathesar_ui/src/packages/json-rpc-client-builder/requests.ts index 4609dffa6f..0ad5cfd337 100644 --- a/mathesar_ui/src/packages/json-rpc-client-builder/requests.ts +++ b/mathesar_ui/src/packages/json-rpc-client-builder/requests.ts @@ -137,3 +137,18 @@ export function batchSend[]>( // TODO implement batch sending throw new Error('Not implemented'); } + +/** + * A factory function to builds a function that directly runs a specific RPC + * request. The built function will then accept all the props of the RPC method + * and run the request without needing to call `.run()` on it. + * + * This utility is useful when you want to define a function that runs a + * specific RPC method without having to spell out the type of the method + * parameters. + */ +export function runner( + method: (props: P) => RpcRequest, +): (props: P) => CancellablePromise { + return (props) => method(props).run(); +} diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte index cefec83868..3ce4b725a3 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte @@ -2,10 +2,10 @@ import { _ } from 'svelte-i18n'; import { router } from 'tinro'; - import { columnsApi } from '@mathesar/api/rest/columns'; import type { DataFile } from '@mathesar/api/rest/types/dataFiles'; - import type { Column } from '@mathesar/api/rest/types/tables/columns'; import { getAPI, postAPI } from '@mathesar/api/rest/utils/requestUtils'; + import { api } from '@mathesar/api/rpc'; + import type { Column } from '@mathesar/api/rpc/columns'; import type { Database } from '@mathesar/api/rpc/databases'; import type { Schema } from '@mathesar/api/rpc/schemas'; import type { Table } from '@mathesar/api/rpc/tables'; @@ -18,6 +18,7 @@ } from '@mathesar/components/form'; import InfoBox from '@mathesar/components/message-boxes/InfoBox.svelte'; import { iconDeleteMajor } from '@mathesar/icons'; + import { runner } from '@mathesar/packages/json-rpc-client-builder'; import { getImportPreviewPageUrl, getSchemaPageUrl, @@ -25,6 +26,7 @@ } from '@mathesar/routes/urls'; import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; import AsyncStore from '@mathesar/stores/AsyncStore'; + import { currentDatabase } from '@mathesar/stores/databases'; import { currentTables } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; import { @@ -76,7 +78,7 @@ return postAPI(`/api/db/v0/tables/${table.oid}/previews/`, { columns }); } - const columnsFetch = new AsyncStore(columnsApi.list); + const columnsFetch = new AsyncStore(runner(api.columns.list_with_metadata)); const previewRequest = new AsyncStore(generateTablePreview); const typeSuggestionsRequest = new AsyncStore(getTypeSuggestionsForTable); const headerUpdate = makeHeaderUpdateRequest(); @@ -104,8 +106,11 @@ $: processedColumns = processColumns(columns, $currentDbAbstractTypes.data); async function init() { - const columnsResponse = await columnsFetch.run(table.oid); - const fetchedColumns = columnsResponse?.resolvedValue?.results; + const columnsResponse = await columnsFetch.run({ + database_id: $currentDatabase.id, + table_oid: table.oid, + }); + const fetchedColumns = columnsResponse?.resolvedValue; if (!fetchedColumns) { return; } diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewSheet.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewSheet.svelte index e2689764f4..724b44f34b 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewSheet.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewSheet.svelte @@ -1,5 +1,5 @@ diff --git a/mathesar_ui/src/pages/table/TablePage.svelte b/mathesar_ui/src/pages/table/TablePage.svelte index da4603c51e..865735b000 100644 --- a/mathesar_ui/src/pages/table/TablePage.svelte +++ b/mathesar_ui/src/pages/table/TablePage.svelte @@ -7,6 +7,7 @@ import LayoutWithHeader from '@mathesar/layouts/LayoutWithHeader.svelte'; import { makeSimplePageTitle } from '@mathesar/pages/pageTitleUtils'; import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { currentDatabase } from '@mathesar/stores/databases'; import { Meta, TabularData, @@ -36,10 +37,10 @@ $: ({ query } = $router); $: meta = Meta.fromSerialization(query[metaSerializationQueryKey] ?? ''); $: tabularData = new TabularData({ - id: table.oid, + database: $currentDatabase, + table, abstractTypesMap, meta, - table, shareConsumer, }); $: ({ isLoading, selection } = tabularData); @@ -73,7 +74,7 @@
- +
diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/boolean.ts b/mathesar_ui/src/stores/abstract-types/type-configs/boolean.ts index 730490ecc8..b42c7c2ceb 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/boolean.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/boolean.ts @@ -1,7 +1,8 @@ -import type { - BooleanDisplayOptions, - Column, -} from '@mathesar/api/rest/types/tables/columns'; +import { + type BooleanInputType, + type Column, + getColumnDisplayOption, +} from '@mathesar/api/rpc/columns'; import { iconUiTypeBoolean } from '@mathesar/icons'; import type { FormValues } from '@mathesar-component-library/types'; @@ -84,39 +85,31 @@ const displayForm: AbstractTypeConfigForm = { }; function determineDisplayOptions( - dispFormValues: FormValues, + formValues: FormValues, ): Column['display_options'] { const displayOptions: Column['display_options'] = { - input: dispFormValues.displayAs, + bool_input: formValues.displayAs as BooleanInputType, }; - if ( - dispFormValues.displayAs === 'dropdown' && - dispFormValues.useCustomLabels - ) { - displayOptions.custom_labels = { - TRUE: dispFormValues.trueLabel, - FALSE: dispFormValues.falseLabel, - }; + if (formValues.displayAs === 'dropdown' && formValues.useCustomLabels) { + displayOptions.bool_true = formValues.trueLabel as string; + displayOptions.bool_false = formValues.falseLabel as string; } return displayOptions; } function constructDisplayFormValuesFromDisplayOptions( - columnDisplayOpts: Column['display_options'], + displayOptions: Column['display_options'], ): FormValues { - const displayOptions = columnDisplayOpts as BooleanDisplayOptions | null; - const dispFormValues: FormValues = { - displayAs: displayOptions?.input ?? 'checkbox', + const column = { display_options: displayOptions }; + const formValues: FormValues = { + displayAs: getColumnDisplayOption(column, 'bool_input'), }; - if ( - typeof displayOptions?.custom_labels === 'object' && - displayOptions.custom_labels !== null - ) { - dispFormValues.useCustomLabels = true; - dispFormValues.trueLabel = displayOptions.custom_labels.TRUE; - dispFormValues.falseLabel = displayOptions.custom_labels.FALSE; + if (displayOptions?.bool_true || displayOptions?.bool_false) { + formValues.useCustomLabels = true; + formValues.trueLabel = getColumnDisplayOption(column, 'bool_true'); + formValues.falseLabel = getColumnDisplayOption(column, 'bool_false'); } - return dispFormValues; + return formValues; } const booleanType: AbstractTypeConfiguration = { diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/comboTypes/arrayFactory.ts b/mathesar_ui/src/stores/abstract-types/type-configs/comboTypes/arrayFactory.ts index 3238fdfdc3..6f1ef295de 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/comboTypes/arrayFactory.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/comboTypes/arrayFactory.ts @@ -1,4 +1,3 @@ -import type { ArrayTypeOptions } from '@mathesar/api/rest/types/tables/columns'; import { iconUiTypeArray } from '@mathesar/icons'; import type { AbstractTypeConfigurationFactory } from '../../types'; @@ -7,19 +6,12 @@ import { getAbstractTypeForDbType } from '../../utils'; const arrayFactory: AbstractTypeConfigurationFactory = (map) => ({ getIcon: (args) => { const arrayIcon = { ...iconUiTypeArray, label: 'Array' }; - if (args && args.typeOptions) { - const typeOpts = args.typeOptions as ArrayTypeOptions; - const innerAbstractType = getAbstractTypeForDbType( - typeOpts.item_type, - map, - ); - if (innerAbstractType) { - const innerIcon = innerAbstractType.getIcon(); - const innerIcons = Array.isArray(innerIcon) ? innerIcon : [innerIcon]; - return [{ ...iconUiTypeArray, label: 'Array' }, ...innerIcons]; - } - } - return arrayIcon; + const itemType = args?.typeOptions?.item_type ?? undefined; + if (!itemType) return arrayIcon; + const innerAbstractType = getAbstractTypeForDbType(itemType, map); + const innerIcon = innerAbstractType.getIcon(); + const innerIcons = Array.isArray(innerIcon) ? innerIcon : [innerIcon]; + return [arrayIcon, ...innerIcons]; }, cellInfo: { type: 'array', diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/date.ts b/mathesar_ui/src/stores/abstract-types/type-configs/date.ts index b31d33175e..9e1fa9ee47 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/date.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/date.ts @@ -1,8 +1,8 @@ -import type { - Column, - DateDisplayOptions, - DateFormat, -} from '@mathesar/api/rest/types/tables/columns'; +import { + type Column, + type DateFormat, + getColumnDisplayOption, +} from '@mathesar/api/rpc/columns'; import { iconUiTypeDate } from '@mathesar/icons'; import type { FormValues } from '@mathesar-component-library/types'; @@ -35,22 +35,21 @@ const displayForm: AbstractTypeConfigForm = { }; function determineDisplayOptions( - dispFormValues: FormValues, + formValues: FormValues, ): Column['display_options'] { - const displayOptions: DateDisplayOptions = { - format: dispFormValues.format as DateFormat, + return { + date_format: formValues.format as DateFormat, }; - return displayOptions; } function constructDisplayFormValuesFromDisplayOptions( - columnDisplayOpts: Column['display_options'], + displayOptions: Column['display_options'], ): FormValues { - const displayOptions = columnDisplayOpts as DateDisplayOptions | null; - const dispFormValues: FormValues = { - format: displayOptions?.format ?? 'none', + const column = { display_options: displayOptions }; + const formValues: FormValues = { + format: getColumnDisplayOption(column, 'date_format'), }; - return dispFormValues; + return formValues; } const dateType: AbstractTypeConfiguration = { diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/datetime.ts b/mathesar_ui/src/stores/abstract-types/type-configs/datetime.ts index 43e1771aa6..1f366a2740 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/datetime.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/datetime.ts @@ -1,9 +1,9 @@ -import type { - Column, - DateFormat, - TimeFormat, - TimeStampDisplayOptions, -} from '@mathesar/api/rest/types/tables/columns'; +import { + type Column, + type DateFormat, + type TimeFormat, + getColumnDisplayOption, +} from '@mathesar/api/rpc/columns'; import { iconUiTypeDateTime } from '@mathesar/icons'; import type { FormValues } from '@mathesar-component-library/types'; @@ -94,7 +94,7 @@ const displayForm: AbstractTypeConfigForm = { function determineDisplayOptions( dispFormValues: FormValues, ): Column['display_options'] { - const displayOptions: TimeStampDisplayOptions = { + const displayOptions: Column['display_options'] = { date_format: dispFormValues.dateFormat as DateFormat, time_format: dispFormValues.timeFormat as TimeFormat, }; @@ -102,14 +102,14 @@ function determineDisplayOptions( } function constructDisplayFormValuesFromDisplayOptions( - columnDisplayOpts: Column['display_options'], + displayOptions: Column['display_options'], ): FormValues { - const displayOptions = columnDisplayOpts as TimeStampDisplayOptions | null; - const dispFormValues: FormValues = { - dateFormat: displayOptions?.date_format ?? 'none', - timeFormat: displayOptions?.time_format ?? '24hr', + const column = { display_options: displayOptions }; + const formValues: FormValues = { + dateFormat: getColumnDisplayOption(column, 'date_format'), + timeFormat: getColumnDisplayOption(column, 'time_format'), }; - return dispFormValues; + return formValues; } const dateTimeType: AbstractTypeConfiguration = { diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/duration.ts b/mathesar_ui/src/stores/abstract-types/type-configs/duration.ts index fc546fa12e..da7d4fb81f 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/duration.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/duration.ts @@ -1,7 +1,4 @@ -import type { - Column, - DurationDisplayOptions, -} from '@mathesar/api/rest/types/tables/columns'; +import { type Column, getColumnDisplayOption } from '@mathesar/api/rpc/columns'; import { iconUiTypeDuration } from '@mathesar/icons'; import { DurationSpecification } from '@mathesar/utils/duration'; import type { FormValues } from '@mathesar-component-library/types'; @@ -43,26 +40,26 @@ const displayForm: AbstractTypeConfigForm = { }; function determineDisplayOptions( - dispFormValues: FormValues, + formValues: FormValues, ): Column['display_options'] { const displayOptions: Column['display_options'] = { - ...(dispFormValues.durationConfig as Record), - show_units: false, + ...(formValues.durationConfig as Record), + duration_show_units: false, }; return displayOptions; } function constructDisplayFormValuesFromDisplayOptions( - columnDisplayOpts: Column['display_options'], + displayOptions: Column['display_options'], ): FormValues { - const displayOptions = columnDisplayOpts as DurationDisplayOptions | null; - const dispFormValues: FormValues = { + const column = { display_options: displayOptions }; + const formValues: FormValues = { durationConfig: { - max: displayOptions?.max ?? durationDefaults.max, - min: displayOptions?.min ?? durationDefaults.min, + max: getColumnDisplayOption(column, 'duration_max'), + min: getColumnDisplayOption(column, 'duration_min'), }, }; - return dispFormValues; + return formValues; } const durationType: AbstractTypeConfiguration = { diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/money.ts b/mathesar_ui/src/stores/abstract-types/type-configs/money.ts index 83da2fb45b..789dbea426 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/money.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/money.ts @@ -1,8 +1,10 @@ -import type { - Column, - MoneyDisplayOptions, - NumberFormat, -} from '@mathesar/api/rest/types/tables/columns'; +import { + type Column, + type CurrencyLocation, + type NumberFormat, + type NumberGrouping, + getColumnDisplayOption, +} from '@mathesar/api/rpc/columns'; import { iconUiTypeMoney } from '@mathesar/icons'; import type { FormValues } from '@mathesar-component-library/types'; @@ -93,41 +95,43 @@ const displayForm: AbstractTypeConfigForm = { }; interface MoneyFormValues extends Record { - currencySymbol: MoneyDisplayOptions['currency_symbol']; - decimalPlaces: MoneyDisplayOptions['minimum_fraction_digits']; - currencySymbolLocation: MoneyDisplayOptions['currency_symbol_location']; + currencySymbol: string; + decimalPlaces: number | null; + currencySymbolLocation: CurrencyLocation; numberFormat: NumberFormat | 'none'; - useGrouping: MoneyDisplayOptions['use_grouping']; + useGrouping: NumberGrouping; } function determineDisplayOptions(form: FormValues): Column['display_options'] { const f = form as MoneyFormValues; - const opts: Partial = { - currency_symbol: f.currencySymbol, - currency_symbol_location: f.currencySymbolLocation, - number_format: f.numberFormat === 'none' ? null : f.numberFormat, - use_grouping: f.useGrouping, - minimum_fraction_digits: f.decimalPlaces ?? undefined, - maximum_fraction_digits: f.decimalPlaces ?? undefined, + const opts: Partial = { + mon_currency_symbol: f.currencySymbol, + mon_currency_location: f.currencySymbolLocation, + num_format: f.numberFormat === 'none' ? null : f.numberFormat, + num_grouping: f.useGrouping, + num_min_frac_digits: f.decimalPlaces ?? undefined, + num_max_frac_digits: f.decimalPlaces ?? undefined, }; return opts; } function constructDisplayFormValuesFromDisplayOptions( - columnDisplayOpts: Column['display_options'], + displayOptions: Column['display_options'], ): MoneyFormValues { - const displayOptions = columnDisplayOpts as MoneyDisplayOptions | null; + const column = { display_options: displayOptions }; const decimalPlaces = getDecimalPlaces( - displayOptions?.minimum_fraction_digits ?? null, - displayOptions?.maximum_fraction_digits ?? null, + displayOptions?.num_min_frac_digits ?? null, + displayOptions?.num_max_frac_digits ?? null, ); const displayFormValues: MoneyFormValues = { - numberFormat: displayOptions?.number_format ?? 'none', - currencySymbol: displayOptions?.currency_symbol ?? '', + numberFormat: getColumnDisplayOption(column, 'num_format') ?? 'none', + currencySymbol: getColumnDisplayOption(column, 'mon_currency_symbol') ?? '', decimalPlaces, - currencySymbolLocation: - displayOptions?.currency_symbol_location ?? 'after-minus', - useGrouping: displayOptions?.use_grouping ?? 'true', + currencySymbolLocation: getColumnDisplayOption( + column, + 'mon_currency_location', + ), + useGrouping: getColumnDisplayOption(column, 'num_grouping'), }; return displayFormValues; } diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/number.ts b/mathesar_ui/src/stores/abstract-types/type-configs/number.ts index 423b344e25..0986c592ce 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/number.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/number.ts @@ -1,8 +1,9 @@ -import type { - Column, - NumberDisplayOptions, - NumberFormat, -} from '@mathesar/api/rest/types/tables/columns'; +import { + type Column, + type NumberFormat, + type NumberGrouping, + getColumnDisplayOption, +} from '@mathesar/api/rpc/columns'; import type { DbType } from '@mathesar/AppTypes'; import { iconUiTypeNumber } from '@mathesar/icons'; import type { FormValues } from '@mathesar-component-library/types'; @@ -148,10 +149,10 @@ function determineDbTypeAndOptions( if (dbType === DB_TYPES.DECIMAL || dbType === DB_TYPES.NUMERIC) { if (dbFormValues.maxDigits !== null) { - typeOptions.precision = dbFormValues.maxDigits; + typeOptions.precision = Number(dbFormValues.maxDigits); } if (dbFormValues.decimalPlaces !== null) { - typeOptions.scale = dbFormValues.decimalPlaces; + typeOptions.scale = Number(dbFormValues.decimalPlaces); } } @@ -257,17 +258,15 @@ function determineDisplayOptions( formValues: FormValues, ): Column['display_options'] { const decimalPlaces = formValues.decimalPlaces as number | null; - const opts: Partial = { - number_format: + const opts: Partial = { + num_format: formValues.numberFormat === 'none' ? undefined : (formValues.numberFormat as NumberFormat), - use_grouping: - (formValues.useGrouping as - | NumberDisplayOptions['use_grouping'] - | undefined) ?? 'false', - minimum_fraction_digits: decimalPlaces ?? undefined, - maximum_fraction_digits: decimalPlaces ?? undefined, + num_grouping: + (formValues.useGrouping as NumberGrouping | undefined) ?? 'auto', + num_min_frac_digits: decimalPlaces ?? undefined, + num_max_frac_digits: decimalPlaces ?? undefined, }; return opts; } @@ -289,16 +288,16 @@ export function getDecimalPlaces( } function constructDisplayFormValuesFromDisplayOptions( - columnDisplayOpts: Column['display_options'], + displayOptions: Column['display_options'], ): FormValues { - const displayOptions = columnDisplayOpts as NumberDisplayOptions | null; + const column = { display_options: displayOptions }; const decimalPlaces = getDecimalPlaces( - displayOptions?.minimum_fraction_digits ?? null, - displayOptions?.maximum_fraction_digits ?? null, + displayOptions?.num_min_frac_digits ?? null, + displayOptions?.num_max_frac_digits ?? null, ); const formValues: FormValues = { - numberFormat: displayOptions?.number_format ?? 'none', - useGrouping: displayOptions?.use_grouping ?? 'false', + numberFormat: getColumnDisplayOption(column, 'num_format'), + useGrouping: getColumnDisplayOption(column, 'num_grouping'), decimalPlaces, }; return formValues; diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/text.ts b/mathesar_ui/src/stores/abstract-types/type-configs/text.ts index fecf3c0b87..5dbd4fdb56 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/text.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/text.ts @@ -1,4 +1,4 @@ -import type { Column } from '@mathesar/api/rest/types/tables/columns'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { DbType } from '@mathesar/AppTypes'; import { iconUiTypeText } from '@mathesar/icons'; import type { FormValues } from '@mathesar-component-library/types'; @@ -76,7 +76,7 @@ function determineDbTypeAndOptions( const dbType = determineDbType(dbFormValues, columnType); const typeOptions: Column['type_options'] = {}; if (dbType === DB_TYPES.CHAR || dbType === DB_TYPES.VARCHAR) { - typeOptions.length = dbFormValues.length; + typeOptions.length = Number(dbFormValues.length); } return { dbType, diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/time.ts b/mathesar_ui/src/stores/abstract-types/type-configs/time.ts index 8c0905dc44..42e739c7c3 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/time.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/time.ts @@ -1,8 +1,8 @@ -import type { - Column, - TimeDisplayOptions, - TimeFormat, -} from '@mathesar/api/rest/types/tables/columns'; +import { + type Column, + type TimeFormat, + getColumnDisplayOption, +} from '@mathesar/api/rpc/columns'; import { iconUiTypeTime } from '@mathesar/icons'; import type { FormValues } from '@mathesar-component-library/types'; @@ -80,22 +80,20 @@ const displayForm: AbstractTypeConfigForm = { }; function determineDisplayOptions( - dispFormValues: FormValues, + formValues: FormValues, ): Column['display_options'] { - const displayOptions: TimeDisplayOptions = { - format: dispFormValues.format as TimeFormat, + return { + time_format: formValues.format as TimeFormat, }; - return displayOptions; } function constructDisplayFormValuesFromDisplayOptions( - columnDisplayOpts: Column['display_options'], + displayOptions: Column['display_options'], ): FormValues { - const displayOptions = columnDisplayOpts as TimeDisplayOptions | null; - const dispFormValues: FormValues = { - format: displayOptions?.format ?? '24hr', + const column = { display_options: displayOptions }; + return { + format: getColumnDisplayOption(column, 'time_format'), }; - return dispFormValues; } const timeType: AbstractTypeConfiguration = { diff --git a/mathesar_ui/src/stores/abstract-types/type-configs/utils.ts b/mathesar_ui/src/stores/abstract-types/type-configs/utils.ts index bb3774db89..ab126c1d0e 100644 --- a/mathesar_ui/src/stores/abstract-types/type-configs/utils.ts +++ b/mathesar_ui/src/stores/abstract-types/type-configs/utils.ts @@ -1,7 +1,4 @@ -import type { - DateFormat, - TimeFormat, -} from '@mathesar/api/rest/types/tables/columns'; +import type { DateFormat, TimeFormat } from '@mathesar/api/rpc/columns'; import { DateTimeSpecification } from '@mathesar/utils/date-time'; import { dayjs } from '@mathesar-component-library'; diff --git a/mathesar_ui/src/stores/abstract-types/types.ts b/mathesar_ui/src/stores/abstract-types/types.ts index c05a43ed8b..6f04eaf055 100644 --- a/mathesar_ui/src/stores/abstract-types/types.ts +++ b/mathesar_ui/src/stores/abstract-types/types.ts @@ -1,6 +1,6 @@ import type { QuerySummarizationFunctionId } from '@mathesar/api/rest/types/queries'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { States } from '@mathesar/api/rest/utils/requestUtils'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { DbType } from '@mathesar/AppTypes'; import type { CellDataType } from '@mathesar/components/cell-fabric/data-types/typeDefinitions'; import type { diff --git a/mathesar_ui/src/stores/table-data/TableStructure.ts b/mathesar_ui/src/stores/table-data/TableStructure.ts index 0289a340c2..12444c7fa6 100644 --- a/mathesar_ui/src/stores/table-data/TableStructure.ts +++ b/mathesar_ui/src/stores/table-data/TableStructure.ts @@ -1,8 +1,10 @@ import type { Readable } from 'svelte/store'; import { derived } from 'svelte/store'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import { States } from '@mathesar/api/rest/utils/requestUtils'; +import type { Column } from '@mathesar/api/rpc/columns'; +import type { Database } from '@mathesar/api/rpc/databases'; +import type { Table } from '@mathesar/api/rpc/tables'; import type { DBObjectEntry } from '@mathesar/AppTypes'; import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; @@ -13,12 +15,13 @@ import type { ProcessedColumnsStore } from './processedColumns'; import { processColumn } from './processedColumns'; export interface TableStructureProps { - id: DBObjectEntry['id']; + database: Pick; + table: Pick; abstractTypesMap: AbstractTypesMap; } export class TableStructure { - id: DBObjectEntry['id']; + oid: DBObjectEntry['id']; columnsDataStore: ColumnsDataStore; @@ -29,9 +32,12 @@ export class TableStructure { isLoading: Readable; constructor(props: TableStructureProps) { - this.id = props.id; - this.columnsDataStore = new ColumnsDataStore({ tableId: this.id }); - this.constraintsDataStore = new ConstraintsDataStore({ tableId: this.id }); + this.oid = props.table.oid; + this.columnsDataStore = new ColumnsDataStore({ + database: props.database, + tableOid: this.oid, + }); + this.constraintsDataStore = new ConstraintsDataStore({ tableId: this.oid }); this.processedColumns = derived( [this.columnsDataStore.columns, this.constraintsDataStore], ([columns, constraintsData]) => @@ -39,7 +45,7 @@ export class TableStructure { columns.map((column, columnIndex) => [ column.id, processColumn({ - tableId: this.id, + tableId: this.oid, column, columnIndex, constraints: constraintsData.constraints, diff --git a/mathesar_ui/src/stores/table-data/columns.ts b/mathesar_ui/src/stores/table-data/columns.ts index b76dded669..58b9bdfdc8 100644 --- a/mathesar_ui/src/stores/table-data/columns.ts +++ b/mathesar_ui/src/stores/table-data/columns.ts @@ -1,17 +1,13 @@ import { type Readable, derived, writable } from 'svelte/store'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; +import type { RequestStatus } from '@mathesar/api/rest/utils/requestUtils'; +import { api } from '@mathesar/api/rpc'; import type { - PaginatedResponse, - RequestStatus, -} from '@mathesar/api/rest/utils/requestUtils'; -import { - addQueryParamsToUrl, - deleteAPI, - getAPI, - patchAPI, - postAPI, -} from '@mathesar/api/rest/utils/requestUtils'; + Column, + ColumnCreationSpec, + ColumnPatchSpec, +} from '@mathesar/api/rpc/columns'; +import type { Database } from '@mathesar/api/rpc/databases'; import type { Table } from '@mathesar/api/rpc/tables'; import { getErrorMessage } from '@mathesar/utils/errors'; import type { ShareConsumer } from '@mathesar/utils/shares'; @@ -21,36 +17,18 @@ import { WritableSet, } from '@mathesar-component-library'; -function api(url: string) { - return { - get(queryParams: Record) { - const requestUrl = addQueryParamsToUrl(url, queryParams); - return getAPI>(requestUrl); - }, - add(columnDetails: Partial) { - return postAPI>(url, columnDetails); - }, - remove(id: Column['id']) { - return deleteAPI(`${url}${id}/`); - }, - update(id: Column['id'], data: Partial) { - return patchAPI>(`${url}${id}/`, data); - }, - }; -} - export class ColumnsDataStore extends EventHandler<{ - columnRenamed: number; - columnAdded: Partial; - columnDeleted: number; - columnPatched: Partial; - columnsFetched: Column[]; + columnRenamed: void; + columnAdded: void; + columnDeleted: Column['id']; + columnPatched: void; }> { - private tableId: Table['oid']; - - private promise: CancellablePromise> | undefined; + private apiContext: { + database_id: number; + table_oid: Table['oid']; + }; - private api: ReturnType; + private promise: CancellablePromise | undefined; private fetchedColumns = writable([]); @@ -66,19 +44,20 @@ export class ColumnsDataStore extends EventHandler<{ readonly shareConsumer?: ShareConsumer; constructor({ - tableId, + database, + tableOid, hiddenColumns, shareConsumer, }: { - tableId: Table['oid']; + database: Pick; + tableOid: Table['oid']; /** Values are column ids */ hiddenColumns?: Iterable; shareConsumer?: ShareConsumer; }) { super(); - this.tableId = tableId; + this.apiContext = { database_id: database.id, table_oid: tableOid }; this.shareConsumer = shareConsumer; - this.api = api(`/api/db/v0/tables/${this.tableId}/columns/`); this.hiddenColumns = new WritableSet(hiddenColumns); this.columns = derived( [this.fetchedColumns, this.hiddenColumns], @@ -94,15 +73,16 @@ export class ColumnsDataStore extends EventHandler<{ try { this.fetchStatus.set({ state: 'processing' }); this.promise?.cancel(); - this.promise = this.api.get({ - limit: 500, - ...this.shareConsumer?.getQueryParams(), - }); - const response = await this.promise; - const columns = response.results; + // TODO_BETA: For some reason `...this.shareConsumer?.getQueryParams()` + // was getting passed into the API call when it was REST. I don't know + // why. We need to figure out if this is necessary to replicate for the + // RPC call. + this.promise = api.columns + .list_with_metadata({ ...this.apiContext }) + .run(); + const columns = await this.promise; this.fetchedColumns.set(columns); this.fetchStatus.set({ state: 'success' }); - await this.dispatch('columnsFetched', columns); return columns; } catch (e) { this.fetchStatus.set({ state: 'failure', errors: [getErrorMessage(e)] }); @@ -112,33 +92,30 @@ export class ColumnsDataStore extends EventHandler<{ } } - async add(columnDetails: Partial): Promise> { - const column = await this.api.add(columnDetails); - await this.dispatch('columnAdded', column); + async add(columnDetails: ColumnCreationSpec): Promise { + await api.columns + .add({ ...this.apiContext, column_data_list: [columnDetails] }) + .run(); + await this.dispatch('columnAdded'); await this.fetch(); - return column; } - async rename(id: Column['id'], newName: string): Promise { - await this.api.update(id, { name: newName }); - await this.dispatch('columnRenamed', id); + async rename(id: Column['id'], name: string): Promise { + await api.columns + .patch({ ...this.apiContext, column_data_list: [{ id, name }] }) + .run(); + await this.dispatch('columnRenamed'); } async updateDescription( id: Column['id'], description: string | null, ): Promise { - await this.api.update(id, { description }); + await api.columns + .patch({ ...this.apiContext, column_data_list: [{ id, description }] }) + .run(); this.fetchedColumns.update((columns) => - columns.map((column) => { - if (column.id === id) { - return { - ...column, - description, - }; - } - return column; - }), + columns.map((c) => (c.id === id ? { ...c, description } : c)), ); } @@ -151,23 +128,24 @@ export class ColumnsDataStore extends EventHandler<{ `Column "${column.name}" cannot allow NULL because it is a primary key.`, ); } - await this.api.update(column.id, { nullable }); + await api.columns + .patch({ + ...this.apiContext, + column_data_list: [{ id: column.id, nullable }], + }) + .run(); await this.fetch(); } - // TODO: Analyze: Might be cleaner to move following functions as a property of Column class - // but are the object instantiations worth it? - - async patch( - columnId: Column['id'], - properties: Omit, 'id'>, - ): Promise> { - const column = await this.api.update(columnId, { - ...properties, - }); + async patch(patchSpec: ColumnPatchSpec): Promise { + await api.columns + .patch({ + ...this.apiContext, + column_data_list: [patchSpec], + }) + .run(); await this.fetch(); - await this.dispatch('columnPatched', column); - return column; + await this.dispatch('columnPatched'); } destroy(): void { @@ -177,7 +155,9 @@ export class ColumnsDataStore extends EventHandler<{ } async deleteColumn(columnId: Column['id']): Promise { - await this.api.remove(columnId); + await api.columns + .delete({ ...this.apiContext, column_attnums: [columnId] }) + .run(); await this.dispatch('columnDeleted', columnId); await this.fetch(); } diff --git a/mathesar_ui/src/stores/table-data/constraints.ts b/mathesar_ui/src/stores/table-data/constraints.ts index ffb5de065f..64b9a05a7b 100644 --- a/mathesar_ui/src/stores/table-data/constraints.ts +++ b/mathesar_ui/src/stores/table-data/constraints.ts @@ -7,7 +7,6 @@ import type { } from 'svelte/store'; import { derived, get as getStoreValue, writable } from 'svelte/store'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { Constraint as ApiConstraint } from '@mathesar/api/rest/types/tables/constraints'; import type { PaginatedResponse } from '@mathesar/api/rest/utils/requestUtils'; import { @@ -17,6 +16,7 @@ import { getAPI, postAPI, } from '@mathesar/api/rest/utils/requestUtils'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { Table } from '@mathesar/api/rpc/tables'; import type { ShareConsumer } from '@mathesar/utils/shares'; import type { CancellablePromise } from '@mathesar-component-library'; diff --git a/mathesar_ui/src/stores/table-data/constraintsUtils.ts b/mathesar_ui/src/stores/table-data/constraintsUtils.ts index 31f616a838..c5a033b823 100644 --- a/mathesar_ui/src/stores/table-data/constraintsUtils.ts +++ b/mathesar_ui/src/stores/table-data/constraintsUtils.ts @@ -1,8 +1,8 @@ -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { Constraint, FkConstraint, } from '@mathesar/api/rest/types/tables/constraints'; +import type { Column } from '@mathesar/api/rpc/columns'; export function constraintIsFk(c: Constraint): c is FkConstraint { return c.type === 'foreignkey'; diff --git a/mathesar_ui/src/stores/table-data/processedColumns.ts b/mathesar_ui/src/stores/table-data/processedColumns.ts index ab1ccad0c5..036aae5b8f 100644 --- a/mathesar_ui/src/stores/table-data/processedColumns.ts +++ b/mathesar_ui/src/stores/table-data/processedColumns.ts @@ -1,7 +1,7 @@ import type { Readable } from 'svelte/store'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { Constraint } from '@mathesar/api/rest/types/tables/constraints'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { Table } from '@mathesar/api/rpc/tables'; import type { CellColumnFabric } from '@mathesar/components/cell-fabric/types'; import { diff --git a/mathesar_ui/src/stores/table-data/records.ts b/mathesar_ui/src/stores/table-data/records.ts index fbc4443993..f536769732 100644 --- a/mathesar_ui/src/stores/table-data/records.ts +++ b/mathesar_ui/src/stores/table-data/records.ts @@ -7,7 +7,6 @@ import { writable, } from 'svelte/store'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { GetRequestParams as ApiGetRequestParams, Group as ApiGroup, @@ -24,6 +23,7 @@ import { patchAPI, postAPI, } from '@mathesar/api/rest/utils/requestUtils'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { Table } from '@mathesar/api/rpc/tables'; import { getErrorMessage } from '@mathesar/utils/errors'; import { pluralize } from '@mathesar/utils/languageUtils'; diff --git a/mathesar_ui/src/stores/table-data/tabularData.ts b/mathesar_ui/src/stores/table-data/tabularData.ts index cead9d366b..c8c719f750 100644 --- a/mathesar_ui/src/stores/table-data/tabularData.ts +++ b/mathesar_ui/src/stores/table-data/tabularData.ts @@ -1,10 +1,10 @@ import { getContext, setContext } from 'svelte'; import { type Readable, type Writable, derived, writable } from 'svelte/store'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import { States } from '@mathesar/api/rest/utils/requestUtils'; +import type { Column } from '@mathesar/api/rpc/columns'; +import type { Database } from '@mathesar/api/rpc/databases'; import type { Table } from '@mathesar/api/rpc/tables'; -import type { DBObjectEntry } from '@mathesar/AppTypes'; import Plane from '@mathesar/components/sheet/selection/Plane'; import Series from '@mathesar/components/sheet/selection/Series'; import SheetSelectionStore from '@mathesar/components/sheet/selection/SheetSelectionStore'; @@ -23,9 +23,9 @@ import type { TableRecordsData } from './records'; import { RecordsData } from './records'; export interface TabularDataProps { - id: DBObjectEntry['id']; - abstractTypesMap: AbstractTypesMap; + database: Pick; table: Table; + abstractTypesMap: AbstractTypesMap; meta?: Meta; shareConsumer?: ShareConsumer; /** @@ -42,7 +42,7 @@ export interface TabularDataProps { } export class TabularData { - id: DBObjectEntry['id']; + table: Table; meta: Meta; @@ -60,27 +60,26 @@ export class TabularData { selection: SheetSelectionStore; - table: Table; - shareConsumer?: ShareConsumer; constructor(props: TabularDataProps) { const contextualFilters = props.contextualFilters ?? new Map(); - this.id = props.id; + this.table = props.table; this.meta = props.meta ?? new Meta(); this.shareConsumer = props.shareConsumer; this.columnsDataStore = new ColumnsDataStore({ - tableId: this.id, + database: props.database, + tableOid: this.table.oid, hiddenColumns: contextualFilters.keys(), shareConsumer: this.shareConsumer, }); this.constraintsDataStore = new ConstraintsDataStore({ - tableId: this.id, + tableId: this.table.oid, shareConsumer: this.shareConsumer, }); this.recordsData = new RecordsData({ - tableId: this.id, + tableId: this.table.oid, meta: this.meta, columnsDataStore: this.columnsDataStore, contextualFilters, @@ -102,7 +101,7 @@ export class TabularData { columns.map((column, columnIndex) => [ column.id, processColumn({ - tableId: this.id, + tableId: this.table.oid, column, columnIndex, constraints: constraintsData.constraints, diff --git a/mathesar_ui/src/stores/table-data/utils.ts b/mathesar_ui/src/stores/table-data/utils.ts index 7557f7f9c8..7eac7ed9fd 100644 --- a/mathesar_ui/src/stores/table-data/utils.ts +++ b/mathesar_ui/src/stores/table-data/utils.ts @@ -1,8 +1,8 @@ import { concat } from 'iter-tools'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { RequestStatus } from '@mathesar/api/rest/utils/requestUtils'; import { getMostImportantRequestStatusState } from '@mathesar/api/rest/utils/requestUtils'; +import type { Column } from '@mathesar/api/rpc/columns'; import { ImmutableMap, ImmutableSet, diff --git a/mathesar_ui/src/stores/tables.ts b/mathesar_ui/src/stores/tables.ts index e09222aae2..92261d8eca 100644 --- a/mathesar_ui/src/stores/tables.ts +++ b/mathesar_ui/src/stores/tables.ts @@ -446,7 +446,12 @@ export function getJoinableTablesResult( tableId: number, maxDepth = 1, ): Promise { - throw new Error('Not implemented'); // TODO_BETA + return Promise.resolve({ + joinable_tables: [], + tables: {}, + columns: {}, + }); + // TODO_BETA: re-implement this with the RPC API. // return getAPI( // `/api/db/v0/tables/${tableId}/joinable_tables/?max_depth=${maxDepth}`, diff --git a/mathesar_ui/src/systems/data-explorer/urlSerializationUtils.ts b/mathesar_ui/src/systems/data-explorer/urlSerializationUtils.ts index 3551b2675e..a077f35d08 100644 --- a/mathesar_ui/src/systems/data-explorer/urlSerializationUtils.ts +++ b/mathesar_ui/src/systems/data-explorer/urlSerializationUtils.ts @@ -1,5 +1,5 @@ import type { QueryInstanceSummarizationTransformation } from '@mathesar/api/rest/types/queries'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { Table } from '@mathesar/api/rpc/tables'; import { getDataExplorerPageUrl } from '@mathesar/routes/urls'; import type { UnsavedQueryInstance } from '@mathesar/stores/queries'; diff --git a/mathesar_ui/src/systems/data-explorer/utils.ts b/mathesar_ui/src/systems/data-explorer/utils.ts index 7a3c77f1fd..b1e187f51c 100644 --- a/mathesar_ui/src/systems/data-explorer/utils.ts +++ b/mathesar_ui/src/systems/data-explorer/utils.ts @@ -1,3 +1,6 @@ +import type { Simplify } from 'type-fest'; +import type { SimplifyDeep } from 'type-fest/source/merge-deep'; + import type { QueryColumnMetaData, QueryGeneratedColumnSource, @@ -5,11 +8,11 @@ import type { QueryResultColumn, QueryRunResponse, } from '@mathesar/api/rest/types/queries'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { JoinableTablesResult, JpPath, } from '@mathesar/api/rest/types/tables/joinable_tables'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { Table } from '@mathesar/api/rpc/tables'; import type { CellColumnFabric } from '@mathesar/components/cell-fabric/types'; import { @@ -305,6 +308,10 @@ export function getTablesThatReferenceBaseTable( return references; } +// type T = SimplifyDeep & +// ProcessedQueryResultColumnSource & +// Partial>; + function processColumn( columnInfo: Pick & ProcessedQueryResultColumnSource & @@ -468,9 +475,14 @@ export function speculateColumnMetaData({ type_options: aggregation.function === 'distinct_aggregate_to_array' ? { - type: - updatedColumnsMetaData.get(aggregation.inputAlias) - ?.column.type ?? 'unknown', + // TODO_3704: Ask Pavish. + // `Column['type_options']` was previously typed loosely + // as `Record | null`. Now it's more + // strict and it doesn't have a `type` property. + // + // type: + // updatedColumnsMetaData.get(aggregation.inputAlias) + // ?.column.type ?? 'unknown', } : null, display_options: null, diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte index b30cd4b009..2305433c06 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorContent.svelte @@ -13,7 +13,6 @@ renderTransitiveRecordSummary, } from '@mathesar/stores/table-data/record-summaries/recordSummaryUtils'; import { getPkValueInRecord } from '@mathesar/stores/table-data/records'; - import { currentTablesData } from '@mathesar/stores/tables'; import { toast } from '@mathesar/stores/toast'; import { getErrorMessage } from '@mathesar/utils/errors'; import { Button, Icon, Spinner } from '@mathesar-component-library'; @@ -39,7 +38,7 @@ isLoading, columnsDataStore, recordsData, - id: tableId, + table, } = tabularData); $: ({ purpose: rowType } = controller); $: ({ columns, fetchStatus } = columnsDataStore); @@ -56,7 +55,10 @@ controller.submit(result); } else if ($rowType === 'navigation') { const { recordId } = result; - const recordPageUrl = $storeToGetRecordPageUrl({ tableId, recordId }); + const recordPageUrl = $storeToGetRecordPageUrl({ + tableId: table.oid, + recordId, + }); if (recordPageUrl) { router.goto(recordPageUrl); controller.cancel(); @@ -70,7 +72,7 @@ } async function submitNewRecord() { - const url = `/api/db/v0/tables/${tableId}/records/`; + const url = `/api/db/v0/tables/${table.oid}/records/`; const body = getDataForNewRecord(); try { isSubmittingNewRecord = true; @@ -78,8 +80,7 @@ const record = response.results[0]; const recordId = getPkValueInRecord(record, $columns); const previewData = response.preview_data ?? []; - const table = $currentTablesData.tablesMap.get(tableId); - const template = table?.metadata?.record_summary_template; + const template = table.metadata?.record_summary_template; // TODO_RS_TEMPLATE // // We need to change the logic here to account for the fact that sometimes diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorController.ts b/mathesar_ui/src/systems/record-selector/RecordSelectorController.ts index 933e5879df..6944a8e64a 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorController.ts +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorController.ts @@ -1,8 +1,8 @@ import { getContext, setContext } from 'svelte'; import { writable } from 'svelte/store'; -import type { Column } from '@mathesar/api/rest/types/tables/columns'; import type { Result as ApiRecord } from '@mathesar/api/rest/types/tables/records'; +import type { Column } from '@mathesar/api/rpc/columns'; import type { DBObjectEntry } from '@mathesar/AppTypes'; import type { RecordSelectorPurpose } from './recordSelectorUtils'; diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte index cf9a36368d..299d786088 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorTable.svelte @@ -1,8 +1,8 @@ -{#key id} +{#key oid} {#if usesVirtualList} import { _ } from 'svelte-i18n'; - import type { Table } from '@mathesar/api/rpc/tables'; import EntityPageHeader from '@mathesar/components/EntityPageHeader.svelte'; import ModificationStatus from '@mathesar/components/ModificationStatus.svelte'; import { iconInspector, iconTable } from '@mathesar/icons'; @@ -18,9 +17,8 @@ const tabularData = getTabularDataStoreFromContext(); export let context: TableActionsContext = 'page'; - export let table: Pick; - $: ({ id, meta, isLoading, display } = $tabularData); + $: ({ table, meta, isLoading, display } = $tabularData); $: ({ filtering, sorting, grouping, sheetState } = meta); $: ({ isTableInspectorVisible } = display); @@ -50,7 +48,7 @@
{#if context === 'page'} - +
- - + +
+
+
+ +
+
+
+
{database.name}
+
{database.server_id}
+
+
+
diff --git a/mathesar_ui/src/pages/home/HomePage.svelte b/mathesar_ui/src/pages/home/HomePage.svelte index 07a7fd0681..22acbadedd 100644 --- a/mathesar_ui/src/pages/home/HomePage.svelte +++ b/mathesar_ui/src/pages/home/HomePage.svelte @@ -43,24 +43,26 @@ - {makeSimplePageTitle($_('connections'))} + {makeSimplePageTitle($_('databases'))} -
+
- {$_('database_connections')} + {$_('databases')} {#if $databases.size}({$databases.size}){/if}
-
+
{#if $databases.size === 0} {:else} @@ -81,9 +83,9 @@ {/if} -

+ {filterQuery} {/if} -

+ {#if filteredDatabases.length} -
-
-
- - -
-
{$_('database_name')}{$_('actions')}
- - - {database.name} - -
- - - - - - - {#each filteredDatabases as database (database.id)} - - {/each} - -
{$_('database_name')}
+
+ {#each filteredDatabases as database (database.id)} + + {/each}
{/if} @@ -122,11 +115,10 @@ From 657e284e9e1881aa1041cd1ad70d0b931dc9aa7e Mon Sep 17 00:00:00 2001 From: pavish Date: Wed, 31 Jul 2024 00:56:36 +0530 Subject: [PATCH 0539/1141] Update text to Databases from Connections in home page --- mathesar_ui/src/components/AppHeader.svelte | 2 +- .../src/components/breadcrumb/BreadcrumbItem.svelte | 2 +- .../src/components/breadcrumb/DatabaseSelector.svelte | 6 +++--- mathesar_ui/src/i18n/languages/en/dict.json | 11 +++++------ mathesar_ui/src/pages/WelcomePage.svelte | 2 +- .../src/systems/databases/DatabasesEmptyState.svelte | 2 +- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mathesar_ui/src/components/AppHeader.svelte b/mathesar_ui/src/components/AppHeader.svelte index fb77c69754..8b58177a59 100644 --- a/mathesar_ui/src/components/AppHeader.svelte +++ b/mathesar_ui/src/components/AppHeader.svelte @@ -124,7 +124,7 @@ - {$_('database_connections')} + {$_('databases')} {#if $userProfile.isSuperUser} - {$_('connections')} + {$_('databases')}
diff --git a/mathesar_ui/src/components/breadcrumb/DatabaseSelector.svelte b/mathesar_ui/src/components/breadcrumb/DatabaseSelector.svelte index ea61f78e71..44948a5df4 100644 --- a/mathesar_ui/src/components/breadcrumb/DatabaseSelector.svelte +++ b/mathesar_ui/src/components/breadcrumb/DatabaseSelector.svelte @@ -29,12 +29,12 @@
- {$_('database_connections')} + {$_('databases')}
diff --git a/mathesar_ui/src/systems/databases/DatabasesEmptyState.svelte b/mathesar_ui/src/systems/databases/DatabasesEmptyState.svelte index 4e3bb0bfd2..050d701fe8 100644 --- a/mathesar_ui/src/systems/databases/DatabasesEmptyState.svelte +++ b/mathesar_ui/src/systems/databases/DatabasesEmptyState.svelte @@ -17,7 +17,7 @@
- {$_('no_database_connections_yet')} + {$_('no_databases_connected')}
{$_('setup_connections_help')} From af451f598e422cdf59cce39425a5ba3de2c1ae4b Mon Sep 17 00:00:00 2001 From: pavish Date: Wed, 31 Jul 2024 01:41:18 +0530 Subject: [PATCH 0540/1141] Styling improvements for header and cards --- mathesar_ui/src/components/Card.svelte | 9 +++++---- mathesar_ui/src/pages/home/DatabaseRow.svelte | 4 ++-- mathesar_ui/src/pages/home/HomePage.svelte | 10 ++++------ 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/mathesar_ui/src/components/Card.svelte b/mathesar_ui/src/components/Card.svelte index a087856d19..5988f778f1 100644 --- a/mathesar_ui/src/components/Card.svelte +++ b/mathesar_ui/src/components/Card.svelte @@ -96,7 +96,7 @@ cursor: pointer; overflow: hidden; height: 100%; - padding: var(--padding-v, var(--padding-v-internal)) 0; + padding: var(--Card__padding-v, var(--padding-v-internal)) 0; } .link:hover { border-color: var(--slate-500); @@ -109,11 +109,12 @@ height: var(--menu-trigger-size, auto); display: flex; align-items: center; - padding: 0 var(--padding-h, var(--padding-h-internal)); + padding: 0 var(--Card__padding-h, var(--padding-h-internal)); } .description:not(:empty) { - padding: var(--size-x-small) var(--padding-h, var(--padding-h-internal)) 0 - var(--padding-h, var(--padding-h-internal)); + padding: var(--size-x-small) + var(--Card__padding-h, var(--padding-h-internal)) 0 + var(--Card__padding-h, var(--padding-h-internal)); font-size: var(--text-size-base); } diff --git a/mathesar_ui/src/pages/home/DatabaseRow.svelte b/mathesar_ui/src/pages/home/DatabaseRow.svelte index 487fcc1db2..3aeccbb30f 100644 --- a/mathesar_ui/src/pages/home/DatabaseRow.svelte +++ b/mathesar_ui/src/pages/home/DatabaseRow.svelte @@ -11,7 +11,7 @@
@@ -31,7 +31,7 @@ .title { display: grid; width: 100%; - grid-template-columns: 4rem 1fr; + grid-template-columns: 3.5rem 1fr; } .icon-holder { display: flex; diff --git a/mathesar_ui/src/pages/home/HomePage.svelte b/mathesar_ui/src/pages/home/HomePage.svelte index 22acbadedd..ea011d28ee 100644 --- a/mathesar_ui/src/pages/home/HomePage.svelte +++ b/mathesar_ui/src/pages/home/HomePage.svelte @@ -49,10 +49,9 @@
@@ -117,20 +116,19 @@ diff --git a/mathesar_ui/src/components/breadcrumb/BreadcrumbItem.svelte b/mathesar_ui/src/components/breadcrumb/BreadcrumbItem.svelte index 7a49f316ee..33004b4a3e 100644 --- a/mathesar_ui/src/components/breadcrumb/BreadcrumbItem.svelte +++ b/mathesar_ui/src/components/breadcrumb/BreadcrumbItem.svelte @@ -6,7 +6,11 @@ import NameWithIcon from '@mathesar/components/NameWithIcon.svelte'; import SchemaName from '@mathesar/components/SchemaName.svelte'; import TableName from '@mathesar/components/TableName.svelte'; - import { iconConnection, iconExploration, iconRecord } from '@mathesar/icons'; + import { + iconConnectDatabase, + iconExploration, + iconRecord, + } from '@mathesar/icons'; import { HOME_URL, getDatabasePageUrl, @@ -33,7 +37,9 @@ {:else if item.type === 'database'} diff --git a/mathesar_ui/src/components/breadcrumb/DatabaseSelector.svelte b/mathesar_ui/src/components/breadcrumb/DatabaseSelector.svelte index 44948a5df4..261a33469d 100644 --- a/mathesar_ui/src/components/breadcrumb/DatabaseSelector.svelte +++ b/mathesar_ui/src/components/breadcrumb/DatabaseSelector.svelte @@ -2,7 +2,7 @@ import { _ } from 'svelte-i18n'; import type { Database } from '@mathesar/api/rpc/databases'; - import { iconConnection, iconDatabase } from '@mathesar/icons'; + import { iconConnectDatabase, iconDatabase } from '@mathesar/icons'; import { HOME_URL, getDatabasePageUrl } from '@mathesar/routes/urls'; import { databasesStore } from '@mathesar/stores/databases'; @@ -36,7 +36,10 @@ type: 'simple', label: $_('manage_databases'), href: HOME_URL, - icon: iconConnection, + icon: { + ...iconConnectDatabase, + size: '1.4rem', + }, // TODO: Handle active states for persistent links isActive: () => false, }, diff --git a/mathesar_ui/src/icons/customIcons.ts b/mathesar_ui/src/icons/customIcons.ts index 47eb960714..393a5a244c 100644 --- a/mathesar_ui/src/icons/customIcons.ts +++ b/mathesar_ui/src/icons/customIcons.ts @@ -63,3 +63,61 @@ export const explorationIcon: IconProps['data'] = { ], ], }; + +export const createDatabaseIcon: IconProps['data'] = { + icon: [ + 24, + 24, + [], + '', + [ + 'M2.57143 12.4643C2.57143 12.7105 2.93365 13.1387 3.77394 13.5589C4.86104 14.1024 6.40318 14.4286 ' + + '8.07143 14.4286C9.73966 14.4286 11.2818 14.1024 12.3689 13.5589C13.2092 13.1387 13.5714 12.7105 ' + + '13.5714 12.4643V10.7583C12.275 11.5593 10.2929 12.0714 8.07143 12.0714C5.85 12.0714 3.86783 11.5593 ' + + '2.57143 10.7583V12.4643ZM13.5714 14.6868C12.275 15.4879 10.2929 16 8.07143 16C5.85 16 3.86783 ' + + '15.4879 2.57143 14.6868V16.3929C2.57143 16.6391 2.93365 17.0673 3.77394 17.4874C4.86104 18.031 ' + + '6.40318 18.3571 8.07143 18.3571C9.73966 18.3571 11.2818 18.031 12.3689 17.4874C13.2092 17.0673 ' + + '13.5714 16.6391 13.5714 16.3929V14.6868ZM1 16.3929V8.53571C1 6.58299 4.16599 5 8.07143 5C11.9769 ' + + '5 15.1429 6.58299 15.1429 8.53571V16.3929C15.1429 18.3456 11.9769 19.9286 8.07143 19.9286C4.16599 ' + + '19.9286 1 18.3456 1 16.3929ZM8.07143 10.5C9.73966 10.5 11.2818 10.1739 12.3689 9.63032C13.2092 ' + + '9.21017 13.5714 8.78197 13.5714 8.53571C13.5714 8.28946 13.2092 7.86126 12.3689 7.44111C11.2818 ' + + '6.89756 9.73966 6.57143 8.07143 6.57143C6.40318 6.57143 4.86104 6.89756 3.77394 7.44111C2.93365 ' + + '7.86126 2.57143 8.28946 2.57143 8.53571C2.57143 8.78197 2.93365 9.21017 3.77394 9.63032C4.86104 ' + + '10.1739 6.40318 10.5 8.07143 10.5Z', + 'M19.0714 9.51786V11.875H16.7143V13.4464H19.0714V15.8036H20.6429V13.4464H23V11.875H20.6429V9.51786H19.0714Z', + ], + ], +}; + +export const connectDatabaseIcon: IconProps['data'] = { + icon: [ + 24, + 24, + [], + '', + [ + 'M2.13666 11.9219C2.13666 12.1713 2.50345 12.6049 3.35432 13.0303C4.45512 13.5807 6.01668 13.911 ' + + '7.70595 13.911C9.39519 13.911 10.9567 13.5807 12.0575 13.0303C12.9085 12.6049 13.2752 12.1713 13.2752 ' + + '11.9219V10.1944C11.9625 11.0056 9.95538 11.5241 7.70595 11.5241C5.45654 11.5241 3.44939 11.0056 ' + + '2.13666 10.1944V11.9219ZM13.2752 14.1725C11.9625 14.9836 9.95538 15.5022 7.70595 15.5022C5.45654 ' + + '15.5022 3.44939 14.9836 2.13666 14.1725V15.9C2.13666 16.1494 2.50345 16.583 3.35432 17.0084C4.45512 ' + + '17.5588 6.01668 17.889 7.70595 17.889C9.39519 17.889 10.9567 17.5588 12.0575 17.0084C12.9085 16.583 ' + + '13.2752 16.1494 13.2752 15.9V14.1725ZM0.545441 15.9V7.94389C0.545441 5.96657 3.75131 4.36364 7.70595 ' + + '4.36364C11.6606 4.36364 14.8665 5.96657 14.8665 7.94389V15.9C14.8665 17.8773 11.6606 19.4803 7.70595 ' + + '19.4803C3.75131 19.4803 0.545441 17.8773 0.545441 15.9ZM7.70595 9.93292C9.39519 9.93292 10.9567 9.60268 ' + + '12.0575 9.05228C12.9085 8.62684 13.2752 8.19325 13.2752 7.94389C13.2752 7.69453 12.9085 7.26094 12.0575 ' + + '6.8355C10.9567 6.2851 9.39519 5.95486 7.70595 5.95486C6.01668 5.95486 4.45512 6.2851 3.35432 ' + + '6.8355C2.50345 7.26094 2.13666 7.69453 2.13666 7.94389C2.13666 8.19325 2.50345 8.62684 3.35432 ' + + '9.05228C4.45512 9.60268 6.01668 9.93292 7.70595 9.93292Z', + 'M17.2467 12.5676C17.1975 12.3604 17.1714 12.1442 17.1714 11.9219C17.1714 11.6997 17.1975 11.4836 ' + + '17.2466 11.2764L16.4577 10.8209L17.2533 9.44284L18.0428 9.89866C18.3556 9.6028 18.7369 9.37868 ' + + '19.1605 9.25262V8.3417H20.7517V9.25262C21.1753 9.37867 21.5565 9.60279 21.8693 9.89863L22.6588 ' + + '9.44277L23.4545 10.8208L22.6654 11.2763C22.7147 11.4836 22.7407 11.6997 22.7407 11.9219C22.7407 ' + + '12.1442 22.7147 12.3603 22.6655 12.5675L23.4545 13.023L22.6589 14.4011L21.8694 13.9452C21.5566 ' + + '14.2411 21.1753 14.4652 20.7518 14.5912V15.5022H19.1605V14.5913C18.7369 14.4653 18.3556 14.2412 ' + + '18.0429 13.9453L17.2533 14.4012L16.4577 13.0231L17.2467 12.5676ZM19.9561 13.1154C20.6151 13.1154 ' + + '21.1495 12.581 21.1495 11.9219C21.1495 11.2629 20.6151 10.7285 19.9561 10.7285C19.297 10.7285 ' + + '18.7626 11.2629 18.7626 11.9219C18.7626 12.581 19.297 13.1154 19.9561 13.1154Z', + ], + ], +}; diff --git a/mathesar_ui/src/icons/index.ts b/mathesar_ui/src/icons/index.ts index 880ab8b918..ed004cbb89 100644 --- a/mathesar_ui/src/icons/index.ts +++ b/mathesar_ui/src/icons/index.ts @@ -78,6 +78,8 @@ import type { IconProps } from '@mathesar-component-library/types'; import { arrayIcon, + connectDatabaseIcon, + createDatabaseIcon, explorationIcon, outcomeIcon, tableIcon, @@ -103,10 +105,12 @@ export const iconAddFilter: IconProps = { data: faFilter }; export const iconAddNew: IconProps = { data: faPlus }; export const iconAddUser: IconProps = { data: faUserPlus }; export const iconConfigure: IconProps = { data: faCogs }; +export const iconConnectDatabase = { data: connectDatabaseIcon }; export const iconCopyMajor: IconProps = { data: faCopy }; /** TODO: use faBinary once it's available (via newer FontAwesome version) */ export const iconCopyRawContent: IconProps = { data: faCopy }; export const iconCopyFormattedContent: IconProps = { data: faCopy }; +export const iconCreateDatabase = { data: createDatabaseIcon }; /** When you're deleting something significant or difficult to recover */ export const iconDeleteMajor: IconProps = { data: faTrashAlt }; /** When you're deleting something smaller or more ephemeral */ diff --git a/mathesar_ui/src/systems/databases/create-database/ConnectDatabaseModal.svelte b/mathesar_ui/src/systems/databases/create-database/ConnectDatabaseModal.svelte index aa3adf1d8a..3fda336b35 100644 --- a/mathesar_ui/src/systems/databases/create-database/ConnectDatabaseModal.svelte +++ b/mathesar_ui/src/systems/databases/create-database/ConnectDatabaseModal.svelte @@ -3,7 +3,7 @@ import { router } from 'tinro'; import type { Database } from '@mathesar/api/rpc/databases'; - import { iconDatabase } from '@mathesar/icons'; + import { iconConnectDatabase, iconCreateDatabase } from '@mathesar/icons'; import { getDatabasePageUrl } from '@mathesar/routes/urls'; import { ControlledModal, @@ -55,13 +55,13 @@ {#if view === 'base'}
setView('create')} /> setView('connect')} diff --git a/mathesar_ui/src/systems/databases/create-database/ConnectOption.svelte b/mathesar_ui/src/systems/databases/create-database/ConnectOption.svelte index 9cad5d0f40..a56f5ed138 100644 --- a/mathesar_ui/src/systems/databases/create-database/ConnectOption.svelte +++ b/mathesar_ui/src/systems/databases/create-database/ConnectOption.svelte @@ -12,7 +12,7 @@ diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index ffc63e4e3d..7843238c83 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -131,6 +131,7 @@ "database": "Database", "database_name": "Database Name", "database_not_found": "Database with id [connectionId] is not found.", + "database_permissions": "Database Permissions", "database_server_credentials": "Database server credentials", "database_type": "Database Type", "databases": "Databases", @@ -148,6 +149,7 @@ "delete_connection_db_delete_info": "If you would like to delete the database too, you will need to do so from outside Mathesar by deleting it directly in PostgreSQL.", "delete_connection_info": "The database will not be deleted and will still be accessible outside Mathesar. You may choose to reconnect to it in the future; however, upon disconnecting you will lose Mathesar-specific metadata such as saved explorations, customized column display options, and customized record summary templates.", "delete_connection_with_name": "Delete [connectionName] Database Connection?", + "delete_database": "Delete Database", "delete_exploration": "Delete Exploration", "delete_import": "Delete Import", "delete_item": "Delete {item}", @@ -168,6 +170,7 @@ "disallow_null_values_help": "Enable this option to prevent null values in the column. Null values are empty values that are not the same as zero or an empty string.", "discard_changes": "Discard Changes", "disconnect": "Disconnect", + "disconnect_database": "Disconnect Database", "display_language": "Display Language", "display_name": "Display Name", "documentation_and_resources": "Documentation & Resources", @@ -470,6 +473,7 @@ "select_user": "Select User", "set_constraint_name": "Set Constraint Name", "set_to": "Set to", + "settings": "Settings", "setup_connections_help": "Seems you haven't set up any databases. To use Mathesar, you'll need to connect one.", "share": "Share", "share_exploration": "Share Exploration", diff --git a/mathesar_ui/src/pages/database/DatabasePage.svelte b/mathesar_ui/src/pages/database/DatabasePage.svelte index b4f2d16aff..37ad749074 100644 --- a/mathesar_ui/src/pages/database/DatabasePage.svelte +++ b/mathesar_ui/src/pages/database/DatabasePage.svelte @@ -1,11 +1,49 @@ @@ -13,14 +51,46 @@ - {#key database.id} - - {/key} + +
+ + + + + {$_('disconnect_database')} + + + {$_('delete_database')} + + +
+
+ + + {#if activeTab?.id === 'schemas'} + + {:else if activeTab?.id === 'settings'} + + {/if} +
diff --git a/mathesar_ui/src/pages/database/DatabaseDetails.svelte b/mathesar_ui/src/pages/database/SchemasSection.svelte similarity index 64% rename from mathesar_ui/src/pages/database/DatabaseDetails.svelte rename to mathesar_ui/src/pages/database/SchemasSection.svelte index 8008fa11a0..0f50b468ac 100644 --- a/mathesar_ui/src/pages/database/DatabaseDetails.svelte +++ b/mathesar_ui/src/pages/database/SchemasSection.svelte @@ -1,21 +1,12 @@ - - -
- - -
- {$_('sync_external_changes')} - -

- {$_('sync_external_changes_structure_help')} -

-

- {$_('sync_external_changes_data_help')} -

-
-
-
- {#if userProfile?.isSuperUser} - editConnectionModal.open()} - > - {$_('edit_connection')} - - deleteConnectionModal.open()} - > - {$_('delete_connection')} - - {/if} -
-
-
-
-

{$_('schemas')} ({schemasMap.size})

diff --git a/mathesar_ui/src/pages/database/SettingsSection.svelte b/mathesar_ui/src/pages/database/SettingsSection.svelte new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mathesar_ui/src/pages/schema/SchemaPage.svelte b/mathesar_ui/src/pages/schema/SchemaPage.svelte index 06f233263c..7340f7c5f2 100644 --- a/mathesar_ui/src/pages/schema/SchemaPage.svelte +++ b/mathesar_ui/src/pages/schema/SchemaPage.svelte @@ -91,10 +91,8 @@ > diff --git a/mathesar_ui/src/routes/AdminRoute.svelte b/mathesar_ui/src/routes/AdminRoute.svelte index ed9ad6a965..48cad96d58 100644 --- a/mathesar_ui/src/routes/AdminRoute.svelte +++ b/mathesar_ui/src/routes/AdminRoute.svelte @@ -34,7 +34,6 @@ > - - - + + + + + - + Date: Mon, 12 Aug 2024 23:56:35 +0530 Subject: [PATCH 0620/1141] Set max depth to 1 for record page widgets list --- mathesar_ui/src/pages/record/RecordPageContent.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/mathesar_ui/src/pages/record/RecordPageContent.svelte b/mathesar_ui/src/pages/record/RecordPageContent.svelte index 19fa289d96..e442d051a2 100644 --- a/mathesar_ui/src/pages/record/RecordPageContent.svelte +++ b/mathesar_ui/src/pages/record/RecordPageContent.svelte @@ -45,6 +45,7 @@ .list_joinable({ database_id: $currentDatabase.id, table_oid: tableId, + max_depth: 1, }) .run(); } From 6f842c258d72f1fe73ab3afb3b718b42f531d855 Mon Sep 17 00:00:00 2001 From: pavish Date: Tue, 13 Aug 2024 00:50:07 +0530 Subject: [PATCH 0621/1141] Implement Database model class --- mathesar_ui/src/api/rpc/configured_roles.ts | 20 +++++------ mathesar_ui/src/api/rpc/database_setup.ts | 22 ++++++------ mathesar_ui/src/api/rpc/databases.ts | 36 +++---------------- mathesar_ui/src/api/rpc/servers.ts | 4 +-- .../src/components/DatabaseName.svelte | 2 +- .../breadcrumb/DatabaseSelector.svelte | 2 +- .../breadcrumb/EntitySelector.svelte | 2 +- .../breadcrumb/SchemaSelector.svelte | 2 +- .../components/breadcrumb/breadcrumbTypes.ts | 2 +- mathesar_ui/src/models/databases.ts | 20 +++++++++++ mathesar_ui/src/models/servers.ts | 15 ++++++++ .../data-explorer/DataExplorerPage.svelte | 2 +- .../pages/database/AddEditSchemaModal.svelte | 2 +- .../src/pages/database/DatabasePage.svelte | 4 +-- .../src/pages/database/SchemaRow.svelte | 2 +- .../src/pages/database/SchemasSection.svelte | 2 +- .../pages/exploration/ExplorationPage.svelte | 2 +- .../src/pages/exploration/Header.svelte | 2 +- mathesar_ui/src/pages/home/DatabaseRow.svelte | 4 +-- mathesar_ui/src/pages/home/HomePage.svelte | 2 +- .../preview/ImportPreviewContent.svelte | 2 +- .../import/preview/ImportPreviewPage.svelte | 2 +- .../import/preview/importPreviewPageUtils.ts | 2 +- .../import/upload/ImportUploadPage.svelte | 2 +- .../schema/CreateEmptyTableButton.svelte | 2 +- .../CreateNewExplorationTutorial.svelte | 2 +- .../pages/schema/CreateNewTableButton.svelte | 2 +- .../schema/CreateNewTableTutorial.svelte | 2 +- .../src/pages/schema/ExplorationItem.svelte | 2 +- .../src/pages/schema/ExplorationsList.svelte | 2 +- .../pages/schema/SchemaExplorations.svelte | 2 +- .../src/pages/schema/SchemaOverview.svelte | 2 +- .../src/pages/schema/SchemaPage.svelte | 2 +- .../src/pages/schema/SchemaTables.svelte | 2 +- mathesar_ui/src/pages/schema/TableCard.svelte | 2 +- .../src/pages/schema/TablesList.svelte | 2 +- .../src/routes/DataExplorerRoute.svelte | 2 +- mathesar_ui/src/routes/DatabaseRoute.svelte | 2 +- .../src/routes/ExplorationRoute.svelte | 2 +- mathesar_ui/src/routes/ImportRoute.svelte | 2 +- mathesar_ui/src/routes/RecordPageRoute.svelte | 2 +- mathesar_ui/src/routes/SchemaRoute.svelte | 2 +- mathesar_ui/src/routes/TableRoute.svelte | 2 +- .../src/stores/abstract-types/store.ts | 2 +- mathesar_ui/src/stores/databases.ts | 19 +++++++--- mathesar_ui/src/stores/schemas.ts | 2 +- .../src/stores/table-data/TableStructure.ts | 2 +- mathesar_ui/src/stores/table-data/columns.ts | 2 +- .../src/stores/table-data/constraints.ts | 2 +- mathesar_ui/src/stores/table-data/records.ts | 2 +- .../src/stores/table-data/tabularData.ts | 2 +- mathesar_ui/src/stores/tables.ts | 2 +- .../ConnectDatabaseModal.svelte | 2 +- .../ConnectExistingDatabase.svelte | 2 +- .../create-database/CreateNewDatabase.svelte | 2 +- mathesar_ui/src/utils/preloadData.ts | 10 +++--- 56 files changed, 132 insertions(+), 114 deletions(-) create mode 100644 mathesar_ui/src/models/databases.ts create mode 100644 mathesar_ui/src/models/servers.ts diff --git a/mathesar_ui/src/api/rpc/configured_roles.ts b/mathesar_ui/src/api/rpc/configured_roles.ts index 351452b2f6..bb30bae9ec 100644 --- a/mathesar_ui/src/api/rpc/configured_roles.ts +++ b/mathesar_ui/src/api/rpc/configured_roles.ts @@ -1,10 +1,10 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; -import type { Server } from './servers'; +import type { RawServer } from './servers'; -export interface ConfiguredRole { +export interface RawConfiguredRole { id: number; - server_id: Server['id']; + server_id: RawServer['id']; name: string; } @@ -12,30 +12,30 @@ export interface ConfiguredRole { export const configured_roles = { list: rpcMethodTypeContainer< { - server_id: ConfiguredRole['server_id']; + server_id: RawConfiguredRole['server_id']; }, - Array + Array >(), add: rpcMethodTypeContainer< { - server_id: ConfiguredRole['server_id']; - name: ConfiguredRole['name']; + server_id: RawConfiguredRole['server_id']; + name: RawConfiguredRole['name']; password: string; }, - ConfiguredRole + RawConfiguredRole >(), delete: rpcMethodTypeContainer< { - configured_role_id: ConfiguredRole['id']; + configured_role_id: RawConfiguredRole['id']; }, void >(), set_password: rpcMethodTypeContainer< { - configured_role_id: ConfiguredRole['id']; + configured_role_id: RawConfiguredRole['id']; password: string; }, void diff --git a/mathesar_ui/src/api/rpc/database_setup.ts b/mathesar_ui/src/api/rpc/database_setup.ts index 41f42ef584..81c5a0f93b 100644 --- a/mathesar_ui/src/api/rpc/database_setup.ts +++ b/mathesar_ui/src/api/rpc/database_setup.ts @@ -1,8 +1,8 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; -import type { ConfiguredRole } from './configured_roles'; -import type { DatabaseResponse } from './databases'; -import type { Server } from './servers'; +import type { RawConfiguredRole } from './configured_roles'; +import type { RawDatabase } from './databases'; +import type { RawServer } from './servers'; export const sampleDataOptions = [ 'library_management', @@ -12,16 +12,16 @@ export const sampleDataOptions = [ export type SampleDataSchemaIdentifier = (typeof sampleDataOptions)[number]; export interface DatabaseConnectionResult { - server: Server; - database: DatabaseResponse; - configured_role: ConfiguredRole; + server: RawServer; + database: RawDatabase; + configured_role: RawConfiguredRole; } // eslint-disable-next-line @typescript-eslint/naming-convention export const database_setup = { create_new: rpcMethodTypeContainer< { - database: DatabaseResponse['name']; + database: RawDatabase['name']; sample_data?: SampleDataSchemaIdentifier[]; }, DatabaseConnectionResult @@ -29,10 +29,10 @@ export const database_setup = { connect_existing: rpcMethodTypeContainer< { - host: Server['host']; - port: Server['port']; - database: DatabaseResponse['name']; - role: ConfiguredRole['name']; + host: RawServer['host']; + port: RawServer['port']; + database: RawDatabase['name']; + role: RawConfiguredRole['name']; password: string; sample_data?: SampleDataSchemaIdentifier[]; }, diff --git a/mathesar_ui/src/api/rpc/databases.ts b/mathesar_ui/src/api/rpc/databases.ts index 1fa2b32e37..10e9a11628 100644 --- a/mathesar_ui/src/api/rpc/databases.ts +++ b/mathesar_ui/src/api/rpc/databases.ts @@ -1,44 +1,18 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; -import type { Server } from './servers'; +import type { RawServer } from './servers'; -export interface DatabaseResponse { +export interface RawDatabase { id: number; name: string; - server_id: Server['id']; -} - -/** - * TODO_BETA: Modify after store discussion is resolved - */ -export class Database implements DatabaseResponse { - readonly id: number; - - readonly name: string; - - readonly server_id: number; - - readonly server_host: string; - - readonly server_port: number; - - constructor(databaseResponse: DatabaseResponse, server: Server) { - this.id = databaseResponse.id; - this.name = databaseResponse.name; - if (databaseResponse.server_id !== server.id) { - throw new Error('Server ids do not match'); - } - this.server_id = databaseResponse.server_id; - this.server_host = server.host; - this.server_port = server.port; - } + server_id: RawServer['id']; } export const databases = { list: rpcMethodTypeContainer< { - server_id?: DatabaseResponse['server_id']; + server_id?: RawDatabase['server_id']; }, - Array + Array >(), }; diff --git a/mathesar_ui/src/api/rpc/servers.ts b/mathesar_ui/src/api/rpc/servers.ts index eaa9faec31..f92a29bebb 100644 --- a/mathesar_ui/src/api/rpc/servers.ts +++ b/mathesar_ui/src/api/rpc/servers.ts @@ -1,11 +1,11 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; -export interface Server { +export interface RawServer { id: number; host: string; port: number; } export const servers = { - list: rpcMethodTypeContainer>(), + list: rpcMethodTypeContainer>(), }; diff --git a/mathesar_ui/src/components/DatabaseName.svelte b/mathesar_ui/src/components/DatabaseName.svelte index f7855f97eb..186344afc2 100644 --- a/mathesar_ui/src/components/DatabaseName.svelte +++ b/mathesar_ui/src/components/DatabaseName.svelte @@ -1,6 +1,6 @@ diff --git a/mathesar_ui/src/pages/schema/SchemaPage.svelte b/mathesar_ui/src/pages/schema/SchemaPage.svelte index 7159d8d095..31e3c9adee 100644 --- a/mathesar_ui/src/pages/schema/SchemaPage.svelte +++ b/mathesar_ui/src/pages/schema/SchemaPage.svelte @@ -15,11 +15,10 @@ import { modal } from '@mathesar/stores/modal'; import { queries } from '@mathesar/stores/queries'; import { currentTablesData as tablesStore } from '@mathesar/stores/tables'; + import AddEditSchemaModal from '@mathesar/systems/schemas/AddEditSchemaModal.svelte'; import { logEvent } from '@mathesar/utils/telemetry'; import { Button, Icon, TabContainer } from '@mathesar-component-library'; - import AddEditSchemaModal from '../database/AddEditSchemaModal.svelte'; - import ExplorationSkeleton from './ExplorationSkeleton.svelte'; import SchemaExplorations from './SchemaExplorations.svelte'; import SchemaOverview from './SchemaOverview.svelte'; diff --git a/mathesar_ui/src/pages/database/AddEditSchemaModal.svelte b/mathesar_ui/src/systems/schemas/AddEditSchemaModal.svelte similarity index 97% rename from mathesar_ui/src/pages/database/AddEditSchemaModal.svelte rename to mathesar_ui/src/systems/schemas/AddEditSchemaModal.svelte index a1bc8d8cbe..c96c8faf39 100644 --- a/mathesar_ui/src/pages/database/AddEditSchemaModal.svelte +++ b/mathesar_ui/src/systems/schemas/AddEditSchemaModal.svelte @@ -1,4 +1,3 @@ - - + diff --git a/mathesar_ui/src/components/routing/MultiPathRoute.svelte b/mathesar_ui/src/components/routing/MultiPathRoute.svelte index 7be61b5539..abc4ecbe59 100644 --- a/mathesar_ui/src/components/routing/MultiPathRoute.svelte +++ b/mathesar_ui/src/components/routing/MultiPathRoute.svelte @@ -37,8 +37,8 @@ {#each paths as rp (rp.name)} setPath(rp, e.detail)} - on:unload={() => clearPath(rp)} + onLoad={(meta) => setPath(rp, meta)} + onUnload={() => clearPath(rp)} firstmatch /> {/each} diff --git a/mathesar_ui/src/components/routing/RouteObserver.svelte b/mathesar_ui/src/components/routing/RouteObserver.svelte index 105f5c5df8..55f214cb28 100644 --- a/mathesar_ui/src/components/routing/RouteObserver.svelte +++ b/mathesar_ui/src/components/routing/RouteObserver.svelte @@ -1,33 +1,21 @@ From 8133181e670734991fe7382e6dc24295536274f1 Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 15 Aug 2024 21:48:38 +0530 Subject: [PATCH 0641/1141] Add routes for all sections in database page --- mathesar/urls.py | 6 +- mathesar_ui/src/i18n/languages/en/dict.json | 5 ++ mathesar_ui/src/models/databases.ts | 5 +- ...Page.svelte => DatabasePageWrapper.svelte} | 21 ++--- .../database/settings/Collaborators.svelte | 5 ++ .../settings/RoleConfiguration.svelte | 5 ++ .../src/pages/database/settings/Roles.svelte | 5 ++ .../database/settings/SettingsSection.svelte | 3 - .../database/settings/SettingsWrapper.svelte | 77 +++++++++++++++++++ mathesar_ui/src/routes/DatabaseRoute.svelte | 61 +++++++++++---- mathesar_ui/src/routes/urls.ts | 12 +++ 11 files changed, 170 insertions(+), 35 deletions(-) rename mathesar_ui/src/pages/database/{DatabasePage.svelte => DatabasePageWrapper.svelte} (81%) create mode 100644 mathesar_ui/src/pages/database/settings/Collaborators.svelte create mode 100644 mathesar_ui/src/pages/database/settings/RoleConfiguration.svelte create mode 100644 mathesar_ui/src/pages/database/settings/Roles.svelte delete mode 100644 mathesar_ui/src/pages/database/settings/SettingsSection.svelte create mode 100644 mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte diff --git a/mathesar/urls.py b/mathesar/urls.py index 3a8d1d7c4c..004d216c42 100644 --- a/mathesar/urls.py +++ b/mathesar/urls.py @@ -57,11 +57,15 @@ path('shares/tables//', views.shared_table, name='shared_table'), path('shares/explorations//', views.shared_query, name='shared_query'), path('databases/', views.databases, name='databases'), - path('db//', views.schemas, name='schemas'), path('i18n/', include('django.conf.urls.i18n')), re_path( r'^db/(?P\w+)/schemas/(?P\w+)/', views.schemas_home, name='schema_home' ), + re_path( + r'^db/(?P\w+)/((schemas|settings)/)?', + views.schemas, + name='schemas' + ), ] diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index 7843238c83..0733249cf7 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -51,6 +51,7 @@ "cleaning_up": "Cleaning up", "clear": "Clear", "clear_value": "Clear value", + "collaborators": "Collaborators", "column": "Column", "column_added_number_of_times": "{count, plural, one {This column has been added once.} other {This column has been added {count} times.}}", "column_data_types": "Column Data Types", @@ -251,6 +252,7 @@ "if_upgrade_succeeds_help": "If the upgrade succeeds, you will see that you're running the latest version.", "import": "Import", "import_from_file": "Import from a File", + "in_mathesar": "In Mathesar", "in_this_table": "In this table", "individual_permissions_admin_modify_warning": "Individual permissions cannot be modified for users with Admin access.", "inherited": "Inherited", @@ -358,6 +360,7 @@ "number_of_matches_in_category": "{count, plural, one {{count} match for [searchValue] in [categoryName]} other {{count} matches for [searchValue] in [categoryName]}}", "old_password": "Old Password", "oldest_to_newest_sort": "Oldest-Newest", + "on_the_server": "On The Server", "one_column_from_base_is_required": "At least one column from the base table is required to add columns from linked tables.", "one_to_many": "One to Many", "one_to_many_link_desc": "One [baseTable] record can be linked from multiple [targetTable] records.", @@ -431,6 +434,8 @@ "retry": "Retry", "reuse_credentials_from_known_connection": "Reuse credentials from a known connection", "role": "Role", + "role_configuration": "Role Configuration", + "roles": "Roles", "row": "Row", "running_latest_version": "You are running the latest version", "sample_data_library_help": "Sample data from a fictional library.", diff --git a/mathesar_ui/src/models/databases.ts b/mathesar_ui/src/models/databases.ts index 96b50c3f1e..4d56f06833 100644 --- a/mathesar_ui/src/models/databases.ts +++ b/mathesar_ui/src/models/databases.ts @@ -5,16 +5,13 @@ import type { Server } from './servers'; export class Database { readonly id: number; - name: string; + readonly name: string; readonly server: Server; constructor(props: { server: Server; rawDatabase: RawDatabase }) { this.id = props.rawDatabase.id; this.name = props.rawDatabase.name; - if (props.rawDatabase.server_id !== props.server.id) { - throw new Error('Server ids do not match'); - } this.server = props.server; } } diff --git a/mathesar_ui/src/pages/database/DatabasePage.svelte b/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte similarity index 81% rename from mathesar_ui/src/pages/database/DatabasePage.svelte rename to mathesar_ui/src/pages/database/DatabasePageWrapper.svelte index e286b3f829..2e82d9ffb3 100644 --- a/mathesar_ui/src/pages/database/DatabasePage.svelte +++ b/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte @@ -21,11 +21,10 @@ TabContainer, } from '@mathesar-component-library'; - import SchemasSection from './schemas/SchemasSection.svelte'; - import SettingsSection from './settings/SettingsSection.svelte'; - export let database: Database; - export let section: string; + + type Section = 'schemas' | 'settings'; + let section: Section = 'schemas'; $: tabs = [ { @@ -39,10 +38,10 @@ href: getDatabasePageSettingsSectionUrl(database.id), }, ]; - $: activeTab = tabs.find((tab) => tab.id === section) ?? tabs[0]; + $: activeTab = tabs.find((tab) => tab.id === section); - function openPermissionsModal() { - // + export function setSection(_section: Section) { + section = _section; } @@ -65,7 +64,7 @@ }} >
- @@ -88,11 +87,7 @@
- {#if activeTab?.id === 'schemas'} - - {:else if activeTab?.id === 'settings'} - - {/if} +
diff --git a/mathesar_ui/src/pages/database/settings/Collaborators.svelte b/mathesar_ui/src/pages/database/settings/Collaborators.svelte new file mode 100644 index 0000000000..5c75a672eb --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/Collaborators.svelte @@ -0,0 +1,5 @@ + + +{$_('collaborators')} diff --git a/mathesar_ui/src/pages/database/settings/RoleConfiguration.svelte b/mathesar_ui/src/pages/database/settings/RoleConfiguration.svelte new file mode 100644 index 0000000000..23aa8257fd --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/RoleConfiguration.svelte @@ -0,0 +1,5 @@ + + +{$_('role_configuration')} diff --git a/mathesar_ui/src/pages/database/settings/Roles.svelte b/mathesar_ui/src/pages/database/settings/Roles.svelte new file mode 100644 index 0000000000..3636bb90da --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/Roles.svelte @@ -0,0 +1,5 @@ + + +{$_('roles')} diff --git a/mathesar_ui/src/pages/database/settings/SettingsSection.svelte b/mathesar_ui/src/pages/database/settings/SettingsSection.svelte deleted file mode 100644 index 220a4ef6f0..0000000000 --- a/mathesar_ui/src/pages/database/settings/SettingsSection.svelte +++ /dev/null @@ -1,3 +0,0 @@ - diff --git a/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte b/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte new file mode 100644 index 0000000000..320be93c8f --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte @@ -0,0 +1,77 @@ + + + + +
+ +
+
+ + diff --git a/mathesar_ui/src/routes/DatabaseRoute.svelte b/mathesar_ui/src/routes/DatabaseRoute.svelte index f4ade64fbe..5ffacc0b19 100644 --- a/mathesar_ui/src/routes/DatabaseRoute.svelte +++ b/mathesar_ui/src/routes/DatabaseRoute.svelte @@ -6,9 +6,14 @@ import AppendBreadcrumb from '@mathesar/components/breadcrumb/AppendBreadcrumb.svelte'; import Identifier from '@mathesar/components/Identifier.svelte'; import { RichText } from '@mathesar/components/rich-text'; - import MultiPathRoute from '@mathesar/components/routing/MultiPathRoute.svelte'; + import EventfulRoute from '@mathesar/components/routing/EventfulRoute.svelte'; import type { Database } from '@mathesar/models/databases'; - import DatabasePage from '@mathesar/pages/database/DatabasePage.svelte'; + import DatabasePageWrapper from '@mathesar/pages/database/DatabasePageWrapper.svelte'; + import DatabasePageSchemasSection from '@mathesar/pages/database/schemas/SchemasSection.svelte'; + import DatabaseCollaborators from '@mathesar/pages/database/settings/Collaborators.svelte'; + import DatabaseRoleConfiguration from '@mathesar/pages/database/settings/RoleConfiguration.svelte'; + import DatabaseRoles from '@mathesar/pages/database/settings/Roles.svelte'; + import DatabasePageSettingsWrapper from '@mathesar/pages/database/settings/SettingsWrapper.svelte'; import ErrorPage from '@mathesar/pages/ErrorPage.svelte'; import { databasesStore } from '@mathesar/stores/databases'; @@ -29,24 +34,52 @@ {#if $currentDatabase} - - - - - - + + + + + + setSection('schemas')}> + + + setSection('settings')} + firstmatch + > + + + setSettingsSection('roleConfiguration')} + > + + + setSettingsSection('collaborators')} + > + + + setSettingsSection('roles')} + > + + + + + + {:else} diff --git a/mathesar_ui/src/routes/urls.ts b/mathesar_ui/src/routes/urls.ts index bd7bbfbab0..879c3874ce 100644 --- a/mathesar_ui/src/routes/urls.ts +++ b/mathesar_ui/src/routes/urls.ts @@ -10,6 +10,18 @@ export function getDatabasePageSettingsSectionUrl(databaseId: number): string { return `/db/${databaseId}/settings/`; } +export function getDatabaseRoleConfigurationUrl(databaseId: number): string { + return `${getDatabasePageSettingsSectionUrl(databaseId)}role-configuration/`; +} + +export function getDatabaseCollaboratorsUrl(databaseId: number): string { + return `${getDatabasePageSettingsSectionUrl(databaseId)}collaborators/`; +} + +export function getDatabaseRolesUrl(databaseId: number): string { + return `${getDatabasePageSettingsSectionUrl(databaseId)}roles/`; +} + export function getSchemaPageUrl(databaseId: number, schemaId: number): string { return `${getDatabasePageSchemasSectionUrl(databaseId)}${schemaId}/`; } From 4ce7ec5e3681e383029f35e09638b412ce515d9c Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 15 Aug 2024 22:18:18 +0530 Subject: [PATCH 0642/1141] remove cte and change endpoint name --- db/roles/operations/select.py | 2 +- db/sql/00_msar.sql | 21 ++++++++------------- mathesar/rpc/database_privileges.py | 5 +++-- mathesar/tests/rpc/test_endpoints.py | 4 ++-- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py index 1d2b29aa94..1d0d94434b 100644 --- a/db/roles/operations/select.py +++ b/db/roles/operations/select.py @@ -10,4 +10,4 @@ def list_db_priv(db_name, conn): def get_curr_role_db_priv(db_name, conn): - return exec_msar_func(conn, 'get_curr_role_db_priv', db_name).fetchone()[0] + return exec_msar_func(conn, 'get_owner_oid_and_curr_role_db_priv', db_name).fetchone()[0] diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index a902607320..b0c400ee99 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -974,7 +974,7 @@ SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION msar.get_curr_role_db_priv(db_name text) RETURNS jsonb AS $$/* +CREATE OR REPLACE FUNCTION msar.get_owner_oid_and_curr_role_db_priv(db_name text) RETURNS jsonb AS $$/* Given a database name, returns a json object with database owner oid and database privileges for the role executing the function. @@ -984,22 +984,17 @@ The returned JSON object has the form: "current_role_db_priv" [] } */ -WITH priv_cte AS ( - SELECT unnest( +SELECT jsonb_build_object( + 'owner_oid', pgd.datdba, + 'current_role_db_priv', array_remove( ARRAY[ CASE WHEN has_database_privilege(pgd.oid, 'CREATE') THEN 'CREATE' END, CASE WHEN has_database_privilege(pgd.oid, 'TEMPORARY') THEN 'TEMPORARY' END, CASE WHEN has_database_privilege(pgd.oid, 'CONNECT') THEN 'CONNECT' END - ] - ) AS p - FROM pg_catalog.pg_database AS pgd WHERE pgd.datname = db_name -) -SELECT jsonb_build_object( - 'owner_oid', pgd.datdba, - 'current_role_db_priv', COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) -) FROM pg_catalog.pg_database AS pgd, priv_cte -WHERE pgd.datname = db_name AND priv_cte.p IS NOT NULL -GROUP BY pgd.datdba; + ], NULL + ) +) FROM pg_catalog.pg_database AS pgd +WHERE pgd.datname = db_name; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; diff --git a/mathesar/rpc/database_privileges.py b/mathesar/rpc/database_privileges.py index 0ab9e0effd..16d8971677 100644 --- a/mathesar/rpc/database_privileges.py +++ b/mathesar/rpc/database_privileges.py @@ -67,10 +67,11 @@ def list_direct(*, database_id: int, **kwargs) -> list[DBPrivileges]: return [DBPrivileges.from_dict(i) for i in raw_db_priv] -@rpc_method(name="database_privileges.get_curr_role_priv") +# TODO: Think of something concise for the endpoint name. +@rpc_method(name="database_privileges.get_owner_oid_and_curr_role_db_priv") @http_basic_auth_login_required @handle_rpc_exceptions -def get_curr_role_priv(*, database_id: int, **kwargs) -> CurrentDBPrivileges: +def get_owner_oid_and_curr_role_db_priv(*, database_id: int, **kwargs) -> CurrentDBPrivileges: """ Get database privileges for the current user. diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 8062e2b8e7..067bff05d1 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -136,8 +136,8 @@ [user_is_authenticated] ), ( - database_privileges.get_curr_role_priv, - "database_privileges.get_curr_role_priv", + database_privileges.get_owner_oid_and_curr_role_db_priv, + "database_privileges.get_owner_oid_and_curr_role_db_priv", [user_is_authenticated] ), ( From 7ebf47a184a175abdd2627a48c2605ce72735bf5 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 15 Aug 2024 23:19:58 +0530 Subject: [PATCH 0643/1141] update docs --- docs/docs/api/rpc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 23eb0deea9..1dc1378d00 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -83,7 +83,7 @@ To use an RPC function: options: members: - list_direct - - get_curr_role_priv + - get_owner_oid_and_curr_role_db_priv - DBPrivileges - CurrentDBPrivileges From a2981e03fb0b2bba22e6e1450c68ad5f2abba186 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 16 Aug 2024 13:53:47 +0530 Subject: [PATCH 0644/1141] Implement batching api requests --- .../json-rpc-client-builder/requests.ts | 80 +++++++++++++++---- 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/mathesar_ui/src/packages/json-rpc-client-builder/requests.ts b/mathesar_ui/src/packages/json-rpc-client-builder/requests.ts index 0ad5cfd337..c25af1f163 100644 --- a/mathesar_ui/src/packages/json-rpc-client-builder/requests.ts +++ b/mathesar_ui/src/packages/json-rpc-client-builder/requests.ts @@ -26,6 +26,15 @@ function cancellableFetch( ); } +function getRpcRequestBody(request: RpcRequest, id = 0) { + return { + jsonrpc, + id, + method: request.method, + params: request.params, + }; +} + function makeRpcResponse(value: unknown): RpcResponse { if (hasProperty(value, 'result')) { const response: RpcResult = { @@ -44,12 +53,7 @@ function send(request: RpcRequest): CancellablePromise> { ...request.getHeaders(), 'Content-Type': 'application/json', }, - body: JSON.stringify({ - jsonrpc, - id: 0, - method: request.method, - params: request.params, - }), + body: JSON.stringify(getRpcRequestBody(request)), }); return new CancellablePromise( (resolve) => @@ -67,6 +71,47 @@ function send(request: RpcRequest): CancellablePromise> { ); } +function makeRpcBatchResponse[]>( + values: unknown, +): RpcBatchResponse { + if (!Array.isArray(values)) { + throw new Error('Response is not an array'); + } + return values.map((value) => makeRpcResponse(value)) as RpcBatchResponse; +} + +function sendBatchRequest[]>( + endpoint: string, + headers: Record, + requests: T, +): CancellablePromise> { + const fetch = cancellableFetch(endpoint, { + method: 'POST', + headers: { + ...headers, + 'Content-Type': 'application/json', + }, + body: JSON.stringify( + requests.map((request, index) => getRpcRequestBody(request, index)), + ), + }); + return new CancellablePromise( + (resolve) => + void fetch + .then( + (response) => response.json(), + (rejectionReason) => + resolve( + requests.map(() => + RpcError.fromAnything(rejectionReason), + ) as RpcBatchResponse, + ), + ) + .then((json) => resolve(makeRpcBatchResponse(json))), + () => fetch.cancel(), + ); +} + export type GetHeaders = () => Record; export class RpcRequest { @@ -126,16 +171,23 @@ export class RpcRequest { } } -export type RpcBatchResponse[]> = - CancellablePromise<{ - [K in keyof T]: T[K] extends RpcRequest ? RpcResponse : never; - }>; +export type RpcBatchResponse[]> = { + [K in keyof T]: T[K] extends RpcRequest ? RpcResponse : never; +}; export function batchSend[]>( - ...requests: T -): RpcBatchResponse { - // TODO implement batch sending - throw new Error('Not implemented'); + requests: T, +): CancellablePromise> { + if (requests.length === 0) { + throw new Error('There must be atleast one request'); + } + const [firstRequest, ...rest] = requests; + const { endpoint } = firstRequest; + if (rest.some((request) => request.endpoint !== endpoint)) { + throw new Error('Only RPC requests to the same endpoint can be batched'); + } + // TODO: Decide if headers need to be merged + return sendBatchRequest(endpoint, firstRequest.getHeaders(), requests); } /** From 805c6d0e2ee439ba809405226f7f00b0fdf73cc7 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 16 Aug 2024 13:54:14 +0530 Subject: [PATCH 0645/1141] Implement AsyncRpcApiStore --- mathesar_ui/src/stores/AsyncRpcApiStore.ts | 69 ++++++++++++++++++++++ mathesar_ui/src/stores/AsyncStore.ts | 44 ++++++++++++-- 2 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 mathesar_ui/src/stores/AsyncRpcApiStore.ts diff --git a/mathesar_ui/src/stores/AsyncRpcApiStore.ts b/mathesar_ui/src/stores/AsyncRpcApiStore.ts new file mode 100644 index 0000000000..68c1057bdc --- /dev/null +++ b/mathesar_ui/src/stores/AsyncRpcApiStore.ts @@ -0,0 +1,69 @@ +import { CancellablePromise } from '@mathesar/component-library'; +import { + type RpcRequest, + type RpcResponse, + batchSend, +} from '@mathesar/packages/json-rpc-client-builder'; + +import AsyncStore from './AsyncStore'; + +export default class AsyncRpcApiStore extends AsyncStore< + Props, + U +> { + apiRpcFn: (props: Props) => RpcRequest; + + postProcess: (response: T) => U; + + constructor( + rpcFn: (props: Props) => RpcRequest, + options?: Partial<{ + getError: (caughtValue: unknown) => string; + initialValue: U; + postProcess: (response: T) => U; + }>, + ) { + const postProcess = + options?.postProcess ?? ((response: T) => response as unknown as U); + super( + (props: Props) => + new CancellablePromise((resolve, reject) => { + rpcFn(props) + .run() + .then( + (value) => resolve(postProcess(value)), + (error) => reject(error), + ) + .catch((error) => reject(error)); + }), + ); + this.apiRpcFn = rpcFn; + this.postProcess = postProcess; + } + + batchRunner( + props: Props, + ): [RpcRequest, (response: RpcResponse) => void] { + const onReponse = (response: RpcResponse) => { + if (response.status === 'ok') { + this.setResolvedValue(this.postProcess(response.value)); + } else { + this.setRejectedError(response); + } + }; + return [this.apiRpcFn(props), onReponse]; + } + + static async runBatched( + batchRunners: [ + RpcRequest, + (response: RpcResponse) => void, + ][], + ) { + const requests = batchRunners.map((runner) => runner[0]); + const results = await batchSend(requests); + batchRunners.forEach((runner, index) => { + runner[1](results[index]); + }); + } +} diff --git a/mathesar_ui/src/stores/AsyncStore.ts b/mathesar_ui/src/stores/AsyncStore.ts index a21be3ae1d..33830f3d0e 100644 --- a/mathesar_ui/src/stores/AsyncStore.ts +++ b/mathesar_ui/src/stores/AsyncStore.ts @@ -10,7 +10,10 @@ import { } from 'svelte/store'; import { getErrorMessage } from '@mathesar/utils/errors'; -import type { CancellablePromise } from '@mathesar-component-library'; +import { + type CancellablePromise, + hasProperty, +} from '@mathesar-component-library'; export type AsyncStoreSettlement = | { state: 'resolved'; value: T } @@ -111,19 +114,30 @@ export default class AsyncStore constructor( run: (props: Props) => Promise | CancellablePromise, - getError: (caughtValue: unknown) => string = getErrorMessage, + options?: Partial<{ + getError: (caughtValue: unknown) => string; + initialValue: T; + }>, ) { this.runFn = run; - this.getError = getError; + this.getError = options?.getError ?? getErrorMessage; + if (hasProperty(options, 'initialValue')) { + this.value = writable( + new AsyncStoreValue({ + isLoading: false, + settlement: { state: 'resolved', value: options.initialValue as T }, + }), + ); + } } subscribe( - run: Subscriber>, + subscriber: Subscriber>, invalidate?: | ((value?: AsyncStoreValue | undefined) => void) | undefined, ): Unsubscriber { - return this.value.subscribe(run, invalidate); + return this.value.subscribe(subscriber, invalidate); } async run(props: Props): Promise> { @@ -165,6 +179,26 @@ export default class AsyncStore this.cancel(); this.value.set(new AsyncStoreValue({ isLoading: false })); } + + setResolvedValue(value: T) { + this.cancel(); + this.value.set( + new AsyncStoreValue({ + isLoading: false, + settlement: { state: 'resolved', value }, + }), + ); + } + + setRejectedError(error: unknown) { + this.cancel(); + this.value.set( + new AsyncStoreValue({ + settlement: { state: 'rejected', error: this.getError(error) }, + isLoading: false, + }), + ); + } } /* eslint-enable max-classes-per-file */ From 52abcdb4c9f966fba0f6cf9ec1f28aebabd125b5 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 16 Aug 2024 14:57:58 +0530 Subject: [PATCH 0646/1141] Name model class files appropriately --- mathesar_ui/src/components/DatabaseName.svelte | 2 +- mathesar_ui/src/components/breadcrumb/DatabaseSelector.svelte | 2 +- mathesar_ui/src/components/breadcrumb/EntitySelector.svelte | 2 +- mathesar_ui/src/components/breadcrumb/SchemaSelector.svelte | 2 +- mathesar_ui/src/components/breadcrumb/breadcrumbTypes.ts | 2 +- mathesar_ui/src/models/{databases.ts => Database.ts} | 2 +- mathesar_ui/src/models/{servers.ts => Server.ts} | 0 mathesar_ui/src/pages/data-explorer/DataExplorerPage.svelte | 2 +- mathesar_ui/src/pages/database/DatabasePageWrapper.svelte | 2 +- mathesar_ui/src/pages/database/schemas/SchemaRow.svelte | 2 +- mathesar_ui/src/pages/database/schemas/SchemasSection.svelte | 2 +- .../src/pages/database/settings/SettingsWrapper.svelte | 2 +- mathesar_ui/src/pages/exploration/ExplorationPage.svelte | 2 +- mathesar_ui/src/pages/exploration/Header.svelte | 2 +- mathesar_ui/src/pages/home/DatabaseRow.svelte | 2 +- mathesar_ui/src/pages/home/HomePage.svelte | 2 +- .../src/pages/import/preview/ImportPreviewContent.svelte | 2 +- mathesar_ui/src/pages/import/preview/ImportPreviewPage.svelte | 2 +- .../src/pages/import/preview/importPreviewPageUtils.ts | 2 +- mathesar_ui/src/pages/import/upload/ImportUploadPage.svelte | 2 +- mathesar_ui/src/pages/schema/CreateEmptyTableButton.svelte | 2 +- .../src/pages/schema/CreateNewExplorationTutorial.svelte | 2 +- mathesar_ui/src/pages/schema/CreateNewTableButton.svelte | 2 +- mathesar_ui/src/pages/schema/CreateNewTableTutorial.svelte | 2 +- mathesar_ui/src/pages/schema/ExplorationItem.svelte | 2 +- mathesar_ui/src/pages/schema/ExplorationsList.svelte | 2 +- mathesar_ui/src/pages/schema/SchemaExplorations.svelte | 2 +- mathesar_ui/src/pages/schema/SchemaOverview.svelte | 2 +- mathesar_ui/src/pages/schema/SchemaPage.svelte | 2 +- mathesar_ui/src/pages/schema/SchemaTables.svelte | 2 +- mathesar_ui/src/pages/schema/TableCard.svelte | 2 +- mathesar_ui/src/pages/schema/TablesList.svelte | 2 +- mathesar_ui/src/routes/DataExplorerRoute.svelte | 2 +- mathesar_ui/src/routes/DatabaseRoute.svelte | 2 +- mathesar_ui/src/routes/ExplorationRoute.svelte | 2 +- mathesar_ui/src/routes/ImportRoute.svelte | 2 +- mathesar_ui/src/routes/RecordPageRoute.svelte | 2 +- mathesar_ui/src/routes/SchemaRoute.svelte | 2 +- mathesar_ui/src/routes/TableRoute.svelte | 2 +- mathesar_ui/src/stores/abstract-types/store.ts | 2 +- mathesar_ui/src/stores/databases.ts | 4 ++-- mathesar_ui/src/stores/schemas.ts | 2 +- mathesar_ui/src/stores/table-data/TableStructure.ts | 2 +- mathesar_ui/src/stores/table-data/columns.ts | 2 +- mathesar_ui/src/stores/table-data/constraints.ts | 2 +- mathesar_ui/src/stores/table-data/records.ts | 2 +- mathesar_ui/src/stores/table-data/tabularData.ts | 2 +- mathesar_ui/src/stores/tables.ts | 2 +- .../databases/create-database/ConnectDatabaseModal.svelte | 2 +- .../databases/create-database/ConnectExistingDatabase.svelte | 2 +- .../databases/create-database/CreateNewDatabase.svelte | 2 +- mathesar_ui/src/systems/schemas/AddEditSchemaModal.svelte | 2 +- 52 files changed, 52 insertions(+), 52 deletions(-) rename mathesar_ui/src/models/{databases.ts => Database.ts} (89%) rename mathesar_ui/src/models/{servers.ts => Server.ts} (100%) diff --git a/mathesar_ui/src/components/DatabaseName.svelte b/mathesar_ui/src/components/DatabaseName.svelte index 186344afc2..3876453dce 100644 --- a/mathesar_ui/src/components/DatabaseName.svelte +++ b/mathesar_ui/src/components/DatabaseName.svelte @@ -1,6 +1,6 @@ + +
+ +
+ + From c49cf79b432d89667e5a9e36ec3c2a8d16069124 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 16 Aug 2024 23:27:57 +0530 Subject: [PATCH 0652/1141] Add database page settings content layout component --- .../settings/SettingsContentLayout.svelte | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 mathesar_ui/src/pages/database/settings/SettingsContentLayout.svelte diff --git a/mathesar_ui/src/pages/database/settings/SettingsContentLayout.svelte b/mathesar_ui/src/pages/database/settings/SettingsContentLayout.svelte new file mode 100644 index 0000000000..a9bc582a92 --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/SettingsContentLayout.svelte @@ -0,0 +1,34 @@ +
+
+
+ +
+
+ +
+
+
+ +
+
+ + From df5495afc1938cff8940737cd78a77ace4e49bb5 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 16 Aug 2024 23:28:15 +0530 Subject: [PATCH 0653/1141] Add methods to fetch configured roles and roles from database model --- mathesar_ui/src/models/Database.ts | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/mathesar_ui/src/models/Database.ts b/mathesar_ui/src/models/Database.ts index f9c3093975..c8b1771553 100644 --- a/mathesar_ui/src/models/Database.ts +++ b/mathesar_ui/src/models/Database.ts @@ -1,5 +1,10 @@ +import { api } from '@mathesar/api/rpc'; import type { RawDatabase } from '@mathesar/api/rpc/databases'; +import AsyncRpcApiStore from '@mathesar/stores/AsyncRpcApiStore'; +import { SortedImmutableMap } from '@mathesar-component-library'; +import { ConfiguredRole } from './ConfiguredRole'; +import { Role } from './Role'; import type { Server } from './Server'; export class Database { @@ -14,4 +19,30 @@ export class Database { this.name = props.rawDatabase.name; this.server = props.server; } + + fetchConfiguredRoles() { + return new AsyncRpcApiStore(api.configured_roles.list, { + postProcess: (rawConfiguredRoles) => + new SortedImmutableMap( + (v) => [...v].sort(([, a], [, b]) => a.name.localeCompare(b.name)), + rawConfiguredRoles.map((rawConfiguredRole) => [ + rawConfiguredRole.id, + new ConfiguredRole({ database: this, rawConfiguredRole }), + ]), + ), + }); + } + + fetchRoles() { + return new AsyncRpcApiStore(api.roles.list, { + postProcess: (rawRoles) => + new SortedImmutableMap( + (v) => [...v].sort(([, a], [, b]) => a.name.localeCompare(b.name)), + rawRoles.map((rawRole) => [ + rawRole.oid, + new Role({ database: this, rawRole }), + ]), + ), + }); + } } From 8e79ac0f1fa7441b748ca26554b567c00fc2a1f0 Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 16 Aug 2024 23:31:39 +0530 Subject: [PATCH 0654/1141] Implement database settings context and basic RoleConfiguration page --- .../common/styles/variables.scss | 1 + mathesar_ui/src/i18n/languages/en/dict.json | 8 ++ .../pages/database/DatabasePageWrapper.svelte | 1 + .../settings/RoleConfiguration.svelte | 72 +++++++++++++++++- .../database/settings/SettingsWrapper.svelte | 4 + .../settings/databaseSettingsUtils.ts | 76 +++++++++++++++++++ 6 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts diff --git a/mathesar_ui/src/component-library/common/styles/variables.scss b/mathesar_ui/src/component-library/common/styles/variables.scss index 4e31fd79b5..48e55adc2d 100644 --- a/mathesar_ui/src/component-library/common/styles/variables.scss +++ b/mathesar_ui/src/component-library/common/styles/variables.scss @@ -34,6 +34,7 @@ --slate-700: #424952; --slate-800: #25292e; + --sand-50: #fcfbf8; --sand-100: #f9f8f6; --sand-200: #efece7; --sand-300: #e2dcd4; diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index 0733249cf7..3e5c6117be 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -45,6 +45,7 @@ "cell": "Cell", "change_password": "Change Password", "check_for_updates": "Check for Updates", + "child_roles": "Child Roles", "choose_database": "Choose a Database", "choose_schema": "Choose a Schema", "choose_table_or_exploration": "Choose a Table or Exploration", @@ -75,6 +76,8 @@ "columns_removed_from_table_added_to_target": "{count, plural, one {The column above will be removed from [tableName] and added to [targetTableName]} other {The columns above will be removed from [tableName] and added to [targetTableName]}}", "columns_to_extract": "Columns to Extract", "columns_to_move": "Columns to Move", + "configure_in_mathesar": "Configure in Mathesar", + "configure_password": "Configure Password", "confirm_and_create_table": "Confirm & create table", "confirm_delete_table": "To confirm the deletion of the [tableName] table, please enter the table name into the input field below.", "confirm_password": "Confirm Password", @@ -114,6 +117,7 @@ "create_new_pg_user": "Create a new PostgreSQL user", "create_new_schema": "Create New Schema", "create_record_from_search": "Create Record From Search Criteria", + "create_role": "Create Role", "create_schema": "Create Schema", "create_share_explorations_of_your_data": "Create and Share Explorations of Your Data", "create_table_move_columns": "Create Table and Move Columns", @@ -175,6 +179,7 @@ "display_language": "Display Language", "display_name": "Display Name", "documentation_and_resources": "Documentation & Resources", + "drop_role": "Drop Role", "edit": "Edit", "edit_connection": "Edit Connection", "edit_connection_with_name": "Edit Connection: [connectionName]", @@ -333,6 +338,7 @@ "new_user_name": "New user name", "new_version_available": "New Version Available", "newest_to_oldest_sort": "Newest-Oldest", + "no": "No", "no_actions_selected_record": "There are no actions to perform on the selected record(s).", "no_constraints": "No Constraints", "no_continue_without_summarization": "No, continue without summarizing", @@ -421,6 +427,7 @@ "release_notes": "Release Notes", "release_notes_and_upgrade_instructions": "Release Notes and Upgrade Instructions", "released_date": "Released {date}", + "remove": "Remove", "remove_filters": "{count, plural, one {Remove Filter} other {Remove {count} Filters}}", "remove_grouping": "Remove Grouping", "remove_old_link_create_new": "Remove old link and create a new link?", @@ -603,6 +610,7 @@ "while_upgrading": "While Upgrading", "why_is_this_needed": "Why is this needed?", "window_remains_open_mathesar_unusable": "This window will remain open but all features within Mathesar will be unusable.", + "yes": "Yes", "yes_summarize_as_list": "Yes, summarize as a list", "yesterday": "Yesterday" } diff --git a/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte b/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte index 2efe2a7c06..67915d4daf 100644 --- a/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte +++ b/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte @@ -53,6 +53,7 @@ restrictWidth cssVariables={{ '--max-layout-width': 'var(--max-layout-width-console-pages)', + '--layout-background-color': 'var(--sand-50)' }} > import { _ } from 'svelte-i18n'; + + import GridTable from '@mathesar/components/grid-table/GridTable.svelte'; + import GridTableCell from '@mathesar/components/grid-table/GridTableCell.svelte'; + import AsyncRpcApiStore from '@mathesar/stores/AsyncRpcApiStore'; + import { Button, Spinner } from '@mathesar-component-library'; + + import { getDatabaseSettingsContext } from './databaseSettingsUtils'; + import SettingsContentLayout from './SettingsContentLayout.svelte'; + + const databaseContext = getDatabaseSettingsContext(); + $: ({ database, configuredRoles, roles, combinedRoles } = $databaseContext); + + $: void AsyncRpcApiStore.runBatched( + [ + configuredRoles.batchRunner({ server_id: database.server.id }), + roles.batchRunner({ database_id: database.id }), + ], + { onlyRunIfNotInitialized: true }, + ); + $: isLoading = $configuredRoles.isLoading || $roles.isLoading; -{$_('role_configuration')} + + + {$_('role_configuration')} + + {#if isLoading} + + {:else} + {#if $combinedRoles.length > 0} +
+ + {$_('role')} + {$_('actions')} + + {#each $combinedRoles as combinedRole (combinedRole.name)} + {combinedRole.name} + + {#if combinedRole.configuredRole} +
+ + +
+ {:else if combinedRole.role} + + {/if} +
+ {/each} +
+
+ {/if} + + {#if $configuredRoles.error} + {$configuredRoles.error} + {/if} + {#if $roles.error} + {$roles.error} + {/if} + {/if} +
+ + diff --git a/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte b/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte index 69b1a23ffc..786b7f3ebc 100644 --- a/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte +++ b/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte @@ -10,7 +10,10 @@ } from '@mathesar/routes/urls'; import { LinkMenuItem, Menu, MenuHeading } from '@mathesar-component-library'; + import { setDatabaseSettingsContext } from './databaseSettingsUtils'; + export let database: Database; + $: setDatabaseSettingsContext(database); type Section = 'roleConfiguration' | 'collaborators' | 'roles'; let section = 'roleConfiguration'; @@ -63,6 +66,7 @@ --Menu__item-active-background: var(--sand-200); --Menu__item-active-hover-background: var(--sand-200); --Menu__item-focus-outline-color: var(--sand-300); + padding: var(--size-x-small) 0; .heading { font-size: var(--text-size-small); diff --git a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts new file mode 100644 index 0000000000..2bc64cf4b6 --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts @@ -0,0 +1,76 @@ +import { getContext, setContext } from 'svelte'; +import { type Readable, type Writable, derived, writable } from 'svelte/store'; + +import { ConfiguredRole } from '@mathesar/models/ConfiguredRole'; +import type { Database } from '@mathesar/models/Database'; +import { Role } from '@mathesar/models/Role'; + +const contextKey = Symbol('database settings store'); + +class DatabaseSettingsContext { + database: Database; + + configuredRoles; + + roles; + + combinedRoles: Readable< + { name: string; role?: Role; configuredRole?: ConfiguredRole }[] + >; + + constructor(database: Database) { + this.database = database; + this.configuredRoles = database.fetchConfiguredRoles(); + this.roles = database.fetchRoles(); + this.combinedRoles = derived( + [this.roles, this.configuredRoles], + ([$roles, $configuredRoles]) => { + const isLoading = $configuredRoles.isLoading || $roles.isLoading; + if (isLoading) { + return []; + } + const isStable = $configuredRoles.isStable && $roles.isStable; + const roles = $roles.resolvedValue; + const configuredRoles = $configuredRoles.resolvedValue?.mapKeys( + (cr) => cr.name, + ); + if (isStable && roles && configuredRoles) { + return [...roles.values()].map((role) => ({ + name: role.name, + role, + configuredRole: configuredRoles.get(role.name), + })); + } + if ($configuredRoles.isStable && configuredRoles) { + [...configuredRoles.values()].map((configuredRole) => ({ + name: configuredRole.name, + configuredRole, + })); + } + return []; + }, + ); + } +} + +export function getDatabaseSettingsContext(): Readable { + const store = getContext>(contextKey); + if (store === undefined) { + throw Error('Database settings context has not been set'); + } + return store; +} + +export function setDatabaseSettingsContext( + database: Database, +): Readable { + let store = getContext>(contextKey); + const databaseSettingsContext = new DatabaseSettingsContext(database); + if (store !== undefined) { + store.set(databaseSettingsContext); + return store; + } + store = writable(databaseSettingsContext); + setContext(contextKey, store); + return store; +} From 3509aabe50966aec5810c9341cb6a5ceff0af3ad Mon Sep 17 00:00:00 2001 From: pavish Date: Fri, 16 Aug 2024 23:32:09 +0530 Subject: [PATCH 0655/1141] Implement basic Roles page --- .../src/pages/database/settings/Roles.svelte | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/mathesar_ui/src/pages/database/settings/Roles.svelte b/mathesar_ui/src/pages/database/settings/Roles.svelte index 3636bb90da..0aadca8891 100644 --- a/mathesar_ui/src/pages/database/settings/Roles.svelte +++ b/mathesar_ui/src/pages/database/settings/Roles.svelte @@ -1,5 +1,67 @@ -{$_('roles')} + + + {$_('roles')} + + + + + {#if $roles.isLoading} + + {:else if $roles.isOk} +
+ + {$_('role')} + + LOGIN + + {$_('child_roles')} + + {$_('actions')} + {#each roleList as role (role.name)} + {role.name} + + {role.login ? $_('yes') : $_('no')} + + + {#each role.members ?? [] as member (member.oid)} + {member.oid} + {/each} + + + + + {/each} + +
+ {:else if $roles.error} + {$roles.error} + {/if} +
+ + From 8ae3daa6dbb8cf4cf15a0d64091a9d96e002cd7a Mon Sep 17 00:00:00 2001 From: pavish Date: Sat, 17 Aug 2024 02:20:14 +0530 Subject: [PATCH 0656/1141] Add collaborators api and model --- mathesar_ui/src/api/rpc/collaborators.ts | 20 ++++++++++++++++++++ mathesar_ui/src/api/rpc/index.ts | 2 ++ mathesar_ui/src/models/Collaborator.ts | 23 +++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 mathesar_ui/src/api/rpc/collaborators.ts create mode 100644 mathesar_ui/src/models/Collaborator.ts diff --git a/mathesar_ui/src/api/rpc/collaborators.ts b/mathesar_ui/src/api/rpc/collaborators.ts new file mode 100644 index 0000000000..1fbfe47a68 --- /dev/null +++ b/mathesar_ui/src/api/rpc/collaborators.ts @@ -0,0 +1,20 @@ +import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; + +import type { RawConfiguredRole } from './configured_roles'; +import type { RawDatabase } from './databases'; + +export interface RawCollaborator { + id: number; + user_id: number; + database_id: RawDatabase['id']; + configured_role_id: RawConfiguredRole['id']; +} + +export const collaborators = { + list: rpcMethodTypeContainer< + { + database_id: RawDatabase['id']; + }, + Array + >(), +}; diff --git a/mathesar_ui/src/api/rpc/index.ts b/mathesar_ui/src/api/rpc/index.ts index cb0400b48f..a48b958e06 100644 --- a/mathesar_ui/src/api/rpc/index.ts +++ b/mathesar_ui/src/api/rpc/index.ts @@ -2,6 +2,7 @@ import Cookies from 'js-cookie'; import { buildRpcApi } from '@mathesar/packages/json-rpc-client-builder'; +import { collaborators } from './collaborators'; import { columns } from './columns'; import { configured_roles } from './configured_roles'; import { constraints } from './constraints'; @@ -18,6 +19,7 @@ export const api = buildRpcApi({ endpoint: '/api/rpc/v0/', getHeaders: () => ({ 'X-CSRFToken': Cookies.get('csrftoken') }), methodTree: { + collaborators, configured_roles, database_setup, databases, diff --git a/mathesar_ui/src/models/Collaborator.ts b/mathesar_ui/src/models/Collaborator.ts new file mode 100644 index 0000000000..48cf7ce151 --- /dev/null +++ b/mathesar_ui/src/models/Collaborator.ts @@ -0,0 +1,23 @@ +import type { RawCollaborator } from '@mathesar/api/rpc/collaborators'; + +import type { Database } from './Database'; + +export class Collaborator { + readonly id; + + readonly user_id; + + readonly configured_role_id; + + readonly database; + + constructor(props: { + database: Database; + rawCollaborator: RawCollaborator; + }) { + this.id = props.rawCollaborator.id; + this.user_id = props.rawCollaborator.user_id; + this.configured_role_id = props.rawCollaborator.configured_role_id; + this.database = props.database; + } +} From 3b5a443c1bae10f4f084fd0cd976b9ab9430a707 Mon Sep 17 00:00:00 2001 From: pavish Date: Sun, 18 Aug 2024 01:28:09 +0530 Subject: [PATCH 0657/1141] Implement Role Configuration page --- mathesar_ui/src/i18n/languages/en/dict.json | 10 ++ mathesar_ui/src/icons/index.ts | 1 + mathesar_ui/src/models/ConfiguredRole.ts | 14 ++ mathesar_ui/src/models/Role.ts | 31 ++++ mathesar_ui/src/models/Server.ts | 4 + .../pages/database/DatabasePageWrapper.svelte | 6 +- .../settings/RoleConfiguration.svelte | 75 -------- .../settings/databaseSettingsUtils.ts | 45 ++++- .../ConfigureRoleModal.svelte | 85 +++++++++ .../RoleConfiguration.svelte | 169 ++++++++++++++++++ mathesar_ui/src/routes/DatabaseRoute.svelte | 2 +- mathesar_ui/src/stores/AsyncStore.ts | 10 +- 12 files changed, 366 insertions(+), 86 deletions(-) delete mode 100644 mathesar_ui/src/pages/database/settings/RoleConfiguration.svelte create mode 100644 mathesar_ui/src/pages/database/settings/role-configuration/ConfigureRoleModal.svelte create mode 100644 mathesar_ui/src/pages/database/settings/role-configuration/RoleConfiguration.svelte diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index 3e5c6117be..fc051a044f 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -32,6 +32,7 @@ "ascending": "Ascending", "ascending_id": "Ascending ID", "attempt_exploration_recovery": "Attempt Exploration recovery", + "authenticate": "Authenticate", "automatically": "Automatically", "base_table_exploration_help": "The base table is the table that is being explored and determines the columns that are available for exploration.", "based_on": "Based on", @@ -78,6 +79,7 @@ "columns_to_move": "Columns to Move", "configure_in_mathesar": "Configure in Mathesar", "configure_password": "Configure Password", + "configure_value": "Configure ''{value}''", "confirm_and_create_table": "Confirm & create table", "confirm_delete_table": "To confirm the deletion of the [tableName] table, please enter the table name into the input field below.", "confirm_password": "Confirm Password", @@ -142,6 +144,7 @@ "databases": "Databases", "databases_matching_search": "{count, plural, one {{count} database matches [searchValue]} other {{count} databases match [searchValue]}}", "days": "Days", + "db_server": "DB server", "default": "Default", "default_value": "Default Value", "delete": "Delete", @@ -428,10 +431,13 @@ "release_notes_and_upgrade_instructions": "Release Notes and Upgrade Instructions", "released_date": "Released {date}", "remove": "Remove", + "remove_configuration": "Remove Configuration", + "remove_configuration_for_identifier": "Remove Configuration for [identifier]?", "remove_filters": "{count, plural, one {Remove Filter} other {Remove {count} Filters}}", "remove_grouping": "Remove Grouping", "remove_old_link_create_new": "Remove old link and create a new link?", "remove_sorting_type": "Remove {sortingType} Sorting", + "removing_role_configuration_warning": "Removing this configuration will prevent collaborators assigned to this role from accessing databases on server ''{server}''.", "rename_schema": "Rename [schemaName] Schema", "reset": "Reset", "restrict_to_unique": "Restrict to Unique", @@ -442,6 +448,10 @@ "reuse_credentials_from_known_connection": "Reuse credentials from a known connection", "role": "Role", "role_configuration": "Role Configuration", + "role_configuration_removed": "The role configuration has been removed successfully", + "role_configured_all_databases_in_server": "This role will be configured for all databases on DB Server ''{server}''", + "role_configured_successfully": "Role configured successfully", + "role_configured_successfully_new_password": "Role configured successfully with new password", "roles": "Roles", "row": "Row", "running_latest_version": "You are running the latest version", diff --git a/mathesar_ui/src/icons/index.ts b/mathesar_ui/src/icons/index.ts index ed004cbb89..9048599c0f 100644 --- a/mathesar_ui/src/icons/index.ts +++ b/mathesar_ui/src/icons/index.ts @@ -105,6 +105,7 @@ export const iconAddFilter: IconProps = { data: faFilter }; export const iconAddNew: IconProps = { data: faPlus }; export const iconAddUser: IconProps = { data: faUserPlus }; export const iconConfigure: IconProps = { data: faCogs }; +export const iconConfigurePassword = { data: faKey }; export const iconConnectDatabase = { data: connectDatabaseIcon }; export const iconCopyMajor: IconProps = { data: faCopy }; /** TODO: use faBinary once it's available (via newer FontAwesome version) */ diff --git a/mathesar_ui/src/models/ConfiguredRole.ts b/mathesar_ui/src/models/ConfiguredRole.ts index 6217f9e760..3ebfaf5a46 100644 --- a/mathesar_ui/src/models/ConfiguredRole.ts +++ b/mathesar_ui/src/models/ConfiguredRole.ts @@ -1,3 +1,4 @@ +import { api } from '@mathesar/api/rpc'; import type { RawConfiguredRole } from '@mathesar/api/rpc/configured_roles'; import type { Database } from './Database'; @@ -17,4 +18,17 @@ export class ConfiguredRole { this.name = props.rawConfiguredRole.name; this.database = props.database; } + + setPassword(password: string) { + return api.configured_roles + .set_password({ + configured_role_id: this.id, + password, + }) + .run(); + } + + delete() { + return api.configured_roles.delete({ configured_role_id: this.id }).run(); + } } diff --git a/mathesar_ui/src/models/Role.ts b/mathesar_ui/src/models/Role.ts index a39f997fcf..683f018311 100644 --- a/mathesar_ui/src/models/Role.ts +++ b/mathesar_ui/src/models/Role.ts @@ -1,5 +1,8 @@ +import { api } from '@mathesar/api/rpc'; import type { RawRole, RawRoleMember } from '@mathesar/api/rpc/roles'; +import { CancellablePromise } from '@mathesar/component-library'; +import { ConfiguredRole } from './ConfiguredRole'; import type { Database } from './Database'; export class Role { @@ -35,4 +38,32 @@ export class Role { this.members = props.rawRole.members; this.database = props.database; } + + configure(password: string): CancellablePromise { + const promise = api.configured_roles + .add({ + server_id: this.database.server.id, + name: this.name, + password, + }) + .run(); + + return new CancellablePromise( + (resolve, reject) => { + promise + .then( + (rawConfiguredRole) => + resolve( + new ConfiguredRole({ + database: this.database, + rawConfiguredRole, + }), + ), + reject, + ) + .catch(reject); + }, + () => promise.cancel(), + ); + } } diff --git a/mathesar_ui/src/models/Server.ts b/mathesar_ui/src/models/Server.ts index 203e13d398..1d51f3ed00 100644 --- a/mathesar_ui/src/models/Server.ts +++ b/mathesar_ui/src/models/Server.ts @@ -12,4 +12,8 @@ export class Server { this.host = props.rawServer.host; this.port = props.rawServer.port; } + + getConnectionString() { + return `${this.host}:${this.port}`; + } } diff --git a/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte b/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte index 67915d4daf..df871fcf34 100644 --- a/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte +++ b/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte @@ -53,7 +53,7 @@ restrictWidth cssVariables={{ '--max-layout-width': 'var(--max-layout-width-console-pages)', - '--layout-background-color': 'var(--sand-50)' + '--layout-background-color': 'var(--sand-50)', }} >
diff --git a/mathesar_ui/src/pages/database/settings/RoleConfiguration.svelte b/mathesar_ui/src/pages/database/settings/RoleConfiguration.svelte deleted file mode 100644 index 199e28d18e..0000000000 --- a/mathesar_ui/src/pages/database/settings/RoleConfiguration.svelte +++ /dev/null @@ -1,75 +0,0 @@ - - - - - {$_('role_configuration')} - - {#if isLoading} - - {:else} - {#if $combinedRoles.length > 0} -
- - {$_('role')} - {$_('actions')} - - {#each $combinedRoles as combinedRole (combinedRole.name)} - {combinedRole.name} - - {#if combinedRole.configuredRole} -
- - -
- {:else if combinedRole.role} - - {/if} -
- {/each} -
-
- {/if} - - {#if $configuredRoles.error} - {$configuredRoles.error} - {/if} - {#if $roles.error} - {$roles.error} - {/if} - {/if} -
- - diff --git a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts index 2bc64cf4b6..8c3a8bad1b 100644 --- a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts +++ b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts @@ -7,6 +7,12 @@ import { Role } from '@mathesar/models/Role'; const contextKey = Symbol('database settings store'); +export type CombinedLoginRole = { + name: string; + role?: Role; + configuredRole?: ConfiguredRole; +}; + class DatabaseSettingsContext { database: Database; @@ -14,15 +20,15 @@ class DatabaseSettingsContext { roles; - combinedRoles: Readable< - { name: string; role?: Role; configuredRole?: ConfiguredRole }[] - >; + combinedLoginRoles: Readable; + + collaborators; constructor(database: Database) { this.database = database; this.configuredRoles = database.fetchConfiguredRoles(); this.roles = database.fetchRoles(); - this.combinedRoles = derived( + this.combinedLoginRoles = derived( [this.roles, this.configuredRoles], ([$roles, $configuredRoles]) => { const isLoading = $configuredRoles.isLoading || $roles.isLoading; @@ -30,12 +36,14 @@ class DatabaseSettingsContext { return []; } const isStable = $configuredRoles.isStable && $roles.isStable; - const roles = $roles.resolvedValue; + const loginRoles = $roles.resolvedValue?.filterValues( + (value) => value.login, + ); const configuredRoles = $configuredRoles.resolvedValue?.mapKeys( (cr) => cr.name, ); - if (isStable && roles && configuredRoles) { - return [...roles.values()].map((role) => ({ + if (isStable && loginRoles && configuredRoles) { + return [...loginRoles.values()].map((role) => ({ name: role.name, role, configuredRole: configuredRoles.get(role.name), @@ -50,6 +58,29 @@ class DatabaseSettingsContext { return []; }, ); + this.collaborators = database.fetchCollaborators(); + } + + async configureRole(combinedLoginRole: CombinedLoginRole, password: string) { + if (combinedLoginRole.configuredRole) { + return combinedLoginRole.configuredRole.setPassword(password); + } + + if (combinedLoginRole.role) { + const configuredRole = await combinedLoginRole.role.configure(password); + this.configuredRoles.updateResolvedValue((configuredRoles) => + configuredRoles.with(configuredRole.id, configuredRole), + ); + } + + return undefined; + } + + async removeConfiguredRole(configuredRole: ConfiguredRole) { + await configuredRole.delete(); + this.configuredRoles.updateResolvedValue((configuredRoles) => + configuredRoles.without(configuredRole.id), + ); } } diff --git a/mathesar_ui/src/pages/database/settings/role-configuration/ConfigureRoleModal.svelte b/mathesar_ui/src/pages/database/settings/role-configuration/ConfigureRoleModal.svelte new file mode 100644 index 0000000000..434047a119 --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/role-configuration/ConfigureRoleModal.svelte @@ -0,0 +1,85 @@ + + + form.reset()}> + + {$_('configure_value', { + values: { value: combinedLoginRole.name }, + })} + +
+ + + + {$_('role_configured_all_databases_in_server', { + values: { + server: database.server.getConnectionString(), + }, + })} + + +
+ +
diff --git a/mathesar_ui/src/pages/database/settings/role-configuration/RoleConfiguration.svelte b/mathesar_ui/src/pages/database/settings/role-configuration/RoleConfiguration.svelte new file mode 100644 index 0000000000..e98b2ec426 --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/role-configuration/RoleConfiguration.svelte @@ -0,0 +1,169 @@ + + + + + {$_('role_configuration')} + + {#if isLoading} + + {:else} + {#if $combinedLoginRoles.length > 0} +
+ + {$_('role')} + {$_('actions')} + + {#each $combinedLoginRoles as combinedLoginRole (combinedLoginRole.name)} + + + {combinedLoginRole.name} + + + + {#if combinedLoginRole.configuredRole} +
+ + + confirm({ + title: { + component: PhraseContainingIdentifier, + props: { + identifier: combinedLoginRole.name, + wrappingString: $_( + 'remove_configuration_for_identifier', + ), + }, + }, + body: $_('removing_role_configuration_warning', { + values: { + server: + combinedLoginRole.configuredRole?.database.server.getConnectionString(), + }, + }), + proceedButton: { + label: $_('remove_configuration'), + icon: iconDeleteMajor, + }, + })} + label={$_('remove')} + onClick={() => removeConfiguredRole(combinedLoginRole)} + /> +
+ {:else if combinedLoginRole.role} + + {/if} +
+ {/each} +
+
+ {/if} + + {#if $configuredRoles.error} + + {$configuredRoles.error} + + {/if} + {#if $roles.error} + + {$roles.error} + + {/if} + {/if} +
+ +{#if targetCombinedLoginRole} + +{/if} + + diff --git a/mathesar_ui/src/routes/DatabaseRoute.svelte b/mathesar_ui/src/routes/DatabaseRoute.svelte index 28b73a564b..40c48b206a 100644 --- a/mathesar_ui/src/routes/DatabaseRoute.svelte +++ b/mathesar_ui/src/routes/DatabaseRoute.svelte @@ -11,7 +11,7 @@ import DatabasePageWrapper from '@mathesar/pages/database/DatabasePageWrapper.svelte'; import DatabasePageSchemasSection from '@mathesar/pages/database/schemas/SchemasSection.svelte'; import DatabaseCollaborators from '@mathesar/pages/database/settings/Collaborators.svelte'; - import DatabaseRoleConfiguration from '@mathesar/pages/database/settings/RoleConfiguration.svelte'; + import DatabaseRoleConfiguration from '@mathesar/pages/database/settings/role-configuration/RoleConfiguration.svelte'; import DatabaseRoles from '@mathesar/pages/database/settings/Roles.svelte'; import DatabasePageSettingsWrapper from '@mathesar/pages/database/settings/SettingsWrapper.svelte'; import ErrorPage from '@mathesar/pages/ErrorPage.svelte'; diff --git a/mathesar_ui/src/stores/AsyncStore.ts b/mathesar_ui/src/stores/AsyncStore.ts index 9ede933009..3d382ede09 100644 --- a/mathesar_ui/src/stores/AsyncStore.ts +++ b/mathesar_ui/src/stores/AsyncStore.ts @@ -177,12 +177,20 @@ export default class AsyncStore this.value.set(new AsyncStoreValue({ isLoading: false })); } + updateResolvedValue(updater: (resolvedValue: T) => T) { + const value = get(this.value); + if (value.isOk && value.resolvedValue) { + const updatedValue = updater(value.resolvedValue); + this.setResolvedValue(updatedValue); + } + } + protected beforeRun() { this.cancel(); this.value.update((v) => new AsyncStoreValue({ ...v, isLoading: true })); } - setResolvedValue(value: T) { + protected setResolvedValue(value: T) { this.cancel(); this.value.set( new AsyncStoreValue({ From dbeae24f511b11231cbab5488716bfd4c3137a7b Mon Sep 17 00:00:00 2001 From: pavish Date: Sun, 18 Aug 2024 01:28:59 +0530 Subject: [PATCH 0658/1141] Implement base collaborator page --- mathesar_ui/src/models/Database.ts | 15 ++++- .../database/settings/Collaborators.svelte | 5 -- .../collaborators/Collaborators.svelte | 67 +++++++++++++++++++ mathesar_ui/src/routes/DatabaseRoute.svelte | 2 +- 4 files changed, 82 insertions(+), 7 deletions(-) delete mode 100644 mathesar_ui/src/pages/database/settings/Collaborators.svelte create mode 100644 mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte diff --git a/mathesar_ui/src/models/Database.ts b/mathesar_ui/src/models/Database.ts index c8b1771553..1f72416ccf 100644 --- a/mathesar_ui/src/models/Database.ts +++ b/mathesar_ui/src/models/Database.ts @@ -1,8 +1,9 @@ import { api } from '@mathesar/api/rpc'; import type { RawDatabase } from '@mathesar/api/rpc/databases'; import AsyncRpcApiStore from '@mathesar/stores/AsyncRpcApiStore'; -import { SortedImmutableMap } from '@mathesar-component-library'; +import { ImmutableMap, SortedImmutableMap } from '@mathesar-component-library'; +import { Collaborator } from './Collaborator'; import { ConfiguredRole } from './ConfiguredRole'; import { Role } from './Role'; import type { Server } from './Server'; @@ -45,4 +46,16 @@ export class Database { ), }); } + + fetchCollaborators() { + return new AsyncRpcApiStore(api.collaborators.list, { + postProcess: (rawCollaborators) => + new ImmutableMap( + rawCollaborators.map((rawCollaborator) => [ + rawCollaborator.id, + new Collaborator({ database: this, rawCollaborator }), + ]), + ), + }); + } } diff --git a/mathesar_ui/src/pages/database/settings/Collaborators.svelte b/mathesar_ui/src/pages/database/settings/Collaborators.svelte deleted file mode 100644 index 5c75a672eb..0000000000 --- a/mathesar_ui/src/pages/database/settings/Collaborators.svelte +++ /dev/null @@ -1,5 +0,0 @@ - - -{$_('collaborators')} diff --git a/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte new file mode 100644 index 0000000000..b87613e613 --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte @@ -0,0 +1,67 @@ + + + + + {$_('collaborators')} + + + + + {#if isLoading} + + {:else if $collaborators.isOk} +
+ + {$_('mathesar_user')} + {$_('role')} + {$_('actions')} + {#each [...($collaborators.resolvedValue?.values() ?? [])] as collaborator (collaborator.id)} + {collaborator.user_id} + + {collaborator.configured_role_id} + + + + + {/each} + +
+ {:else if $roles.error} + {$roles.error} + {/if} +
+ + diff --git a/mathesar_ui/src/routes/DatabaseRoute.svelte b/mathesar_ui/src/routes/DatabaseRoute.svelte index 40c48b206a..9e0b759987 100644 --- a/mathesar_ui/src/routes/DatabaseRoute.svelte +++ b/mathesar_ui/src/routes/DatabaseRoute.svelte @@ -10,7 +10,7 @@ import type { Database } from '@mathesar/models/Database'; import DatabasePageWrapper from '@mathesar/pages/database/DatabasePageWrapper.svelte'; import DatabasePageSchemasSection from '@mathesar/pages/database/schemas/SchemasSection.svelte'; - import DatabaseCollaborators from '@mathesar/pages/database/settings/Collaborators.svelte'; + import DatabaseCollaborators from '@mathesar/pages/database/settings/collaborators/Collaborators.svelte'; import DatabaseRoleConfiguration from '@mathesar/pages/database/settings/role-configuration/RoleConfiguration.svelte'; import DatabaseRoles from '@mathesar/pages/database/settings/Roles.svelte'; import DatabasePageSettingsWrapper from '@mathesar/pages/database/settings/SettingsWrapper.svelte'; From 77dbbf1e3e050e2556408c9e86074964192865bb Mon Sep 17 00:00:00 2001 From: pavish Date: Sun, 18 Aug 2024 16:36:41 +0530 Subject: [PATCH 0659/1141] Display user and role information in collaborators page --- mathesar_ui/src/components/Errors.svelte | 3 +- mathesar_ui/src/i18n/languages/en/dict.json | 2 + .../src/pages/database/settings/Roles.svelte | 5 +- .../collaborators/CollaboratorRow.svelte | 63 +++++++++++++++++++ .../collaborators/Collaborators.svelte | 39 +++++++----- .../settings/databaseSettingsUtils.ts | 25 ++++++++ 6 files changed, 118 insertions(+), 19 deletions(-) create mode 100644 mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte diff --git a/mathesar_ui/src/components/Errors.svelte b/mathesar_ui/src/components/Errors.svelte index 10fb4c0834..f196d95a32 100644 --- a/mathesar_ui/src/components/Errors.svelte +++ b/mathesar_ui/src/components/Errors.svelte @@ -2,10 +2,11 @@ import ErrorBox from '@mathesar/components/message-boxes/ErrorBox.svelte'; export let errors: string[]; + export let fullWidth = false; {#if errors.length} - + {#if errors.length === 1} {errors[0]} {:else} diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index fc051a044f..5d4912658c 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -9,6 +9,7 @@ "action_cannot_be_undone": "This action cannot be undone", "actions": "Actions", "add": "Add", + "add_collaborator": "Add Collaborator", "add_columns_to_exploration_empty_message": "This exploration does not contain any columns. Edit the exploration to add columns to it.", "add_filter": "Add Filter", "add_new_filter": "Add New Filter", @@ -306,6 +307,7 @@ "many_to_one": "Many to One", "many_to_one_link_description": "Multiple [baseTable] records can link to the same [targetTable] record.", "mathesar": "Mathesar", + "mathesar_user": "Mathesar User", "max_time_unit": "Max Time Unit", "milliseconds": "Milliseconds", "min_time_unit": "Min Time Unit", diff --git a/mathesar_ui/src/pages/database/settings/Roles.svelte b/mathesar_ui/src/pages/database/settings/Roles.svelte index 0aadca8891..c0dc96f557 100644 --- a/mathesar_ui/src/pages/database/settings/Roles.svelte +++ b/mathesar_ui/src/pages/database/settings/Roles.svelte @@ -3,6 +3,7 @@ import GridTable from '@mathesar/components/grid-table/GridTable.svelte'; import GridTableCell from '@mathesar/components/grid-table/GridTableCell.svelte'; + import ErrorBox from '@mathesar/components/message-boxes/ErrorBox.svelte'; import { Button, Spinner } from '@mathesar-component-library'; import { getDatabaseSettingsContext } from './databaseSettingsUtils'; @@ -55,7 +56,9 @@
{:else if $roles.error} - {$roles.error} + + {$roles.error} + {/if} diff --git a/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte new file mode 100644 index 0000000000..aaf3a69df3 --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte @@ -0,0 +1,63 @@ + + + +
+ {#if user} +
{user.full_name ?? user.username}
+
{user.email}
+ {:else} + {collaborator.user_id} + {/if} +
+
+ +
+
+ {#if configuredRole} + {configuredRole.name} + {:else} + {collaborator.configured_role_id} + {/if} +
+
+ +
+
+
+ + + + + diff --git a/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte index b87613e613..019283858c 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte @@ -2,26 +2,38 @@ import { _ } from 'svelte-i18n'; import Icon from '@mathesar/component-library/icon/Icon.svelte'; + import Errors from '@mathesar/components/Errors.svelte'; import GridTable from '@mathesar/components/grid-table/GridTable.svelte'; import GridTableCell from '@mathesar/components/grid-table/GridTableCell.svelte'; - import { iconDeleteMajor } from '@mathesar/icons'; + import { iconAddNew } from '@mathesar/icons'; import AsyncRpcApiStore from '@mathesar/stores/AsyncRpcApiStore'; + import { isDefined } from '@mathesar/utils/language'; import { Button, Spinner } from '@mathesar-component-library'; import { getDatabaseSettingsContext } from '../databaseSettingsUtils'; import SettingsContentLayout from '../SettingsContentLayout.svelte'; + import CollaboratorRow from './CollaboratorRow.svelte'; + const databaseContext = getDatabaseSettingsContext(); - $: ({ database, roles, collaborators } = $databaseContext); + $: ({ database, configuredRoles, collaborators, users } = $databaseContext); $: void AsyncRpcApiStore.runBatched( [ collaborators.batchRunner({ database_id: database.id }), - roles.batchRunner({ database_id: database.id }), + configuredRoles.batchRunner({ server_id: database.server.id }), ], { onlyRunIfNotInitialized: true }, ); - $: isLoading = $collaborators.isLoading || $roles.isLoading; + $: void users.runIfNotInitialized(); + $: isLoading = + $collaborators.isLoading || $configuredRoles.isLoading || $users.isLoading; + $: isSuccess = $collaborators.isOk && $configuredRoles.isOk && $users.isOk; + $: errors = [ + $collaborators.error, + $configuredRoles.error, + $users.error, + ].filter((entry): entry is string => isDefined(entry)); @@ -30,32 +42,25 @@ {#if isLoading} - {:else if $collaborators.isOk} + {:else if isSuccess}
{$_('mathesar_user')} {$_('role')} {$_('actions')} {#each [...($collaborators.resolvedValue?.values() ?? [])] as collaborator (collaborator.id)} - {collaborator.user_id} - - {collaborator.configured_role_id} - - - - + {/each}
- {:else if $roles.error} - {$roles.error} + {:else} + {/if}
diff --git a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts index 8c3a8bad1b..d21a914256 100644 --- a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts +++ b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts @@ -1,9 +1,12 @@ import { getContext, setContext } from 'svelte'; import { type Readable, type Writable, derived, writable } from 'svelte/store'; +import userApi, { type User } from '@mathesar/api/rest/users'; import { ConfiguredRole } from '@mathesar/models/ConfiguredRole'; import type { Database } from '@mathesar/models/Database'; import { Role } from '@mathesar/models/Role'; +import AsyncStore from '@mathesar/stores/AsyncStore'; +import { CancellablePromise, ImmutableMap } from '@mathesar-component-library'; const contextKey = Symbol('database settings store'); @@ -13,6 +16,25 @@ export type CombinedLoginRole = { configuredRole?: ConfiguredRole; }; +// TODO: Make CancellablePromise chainable +const getUsersPromise = () => { + const promise = userApi.list(); + return new CancellablePromise>( + (resolve, reject) => { + promise + .then( + (response) => + resolve( + new ImmutableMap(response.results.map((user) => [user.id, user])), + ), + (err) => reject(err), + ) + .catch((err) => reject(err)); + }, + () => promise.cancel(), + ); +}; + class DatabaseSettingsContext { database: Database; @@ -24,6 +46,8 @@ class DatabaseSettingsContext { collaborators; + users: AsyncStore>; + constructor(database: Database) { this.database = database; this.configuredRoles = database.fetchConfiguredRoles(); @@ -59,6 +83,7 @@ class DatabaseSettingsContext { }, ); this.collaborators = database.fetchCollaborators(); + this.users = new AsyncStore(getUsersPromise); } async configureRole(combinedLoginRole: CombinedLoginRole, password: string) { From cd97e2d062b25f2500e3668add0ecea60d3d7591 Mon Sep 17 00:00:00 2001 From: pavish Date: Sun, 18 Aug 2024 18:59:20 +0530 Subject: [PATCH 0660/1141] Implement add collaborator modal --- mathesar_ui/src/api/rpc/collaborators.ts | 8 ++ mathesar_ui/src/i18n/languages/en/dict.json | 2 + mathesar_ui/src/models/Database.ts | 37 +++++- .../collaborators/AddCollaboratorModal.svelte | 110 ++++++++++++++++++ .../collaborators/CollaboratorRow.svelte | 2 +- .../collaborators/Collaborators.svelte | 23 +++- .../settings/databaseSettingsUtils.ts | 13 +++ 7 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte diff --git a/mathesar_ui/src/api/rpc/collaborators.ts b/mathesar_ui/src/api/rpc/collaborators.ts index 1fbfe47a68..fceb67c9dc 100644 --- a/mathesar_ui/src/api/rpc/collaborators.ts +++ b/mathesar_ui/src/api/rpc/collaborators.ts @@ -17,4 +17,12 @@ export const collaborators = { }, Array >(), + add: rpcMethodTypeContainer< + { + database_id: RawDatabase['id']; + user_id: number; + configured_role_id: RawConfiguredRole['id']; + }, + RawCollaborator + >(), }; diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index 5d4912658c..7dc4793ca0 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -54,6 +54,7 @@ "cleaning_up": "Cleaning up", "clear": "Clear", "clear_value": "Clear value", + "collaborator_added_successfully": "Collaborator added successfully", "collaborators": "Collaborators", "column": "Column", "column_added_number_of_times": "{count, plural, one {This column has been added once.} other {This column has been added {count} times.}}", @@ -492,6 +493,7 @@ "select_columns_to_hide": "Select Columns to Hide", "select_columns_view_properties": "Select a column to view it's properties.", "select_permission": "Select Permission", + "select_role": "Select Role", "select_table": "Select Table", "select_type": "Select Type", "select_user": "Select User", diff --git a/mathesar_ui/src/models/Database.ts b/mathesar_ui/src/models/Database.ts index 1f72416ccf..b39c443b6d 100644 --- a/mathesar_ui/src/models/Database.ts +++ b/mathesar_ui/src/models/Database.ts @@ -1,7 +1,11 @@ import { api } from '@mathesar/api/rpc'; import type { RawDatabase } from '@mathesar/api/rpc/databases'; import AsyncRpcApiStore from '@mathesar/stores/AsyncRpcApiStore'; -import { ImmutableMap, SortedImmutableMap } from '@mathesar-component-library'; +import { + CancellablePromise, + ImmutableMap, + SortedImmutableMap, +} from '@mathesar-component-library'; import { Collaborator } from './Collaborator'; import { ConfiguredRole } from './ConfiguredRole'; @@ -58,4 +62,35 @@ export class Database { ), }); } + + addCollaborator( + userId: number, + configuredRoleId: ConfiguredRole['id'], + ): CancellablePromise { + const promise = api.collaborators + .add({ + database_id: this.id, + user_id: userId, + configured_role_id: configuredRoleId, + }) + .run(); + + return new CancellablePromise( + (resolve, reject) => { + promise + .then( + (rawCollaborator) => + resolve( + new Collaborator({ + database: this, + rawCollaborator, + }), + ), + reject, + ) + .catch(reject); + }, + () => promise.cancel(), + ); + } } diff --git a/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte b/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte new file mode 100644 index 0000000000..6b5052735a --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte @@ -0,0 +1,110 @@ + + + form.reset()}> + + {$_('add_collaborator')} + +
+ user.id), + getLabel: (option) => { + if (option) { + return usersMap.get(option)?.username ?? String(option); + } + return $_('select_user'); + }, + autoSelect: 'none', + }, + }} + /> + r.id), + getLabel: (option) => { + if (option) { + return configuredRolesMap.get(option)?.name ?? String(option); + } + return $_('select_role'); + }, + autoSelect: 'none', + }, + }} + /> +
+ +
diff --git a/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte index aaf3a69df3..a4cc0590ff 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte @@ -21,7 +21,7 @@
{#if user} -
{user.full_name ?? user.username}
+
{user.full_name || user.username}
{user.email}
{:else} {collaborator.user_id} diff --git a/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte index 019283858c..a26d9c8078 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte @@ -7,15 +7,19 @@ import GridTableCell from '@mathesar/components/grid-table/GridTableCell.svelte'; import { iconAddNew } from '@mathesar/icons'; import AsyncRpcApiStore from '@mathesar/stores/AsyncRpcApiStore'; + import { modal } from '@mathesar/stores/modal'; import { isDefined } from '@mathesar/utils/language'; import { Button, Spinner } from '@mathesar-component-library'; import { getDatabaseSettingsContext } from '../databaseSettingsUtils'; import SettingsContentLayout from '../SettingsContentLayout.svelte'; + import AddCollaboratorModel from './AddCollaboratorModal.svelte'; import CollaboratorRow from './CollaboratorRow.svelte'; const databaseContext = getDatabaseSettingsContext(); + const addCollaboratorModal = modal.spawnModalController(); + $: ({ database, configuredRoles, collaborators, users } = $databaseContext); $: void AsyncRpcApiStore.runBatched( @@ -41,10 +45,12 @@ {$_('collaborators')} - + {#if isSuccess} + + {/if} {#if isLoading} @@ -64,6 +70,15 @@ {/if} +{#if $users.resolvedValue && $configuredRoles.resolvedValue && $collaborators.resolvedValue} + +{/if} + diff --git a/mathesar_ui/src/routes/DatabaseRoute.svelte b/mathesar_ui/src/routes/DatabaseRoute.svelte index 9e0b759987..bc2f3c465a 100644 --- a/mathesar_ui/src/routes/DatabaseRoute.svelte +++ b/mathesar_ui/src/routes/DatabaseRoute.svelte @@ -12,7 +12,7 @@ import DatabasePageSchemasSection from '@mathesar/pages/database/schemas/SchemasSection.svelte'; import DatabaseCollaborators from '@mathesar/pages/database/settings/collaborators/Collaborators.svelte'; import DatabaseRoleConfiguration from '@mathesar/pages/database/settings/role-configuration/RoleConfiguration.svelte'; - import DatabaseRoles from '@mathesar/pages/database/settings/Roles.svelte'; + import DatabaseRoles from '@mathesar/pages/database/settings/roles/Roles.svelte'; import DatabasePageSettingsWrapper from '@mathesar/pages/database/settings/SettingsWrapper.svelte'; import ErrorPage from '@mathesar/pages/ErrorPage.svelte'; import { databasesStore } from '@mathesar/stores/databases'; From 3566d3d1a1f74f574e9b2deb77044edd0fdb226b Mon Sep 17 00:00:00 2001 From: pavish Date: Sun, 18 Aug 2024 22:41:54 +0530 Subject: [PATCH 0665/1141] fix type errors --- mathesar_ui/src/models/Collaborator.ts | 2 +- .../pages/database/settings/SettingsWrapper.svelte | 2 +- .../collaborators/AddCollaboratorModal.svelte | 6 +++--- .../settings/collaborators/CollaboratorRow.svelte | 2 +- .../settings/collaborators/Collaborators.svelte | 2 +- .../EditRoleForCollaboratorModal.svelte | 6 +++--- .../collaborators/SelectConfiguredRoleField.svelte | 6 +++--- .../database/settings/databaseSettingsUtils.ts | 6 +++--- .../database/settings/roles/CreateRoleModal.svelte | 2 +- mathesar_ui/src/routes/RootRoute.svelte | 8 ++++---- mathesar_ui/src/stores/databases.ts | 14 +++++++++----- 11 files changed, 30 insertions(+), 26 deletions(-) diff --git a/mathesar_ui/src/models/Collaborator.ts b/mathesar_ui/src/models/Collaborator.ts index 7577703ace..4ba7dae50e 100644 --- a/mathesar_ui/src/models/Collaborator.ts +++ b/mathesar_ui/src/models/Collaborator.ts @@ -2,7 +2,7 @@ import { api } from '@mathesar/api/rpc'; import type { RawCollaborator } from '@mathesar/api/rpc/collaborators'; import { CancellablePromise } from '@mathesar/component-library'; -import { ConfiguredRole } from './ConfiguredRole'; +import type { ConfiguredRole } from './ConfiguredRole'; import type { Database } from './Database'; export class Collaborator { diff --git a/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte b/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte index 786b7f3ebc..1831c267a2 100644 --- a/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte +++ b/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte @@ -2,7 +2,7 @@ import { _ } from 'svelte-i18n'; import PageLayoutWithSidebar from '@mathesar/layouts/PageLayoutWithSidebar.svelte'; - import { Database } from '@mathesar/models/Database'; + import type { Database } from '@mathesar/models/Database'; import { getDatabaseCollaboratorsUrl, getDatabaseRoleConfigurationUrl, diff --git a/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte b/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte index 00e45edce3..79b4825a90 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte @@ -8,12 +8,12 @@ requiredField, } from '@mathesar/components/form'; import Field from '@mathesar/components/form/Field.svelte'; - import { Collaborator } from '@mathesar/models/Collaborator'; - import { ConfiguredRole } from '@mathesar/models/ConfiguredRole'; + import type { Collaborator } from '@mathesar/models/Collaborator'; + import type { ConfiguredRole } from '@mathesar/models/ConfiguredRole'; import { toast } from '@mathesar/stores/toast'; import { ControlledModal, - ImmutableMap, + type ImmutableMap, type ModalController, Select, portalToWindowFooter, diff --git a/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte index 90e17bf5f7..3bce77a219 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte @@ -4,7 +4,7 @@ import Icon from '@mathesar/component-library/icon/Icon.svelte'; import GridTableCell from '@mathesar/components/grid-table/GridTableCell.svelte'; import { iconDeleteMajor, iconEdit } from '@mathesar/icons'; - import { Collaborator } from '@mathesar/models/Collaborator'; + import type { Collaborator } from '@mathesar/models/Collaborator'; import { confirmDelete } from '@mathesar/stores/confirmation'; import { Button, SpinnerButton } from '@mathesar-component-library'; diff --git a/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte index ca8ca6cfe0..f74c581d0e 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte @@ -6,7 +6,7 @@ import GridTable from '@mathesar/components/grid-table/GridTable.svelte'; import GridTableCell from '@mathesar/components/grid-table/GridTableCell.svelte'; import { iconAddNew } from '@mathesar/icons'; - import { Collaborator } from '@mathesar/models/Collaborator'; + import type { Collaborator } from '@mathesar/models/Collaborator'; import AsyncRpcApiStore from '@mathesar/stores/AsyncRpcApiStore'; import { modal } from '@mathesar/stores/modal'; import { isDefined } from '@mathesar/utils/language'; diff --git a/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte b/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte index 667692e72c..9d6ca95e6c 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte @@ -7,12 +7,12 @@ makeForm, requiredField, } from '@mathesar/components/form'; - import { Collaborator } from '@mathesar/models/Collaborator'; - import { ConfiguredRole } from '@mathesar/models/ConfiguredRole'; + import type { Collaborator } from '@mathesar/models/Collaborator'; + import type { ConfiguredRole } from '@mathesar/models/ConfiguredRole'; import { toast } from '@mathesar/stores/toast'; import { ControlledModal, - ImmutableMap, + type ImmutableMap, type ModalController, portalToWindowFooter, } from '@mathesar-component-library'; diff --git a/mathesar_ui/src/pages/database/settings/collaborators/SelectConfiguredRoleField.svelte b/mathesar_ui/src/pages/database/settings/collaborators/SelectConfiguredRoleField.svelte index a231aafbbb..b508350f47 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/SelectConfiguredRoleField.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/SelectConfiguredRoleField.svelte @@ -1,10 +1,10 @@ + + form.reset()}> + + {$_('edit_child_roles_for_parent', { + values: { + parent: parentRole.name, + }, + })} + +
+ +
+ {#if $memberOids.size > 0} + {#each [...$memberOids] as memberOid (memberOid)} +
+ + {rolesMap.get(memberOid)?.name ?? memberOid} + + + + +
+ {/each} + {:else} + + {$_('role_has_no_child_roles')} + + {/if} +
+
+ +
+ + diff --git a/mathesar_ui/src/pages/database/settings/roles/RoleRow.svelte b/mathesar_ui/src/pages/database/settings/roles/RoleRow.svelte new file mode 100644 index 0000000000..88ead522d6 --- /dev/null +++ b/mathesar_ui/src/pages/database/settings/roles/RoleRow.svelte @@ -0,0 +1,105 @@ + + +{role.name} + + {#if role.login} + {$_('yes')} + {/if} + + +
+
+ {#each [...$members.values()] as member (member.oid)} + + {rolesMap.get(member.oid)?.name ?? ''} + + {/each} +
+
+ +
+
+
+ + + confirm({ + title: { + component: PhraseContainingIdentifier, + props: { + identifier: role.name, + wrappingString: $_('drop_role_with_identifier'), + }, + }, + body: [ + $_('action_cannot_be_undone'), + $_('drop_role_warning'), + $_('are_you_sure_to_proceed'), + ], + proceedButton: { + label: $_('drop_role'), + icon: iconDeleteMajor, + }, + })} + label={$_('drop_role')} + onClick={dropRole} + /> + + + diff --git a/mathesar_ui/src/pages/database/settings/roles/Roles.svelte b/mathesar_ui/src/pages/database/settings/roles/Roles.svelte index 3a07394bb8..6739b68f8b 100644 --- a/mathesar_ui/src/pages/database/settings/roles/Roles.svelte +++ b/mathesar_ui/src/pages/database/settings/roles/Roles.svelte @@ -4,7 +4,8 @@ import GridTable from '@mathesar/components/grid-table/GridTable.svelte'; import GridTableCell from '@mathesar/components/grid-table/GridTableCell.svelte'; import ErrorBox from '@mathesar/components/message-boxes/ErrorBox.svelte'; - import { iconAddNew, iconEdit } from '@mathesar/icons'; + import { iconAddNew } from '@mathesar/icons'; + import type { Role } from '@mathesar/models/Role'; import { modal } from '@mathesar/stores/modal'; import { Button, Icon, Spinner } from '@mathesar-component-library'; @@ -12,14 +13,24 @@ import SettingsContentLayout from '../SettingsContentLayout.svelte'; import CreateRoleModal from './CreateRoleModal.svelte'; + import ModifyRoleMembers from './ModifyRoleMembers.svelte'; + import RoleRow from './RoleRow.svelte'; const databaseContext = getDatabaseSettingsContext(); const createRoleModalController = modal.spawnModalController(); + const modifyRoleMembersModalController = modal.spawnModalController(); $: ({ database, roles } = $databaseContext); $: void roles.runIfNotInitialized({ database_id: database.id }); $: roleList = [...($roles.resolvedValue?.values() ?? [])]; + + let targetRole: Role | undefined = undefined; + + function modifyMembersForRole(role: Role) { + targetRole = role; + modifyRoleMembersModalController.open(); + } @@ -39,7 +50,7 @@ {#if $roles.isLoading} - {:else if $roles.isOk} + {:else if $roles.isOk && $roles.resolvedValue}
{$_('role')} @@ -49,32 +60,12 @@ {$_('child_roles')} {$_('actions')} - {#each roleList as role (role.name)} - {role.name} - - {role.login ? $_('yes') : $_('no')} - - -
-
- {#each role.members ?? [] as member (member.oid)} - - {$roles.resolvedValue?.get(member.oid)?.name ?? ''} - - {/each} -
-
- -
-
-
- - - + {#each roleList as role (role.oid)} + {/each}
@@ -87,24 +78,17 @@ +{#if $roles.resolvedValue && targetRole} + +{/if} + From d7f14ae969178393cbb3fda9017f2c1ea5afe842 Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 22 Aug 2024 19:05:13 +0530 Subject: [PATCH 0704/1141] Reset stores when items in dependent stores are deleted --- .../pages/database/settings/databaseSettingsUtils.ts | 10 ++++++++++ .../role-configuration/ConfigureRoleModal.svelte | 2 -- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts index 8821495ebd..8699581c6f 100644 --- a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts +++ b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts @@ -107,6 +107,13 @@ class DatabaseSettingsContext { this.configuredRoles.updateResolvedValue((configuredRoles) => configuredRoles.without(configuredRole.id), ); + /** + * When a configured role is removed from the Role Configuration page, + * Collaborators list needs to be reset, since the drop statement cascades. + * + * TODO_BETA: Discuss on whether we should cascade or throw error? + */ + this.collaborators.reset(); } async addCollaborator( @@ -141,6 +148,9 @@ class DatabaseSettingsContext { async deleteRole(role: Role) { await role.delete(); this.roles.updateResolvedValue((r) => r.without(role.oid)); + // When a role is deleted, both Collaborators & ConfiguredRoles needs to be reset + this.configuredRoles.reset(); + this.collaborators.reset(); } } diff --git a/mathesar_ui/src/pages/database/settings/role-configuration/ConfigureRoleModal.svelte b/mathesar_ui/src/pages/database/settings/role-configuration/ConfigureRoleModal.svelte index 434047a119..c7f2eab7a5 100644 --- a/mathesar_ui/src/pages/database/settings/role-configuration/ConfigureRoleModal.svelte +++ b/mathesar_ui/src/pages/database/settings/role-configuration/ConfigureRoleModal.svelte @@ -39,7 +39,6 @@ } else { toast.success($_('role_configured_successfully')); } - form.reset(); } @@ -74,7 +73,6 @@ {form} catchErrors onCancel={() => { - form.reset(); controller.close(); }} onProceed={configureRole} From a9707925705e564a82a1b50a4da303ef6ededa0c Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 22 Aug 2024 19:09:18 +0530 Subject: [PATCH 0705/1141] change get_roles->list_roles and extract pg_roles logic to role_info_table --- db/roles/operations/select.py | 4 +- db/sql/00_msar.sql | 114 +++++++++++------------ db/sql/test_00_msar.sql | 20 ++-- db/tests/roles/operations/test_select.py | 6 +- mathesar/rpc/roles.py | 16 +++- 5 files changed, 81 insertions(+), 79 deletions(-) diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py index 1d0d94434b..9ad7131f21 100644 --- a/db/roles/operations/select.py +++ b/db/roles/operations/select.py @@ -1,8 +1,8 @@ from db.connection import exec_msar_func -def get_roles(conn): - return exec_msar_func(conn, 'get_roles').fetchone()[0] +def list_roles(conn): + return exec_msar_func(conn, 'list_roles').fetchone()[0] def list_db_priv(db_name, conn): diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 015811e03a..b33fe5d133 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -897,7 +897,49 @@ FROM ( $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION msar.get_roles() RETURNS jsonb AS $$/* +CREATE OR REPLACE FUNCTION msar.role_info_table() RETURNS TABLE +( + oid oid, + name name, + super boolean, + inherits boolean, + create_role boolean, + create_db boolean, + login boolean, + description text, + members jsonb +) AS $$/* +Returns a table describing all the roles present on the database server. +*/ +WITH rolemembers as ( + SELECT + pgr.oid AS oid, + jsonb_agg( + jsonb_build_object( + 'oid', pgm.member::bigint, + 'admin', pgm.admin_option + ) + ) AS members + FROM pg_catalog.pg_roles pgr + INNER JOIN pg_catalog.pg_auth_members pgm ON pgr.oid=pgm.roleid + GROUP BY pgr.oid +) +SELECT + r.oid::bigint AS oid, + r.rolname AS name, + r.rolsuper AS super, + r.rolinherit AS inherits, + r.rolcreaterole AS create_role, + r.rolcreatedb AS create_db, + r.rolcanlogin AS login, + pg_catalog.shobj_description(r.oid, 'pg_authid') AS description, + rolemembers.members AS members +FROM pg_catalog.pg_roles r +LEFT OUTER JOIN rolemembers ON r.oid = rolemembers.oid; +$$ LANGUAGE SQL STABLE; + + +CREATE OR REPLACE FUNCTION msar.list_roles() RETURNS jsonb AS $$/* Return a json array of objects with the list of roles in a database server, excluding pg system roles. @@ -916,35 +958,9 @@ Each returned JSON object in the array has the form: ]|null> } */ -WITH rolemembers as ( - SELECT - pgr.oid AS oid, - jsonb_agg( - jsonb_build_object( - 'oid', pgm.member::bigint, - 'admin', pgm.admin_option - ) - ) AS members - FROM pg_catalog.pg_roles pgr - INNER JOIN pg_catalog.pg_auth_members pgm ON pgr.oid=pgm.roleid - GROUP BY pgr.oid -) SELECT jsonb_agg(role_data) -FROM ( - SELECT - r.oid::bigint AS oid, - r.rolname AS name, - r.rolsuper AS super, - r.rolinherit AS inherits, - r.rolcreaterole AS create_role, - r.rolcreatedb AS create_db, - r.rolcanlogin AS login, - pg_catalog.shobj_description(r.oid, 'pg_authid') AS description, - rolemembers.members AS members - FROM pg_catalog.pg_roles r - LEFT OUTER JOIN rolemembers ON r.oid = rolemembers.oid - WHERE r.rolname NOT LIKE 'pg_%' -) AS role_data; +FROM msar.role_info_table() AS role_data +WHERE role_data.name NOT LIKE 'pg_%'; $$ LANGUAGE SQL STABLE; @@ -966,33 +982,9 @@ The returned JSON object has the form: ]|null> } */ -WITH rolemembers as ( - SELECT - pgr.oid AS oid, - jsonb_agg( - jsonb_build_object( - 'oid', pgm.member::bigint, - 'admin', pgm.admin_option - ) - ) AS members - FROM pg_catalog.pg_roles pgr - INNER JOIN pg_catalog.pg_auth_members pgm ON pgr.oid=pgm.roleid - GROUP BY pgr.oid -) -SELECT jsonb_build_object( - 'oid', r.oid::bigint, - 'name', r.rolname, - 'super', r.rolsuper, - 'inherits', r.rolinherit, - 'create_role', r.rolcreaterole, - 'create_db', r.rolcreatedb, - 'login', r.rolcanlogin, - 'description', pg_catalog.shobj_description(r.oid, 'pg_authid'), - 'members', rolemembers.members -) -FROM pg_catalog.pg_roles r - LEFT OUTER JOIN rolemembers ON r.oid = rolemembers.oid -WHERE r.rolname = rolename; +SELECT to_jsonb(role_data) +FROM msar.role_info_table() AS role_data +WHERE role_data.name = rolename; $$ LANGUAGE SQL STABLE; @@ -1069,8 +1061,8 @@ Args: password_: The password for the user to set, unquoted. */ BEGIN - PERFORM __msar.exec_ddl('CREATE USER %I WITH PASSWORD %L', username, password_); - PERFORM __msar.exec_ddl( + EXECUTE format('CREATE USER %I WITH PASSWORD %L', username, password_); + EXECUTE format( 'GRANT CREATE, CONNECT, TEMP ON DATABASE %I TO %I', current_database()::text, username @@ -1129,10 +1121,10 @@ DECLARE mathesar_schemas text[] := ARRAY['mathesar_types', '__msar', 'msar']; BEGIN CASE WHEN login_ THEN - PERFORM create_basic_mathesar_user(rolename, password_); + PERFORM msar.create_basic_mathesar_user(rolename, password_); ELSE - PERFORM __msar.exec_ddl('CREATE ROLE %I', rolename); - PERFORM __msar.exec_ddl( + EXECUTE format('CREATE ROLE %I', rolename); + EXECUTE format( 'GRANT CREATE, TEMP ON DATABASE %I TO %I', current_database()::text, rolename diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 9c0030deaa..4bb11424aa 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2760,18 +2760,18 @@ END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION test_get_roles() RETURNS SETOF TEXT AS $$ +CREATE OR REPLACE FUNCTION test_list_roles() RETURNS SETOF TEXT AS $$ DECLARE initial_role_count int; foo_role jsonb; bar_role jsonb; BEGIN - SELECT jsonb_array_length(msar.get_roles()) INTO initial_role_count; + SELECT jsonb_array_length(msar.list_roles()) INTO initial_role_count; -- Create role and check if role is present in response & count is increased CREATE ROLE foo; - RETURN NEXT is(jsonb_array_length(msar.get_roles()), initial_role_count + 1); - SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; + RETURN NEXT is(jsonb_array_length(msar.list_roles()), initial_role_count + 1); + SELECT jsonb_path_query(msar.list_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; -- Check if role has expected properties RETURN NEXT is(jsonb_typeof(foo_role), 'object'); @@ -2785,7 +2785,7 @@ BEGIN -- Modify properties and check role again ALTER ROLE foo WITH CREATEDB CREATEROLE LOGIN NOINHERIT; - SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; + SELECT jsonb_path_query(msar.list_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; RETURN NEXT is((foo_role->>'super')::boolean, false); RETURN NEXT is((foo_role->>'inherits')::boolean, false); RETURN NEXT is((foo_role->>'create_role')::boolean, true); @@ -2794,15 +2794,15 @@ BEGIN -- Add comment and check if comment is present COMMENT ON ROLE foo IS 'A test role'; - SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; + SELECT jsonb_path_query(msar.list_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; RETURN NEXT is(foo_role->'description'#>>'{}', 'A test role'); -- Add members and check result CREATE ROLE bar; GRANT foo TO bar; - RETURN NEXT is(jsonb_array_length(msar.get_roles()), initial_role_count + 2); - SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; - SELECT jsonb_path_query(msar.get_roles(), '$[*] ? (@.name == "bar")') INTO bar_role; + RETURN NEXT is(jsonb_array_length(msar.list_roles()), initial_role_count + 2); + SELECT jsonb_path_query(msar.list_roles(), '$[*] ? (@.name == "foo")') INTO foo_role; + SELECT jsonb_path_query(msar.list_roles(), '$[*] ? (@.name == "bar")') INTO bar_role; RETURN NEXT is(jsonb_typeof(foo_role->'members'), 'array'); RETURN NEXT is( foo_role->'members'->0->>'oid', bar_role->>'oid' @@ -2811,7 +2811,7 @@ BEGIN -- Drop role and ensure role is not present in response DROP ROLE foo; - RETURN NEXT ok(NOT jsonb_path_exists(msar.get_roles(), '$[*] ? (@.name == "foo")')); + RETURN NEXT ok(NOT jsonb_path_exists(msar.list_roles(), '$[*] ? (@.name == "foo")')); END; $$ LANGUAGE plpgsql; diff --git a/db/tests/roles/operations/test_select.py b/db/tests/roles/operations/test_select.py index 40579e99ac..81536d6967 100644 --- a/db/tests/roles/operations/test_select.py +++ b/db/tests/roles/operations/test_select.py @@ -2,9 +2,9 @@ from db.roles.operations import select as ma_sel -def test_get_roles(): +def test_list_roles(): with patch.object(ma_sel, 'exec_msar_func') as mock_exec: mock_exec.return_value.fetchone = lambda: ('a', 'b') - result = ma_sel.get_roles('conn') - mock_exec.assert_called_once_with('conn', 'get_roles') + result = ma_sel.list_roles('conn') + mock_exec.assert_called_once_with('conn', 'list_roles') assert result == 'a' diff --git a/mathesar/rpc/roles.py b/mathesar/rpc/roles.py index 1c940e3e7f..f88a90ffc2 100644 --- a/mathesar/rpc/roles.py +++ b/mathesar/rpc/roles.py @@ -8,7 +8,7 @@ from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect -from db.roles.operations.select import get_roles +from db.roles.operations.select import list_roles from db.roles.operations.create import create_role @@ -56,7 +56,17 @@ class RoleInfo(TypedDict): @classmethod def from_dict(cls, d): - return cls + return cls( + oid=d["oid"], + name=d["name"], + super=d["super"], + inherits=d["inherits"], + create_role=d["create_role"], + create_db=d["create_db"], + login=d["login"], + description=d["description"], + members=d["members"] + ) @rpc_method(name="roles.list") @@ -75,7 +85,7 @@ def list_(*, database_id: int, **kwargs) -> list[RoleInfo]: """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: - roles = get_roles(conn) + roles = list_roles(conn) return [RoleInfo.from_dict(role) for role in roles] From ad3d5fbc223b36075e0930b7c8f9105751e2b8fb Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 22 Aug 2024 19:22:28 +0530 Subject: [PATCH 0706/1141] Make request for adding roles --- mathesar_ui/src/i18n/languages/en/dict.json | 3 +- mathesar_ui/src/models/Database.ts | 33 +++++++++++++++++++ .../settings/databaseSettingsUtils.ts | 17 ++++++++++ .../settings/roles/CreateRoleModal.svelte | 21 ++++++++++-- 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index ecf6615823..4f6eab083f 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -465,7 +465,8 @@ "role_configured_all_databases_in_server": "This role will be configured for all databases on DB Server ''{server}''", "role_configured_successfully": "Role configured successfully", "role_configured_successfully_new_password": "Role configured successfully with new password", - "role_dropped_successfully": "The role has been dropped successfully", + "role_created_successfully": "The Role has been created successfully", + "role_dropped_successfully": "The Role has been dropped successfully", "role_has_no_child_roles": "The Role does not have any Child Roles", "role_name": "Role Name", "roles": "Roles", diff --git a/mathesar_ui/src/models/Database.ts b/mathesar_ui/src/models/Database.ts index b39c443b6d..fe856c478a 100644 --- a/mathesar_ui/src/models/Database.ts +++ b/mathesar_ui/src/models/Database.ts @@ -93,4 +93,37 @@ export class Database { () => promise.cancel(), ); } + + addRole( + roleName: string, + login: boolean, + password?: string, + ): CancellablePromise { + const promise = api.roles + .add({ + database_id: this.id, + rolename: roleName, + login, + password, + }) + .run(); + + return new CancellablePromise( + (resolve, reject) => { + promise + .then( + (rawRole) => + resolve( + new Role({ + database: this, + rawRole, + }), + ), + reject, + ) + .catch(reject); + }, + () => promise.cancel(), + ); + } } diff --git a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts index 8699581c6f..9a86a04984 100644 --- a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts +++ b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts @@ -145,6 +145,23 @@ class DatabaseSettingsContext { this.collaborators.updateResolvedValue((c) => c.without(collaborator.id)); } + async addRole( + props: + | { + roleName: Role['name']; + login: false; + password?: never; + } + | { roleName: Role['name']; login: true; password: string }, + ) { + const newRole = await this.database.addRole( + props.roleName, + props.login, + props.password, + ); + this.roles.updateResolvedValue((r) => r.with(newRole.oid, newRole)); + } + async deleteRole(role: Role) { await role.delete(); this.roles.updateResolvedValue((r) => r.without(role.oid)); diff --git a/mathesar_ui/src/pages/database/settings/roles/CreateRoleModal.svelte b/mathesar_ui/src/pages/database/settings/roles/CreateRoleModal.svelte index e715031b29..ccc193b4f1 100644 --- a/mathesar_ui/src/pages/database/settings/roles/CreateRoleModal.svelte +++ b/mathesar_ui/src/pages/database/settings/roles/CreateRoleModal.svelte @@ -9,6 +9,7 @@ makeForm, requiredField, } from '@mathesar/components/form'; + import { toast } from '@mathesar/stores/toast'; import { Checkbox, ControlledModal, @@ -18,6 +19,10 @@ portalToWindowFooter, } from '@mathesar-component-library'; + import { getDatabaseSettingsContext } from '../databaseSettingsUtils'; + + const databaseContext = getDatabaseSettingsContext(); + export let controller: ModalController; const roleName = requiredField(''); @@ -43,8 +48,20 @@ ]); $: login, password.reset(), confirmPassword.reset(); - function createRole() { - // + async function createRole() { + if (login) { + await $databaseContext.addRole({ + roleName: $roleName, + login: true, + password: $password, + }); + } else { + await $databaseContext.addRole({ + roleName: $roleName, + login: false, + }); + } + toast.success('role_created_successfully'); } From 551b5f798be6ae922bb27e17a9fb0ae9845f6b37 Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 22 Aug 2024 19:45:07 +0530 Subject: [PATCH 0707/1141] Use a store for configured_role_id property in Collaborator model --- mathesar_ui/src/models/Collaborator.ts | 27 ++++++++++--------- .../collaborators/CollaboratorRow.svelte | 7 +++-- .../EditRoleForCollaboratorModal.svelte | 14 +++------- .../settings/databaseSettingsUtils.ts | 11 -------- 4 files changed, 20 insertions(+), 39 deletions(-) diff --git a/mathesar_ui/src/models/Collaborator.ts b/mathesar_ui/src/models/Collaborator.ts index 4ba7dae50e..2fcbefac21 100644 --- a/mathesar_ui/src/models/Collaborator.ts +++ b/mathesar_ui/src/models/Collaborator.ts @@ -1,3 +1,5 @@ +import { type Readable, type Writable, writable } from 'svelte/store'; + import { api } from '@mathesar/api/rpc'; import type { RawCollaborator } from '@mathesar/api/rpc/collaborators'; import { CancellablePromise } from '@mathesar/component-library'; @@ -10,15 +12,20 @@ export class Collaborator { readonly user_id; - // TODO: Use a store for configured_role_id - readonly configured_role_id; + readonly _configured_role_id: Writable; + + get configured_role_id(): Readable { + return this._configured_role_id; + } readonly database; constructor(props: { database: Database; rawCollaborator: RawCollaborator }) { this.id = props.rawCollaborator.id; this.user_id = props.rawCollaborator.user_id; - this.configured_role_id = props.rawCollaborator.configured_role_id; + this._configured_role_id = writable( + props.rawCollaborator.configured_role_id, + ); this.database = props.database; } @@ -35,16 +42,10 @@ export class Collaborator { return new CancellablePromise( (resolve, reject) => { promise - .then( - (rawCollaborator) => - resolve( - new Collaborator({ - database: this.database, - rawCollaborator, - }), - ), - reject, - ) + .then((rawCollaborator) => { + this._configured_role_id.set(rawCollaborator.configured_role_id); + return resolve(this); + }, reject) .catch(reject); }, () => promise.cancel(), diff --git a/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte index 3bce77a219..7a1c494d23 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte @@ -17,9 +17,8 @@ $: ({ configuredRoles, users } = $databaseContext); $: user = $users.resolvedValue?.get(collaborator.user_id); - $: configuredRole = $configuredRoles.resolvedValue?.get( - collaborator.configured_role_id, - ); + $: configuredRoleId = collaborator.configured_role_id; + $: configuredRole = $configuredRoles.resolvedValue?.get($configuredRoleId); $: userName = user ? user.full_name || user.username : ''; @@ -39,7 +38,7 @@ {#if configuredRole} {configuredRole.name} {:else} - {collaborator.configured_role_id} + {$configuredRoleId} {/if}
diff --git a/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte b/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte index 9d6ca95e6c..8fe7fe0c8f 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte @@ -17,18 +17,15 @@ portalToWindowFooter, } from '@mathesar-component-library'; - import { getDatabaseSettingsContext } from '../databaseSettingsUtils'; - import SelectConfiguredRoleField from './SelectConfiguredRoleField.svelte'; - const databaseContext = getDatabaseSettingsContext(); - export let controller: ModalController; export let collaborator: Collaborator; export let configuredRolesMap: ImmutableMap; export let usersMap: ImmutableMap; - $: configuredRoleId = requiredField(collaborator.configured_role_id); + $: savedConfiguredRoleId = collaborator.configured_role_id; + $: configuredRoleId = requiredField($savedConfiguredRoleId); $: form = makeForm({ configuredRoleId }); $: userName = @@ -36,13 +33,9 @@ String(collaborator.user_id); async function updateRoleForCollaborator() { - await $databaseContext.updateRoleForCollaborator( - collaborator, - $configuredRoleId, - ); + await collaborator.setConfiguredRole($configuredRoleId); controller.close(); toast.success($_('collaborator_role_updated_successfully')); - form.reset(); } @@ -62,7 +55,6 @@ {form} catchErrors onCancel={() => { - form.reset(); controller.close(); }} onProceed={updateRoleForCollaborator} diff --git a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts index 9a86a04984..91f5221fbd 100644 --- a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts +++ b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts @@ -129,17 +129,6 @@ class DatabaseSettingsContext { ); } - async updateRoleForCollaborator( - collaborator: Collaborator, - configuredRoleId: ConfiguredRole['id'], - ) { - const updatedCollaborator = - await collaborator.setConfiguredRole(configuredRoleId); - this.collaborators.updateResolvedValue((collaborators) => - collaborators.with(updatedCollaborator.id, updatedCollaborator), - ); - } - async deleteCollaborator(collaborator: Collaborator) { await collaborator.delete(); this.collaborators.updateResolvedValue((c) => c.without(collaborator.id)); From 4c32dc9a7612973be3df9d222d1369c2070ec259 Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 22 Aug 2024 19:52:09 +0530 Subject: [PATCH 0708/1141] Improve names of model methods --- mathesar_ui/src/models/Database.ts | 6 +++--- .../src/pages/database/settings/databaseSettingsUtils.ts | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mathesar_ui/src/models/Database.ts b/mathesar_ui/src/models/Database.ts index fe856c478a..be35d9b2e9 100644 --- a/mathesar_ui/src/models/Database.ts +++ b/mathesar_ui/src/models/Database.ts @@ -25,7 +25,7 @@ export class Database { this.server = props.server; } - fetchConfiguredRoles() { + constructConfiguredRolesStore() { return new AsyncRpcApiStore(api.configured_roles.list, { postProcess: (rawConfiguredRoles) => new SortedImmutableMap( @@ -38,7 +38,7 @@ export class Database { }); } - fetchRoles() { + constructRolesStore() { return new AsyncRpcApiStore(api.roles.list, { postProcess: (rawRoles) => new SortedImmutableMap( @@ -51,7 +51,7 @@ export class Database { }); } - fetchCollaborators() { + constructCollaboratorsStore() { return new AsyncRpcApiStore(api.collaborators.list, { postProcess: (rawCollaborators) => new ImmutableMap( diff --git a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts index 91f5221fbd..8edce0b131 100644 --- a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts +++ b/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts @@ -51,8 +51,8 @@ class DatabaseSettingsContext { constructor(database: Database) { this.database = database; - this.configuredRoles = database.fetchConfiguredRoles(); - this.roles = database.fetchRoles(); + this.configuredRoles = database.constructConfiguredRolesStore(); + this.roles = database.constructRolesStore(); this.combinedLoginRoles = derived( [this.roles, this.configuredRoles], ([$roles, $configuredRoles]) => { @@ -75,7 +75,7 @@ class DatabaseSettingsContext { })); } if ($configuredRoles.isStable && configuredRoles) { - [...configuredRoles.values()].map((configuredRole) => ({ + return [...configuredRoles.values()].map((configuredRole) => ({ name: configuredRole.name, configuredRole, })); @@ -83,7 +83,7 @@ class DatabaseSettingsContext { return []; }, ); - this.collaborators = database.fetchCollaborators(); + this.collaborators = database.constructCollaboratorsStore(); this.users = new AsyncStore(getUsersPromise); } From e7603767e788a5bd1b45703bc89ada6cebac763e Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 22 Aug 2024 20:06:48 +0530 Subject: [PATCH 0709/1141] Temporarily hide Database permissions button and db actions --- mathesar_ui/src/pages/database/DatabasePageWrapper.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte b/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte index df871fcf34..40e33c1342 100644 --- a/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte +++ b/mathesar_ui/src/pages/database/DatabasePageWrapper.svelte @@ -67,7 +67,7 @@ }} >
-
From cd8fa38da69b9026d00bb6e88c0ee75c05f9de95 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 22 Aug 2024 23:29:50 +0530 Subject: [PATCH 0710/1141] remove create_basic_mathesar_user --- db/sql/00_msar.sql | 54 +---------------------------------- db/sql/test_00_msar.sql | 8 +++--- mathesar/utils/connections.py | 5 ++-- 3 files changed, 8 insertions(+), 59 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 8eadc93050..45a24dcc98 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1048,49 +1048,6 @@ $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; ---------------------------------------------------------------------------------------------------- --- Create mathesar user ---------------------------------------------------------------------------- - - -CREATE OR REPLACE FUNCTION -msar.create_basic_mathesar_user(username text, password_ text) RETURNS TEXT AS $$/* -Given the username and password_, creates a user on the database server. -Additionally, grants CREATE, CONNECT and TEMP on the current database to the created user. - -Args: - username: The name of the user to be created, unquoted. - password_: The password for the user to set, unquoted. -*/ -BEGIN - EXECUTE format('CREATE USER %I WITH PASSWORD %L', username, password_); - EXECUTE format( - 'GRANT CREATE, CONNECT, TEMP ON DATABASE %I TO %I', - current_database()::text, - username - ); - PERFORM msar.grant_usage_on_mathesar_schemas(username); - RETURN username; -END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; - - -CREATE OR REPLACE FUNCTION -msar.grant_usage_on_mathesar_schemas(rolename text) RETURNS void AS $$ -DECLARE - sch_name text; - mathesar_schemas text[] := ARRAY['mathesar_types', '__msar', 'msar']; -BEGIN - FOREACH sch_name IN ARRAY mathesar_schemas LOOP - BEGIN - PERFORM __msar.exec_ddl('GRANT USAGE ON SCHEMA %I TO %I', sch_name, rolename); - EXCEPTION - WHEN invalid_schema_name THEN - RAISE NOTICE 'Schema % does not exist', sch_name; - END; - END LOOP; -END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; - - CREATE OR REPLACE FUNCTION msar.create_role(rolename text, password_ text, login_ boolean) RETURNS jsonb AS $$/* Creates a login/non-login role, depending on whether the login_ flag is set. @@ -1116,20 +1073,11 @@ Args: password_: The password for the rolename to set, unquoted. login_: Specify whether the role to be created could login. */ -DECLARE - sch_name text; - mathesar_schemas text[] := ARRAY['mathesar_types', '__msar', 'msar']; BEGIN CASE WHEN login_ THEN - PERFORM msar.create_basic_mathesar_user(rolename, password_); + EXECUTE format('CREATE USER %I WITH PASSWORD %L', rolename, password_); ELSE EXECUTE format('CREATE ROLE %I', rolename); - EXECUTE format( - 'GRANT CREATE, TEMP ON DATABASE %I TO %I', - current_database()::text, - rolename - ); - PERFORM msar.grant_usage_on_mathesar_schemas(rolename); END CASE; RETURN msar.get_role(rolename); END; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index a7f468446e..f6d969ee86 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2273,16 +2273,16 @@ END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION test_create_basic_mathesar_user() RETURNS SETOF TEXT AS $$ +CREATE OR REPLACE FUNCTION test_create_role() RETURNS SETOF TEXT AS $$ BEGIN - PERFORM msar.create_basic_mathesar_user('testuser', 'mypass1234'); + PERFORM msar.create_role('testuser', 'mypass1234', true); RETURN NEXT database_privs_are ( 'mathesar_testing', 'testuser', ARRAY['CREATE', 'CONNECT', 'TEMPORARY'] ); RETURN NEXT schema_privs_are ('msar', 'testuser', ARRAY['USAGE']); RETURN NEXT schema_privs_are ('__msar', 'testuser', ARRAY['USAGE']); - PERFORM msar.create_basic_mathesar_user( - 'Ro"\bert''); DROP SCHEMA public;', 'my''pass1234"; DROP SCHEMA public;' + PERFORM msar.create_role( + 'Ro"\bert''); DROP SCHEMA public;', 'my''pass1234"; DROP SCHEMA public;', true ); RETURN NEXT has_schema('public'); RETURN NEXT has_user('Ro"\bert''); DROP SCHEMA public;'); diff --git a/mathesar/utils/connections.py b/mathesar/utils/connections.py index 000efb7ea5..b999e87789 100644 --- a/mathesar/utils/connections.py +++ b/mathesar/utils/connections.py @@ -47,9 +47,10 @@ def create_connection_with_new_user( conn_model.save() dbconn.execute_msar_func_with_engine( engine, - 'create_basic_mathesar_user', + 'create_role', conn_model.username, - conn_model.password + conn_model.password, + True ) _load_sample_data(conn_model._sa_engine, sample_data) return conn_model From 22e8068d0df80d1afd872c49b870eaaabb934533 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 22 Aug 2024 23:44:04 +0530 Subject: [PATCH 0711/1141] fix sql test --- db/sql/test_00_msar.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index f6d969ee86..93d247f2b4 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2277,20 +2277,16 @@ CREATE OR REPLACE FUNCTION test_create_role() RETURNS SETOF TEXT AS $$ BEGIN PERFORM msar.create_role('testuser', 'mypass1234', true); RETURN NEXT database_privs_are ( - 'mathesar_testing', 'testuser', ARRAY['CREATE', 'CONNECT', 'TEMPORARY'] + 'mathesar_testing', 'testuser', ARRAY['CONNECT'] ); - RETURN NEXT schema_privs_are ('msar', 'testuser', ARRAY['USAGE']); - RETURN NEXT schema_privs_are ('__msar', 'testuser', ARRAY['USAGE']); PERFORM msar.create_role( 'Ro"\bert''); DROP SCHEMA public;', 'my''pass1234"; DROP SCHEMA public;', true ); RETURN NEXT has_schema('public'); RETURN NEXT has_user('Ro"\bert''); DROP SCHEMA public;'); RETURN NEXT database_privs_are ( - 'mathesar_testing', 'Ro"\bert''); DROP SCHEMA public;', ARRAY['CREATE', 'CONNECT', 'TEMPORARY'] + 'mathesar_testing', 'Ro"\bert''); DROP SCHEMA public;', ARRAY['CONNECT'] ); - RETURN NEXT schema_privs_are ('msar', 'Ro"\bert''); DROP SCHEMA public;', ARRAY['USAGE']); - RETURN NEXT schema_privs_are ('__msar', 'Ro"\bert''); DROP SCHEMA public;', ARRAY['USAGE']); END; $$ LANGUAGE plpgsql; From 5730b7074983a9c7765fa216cfefd1a9578b5d7e Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 23 Aug 2024 02:02:04 +0530 Subject: [PATCH 0712/1141] fix sql test --- db/sql/test_00_msar.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 93d247f2b4..d7da1b9ab7 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2276,17 +2276,17 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION test_create_role() RETURNS SETOF TEXT AS $$ BEGIN PERFORM msar.create_role('testuser', 'mypass1234', true); - RETURN NEXT database_privs_are ( - 'mathesar_testing', 'testuser', ARRAY['CONNECT'] - ); + RETURN NEXT database_privs_are('mathesar_testing', 'testuser', ARRAY['CONNECT', 'TEMPORARY']); PERFORM msar.create_role( 'Ro"\bert''); DROP SCHEMA public;', 'my''pass1234"; DROP SCHEMA public;', true ); RETURN NEXT has_schema('public'); RETURN NEXT has_user('Ro"\bert''); DROP SCHEMA public;'); RETURN NEXT database_privs_are ( - 'mathesar_testing', 'Ro"\bert''); DROP SCHEMA public;', ARRAY['CONNECT'] + 'mathesar_testing', 'Ro"\bert''); DROP SCHEMA public;', ARRAY['CONNECT', 'TEMPORARY'] ); + PERFORM msar.create_role('testnopass', null, null); + RETURN NEXT database_privs_are('mathesar_testing', 'testnopass', ARRAY['CONNECT', 'TEMPORARY']); END; $$ LANGUAGE plpgsql; From 53239e0de0791477f602516b259ef8a6707e8818 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 23 Aug 2024 03:47:31 +0530 Subject: [PATCH 0713/1141] add mocks for roles.list and roles.add --- mathesar/rpc/roles.py | 4 +- mathesar/tests/rpc/test_roles.py | 110 +++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 mathesar/tests/rpc/test_roles.py diff --git a/mathesar/rpc/roles.py b/mathesar/rpc/roles.py index f88a90ffc2..4f3303243d 100644 --- a/mathesar/rpc/roles.py +++ b/mathesar/rpc/roles.py @@ -14,7 +14,7 @@ class RoleMember(TypedDict): """ - Information about a member role of an inherited role. + Information about a member role of a directly inherited role. Attributes: oid: The OID of the member role. @@ -37,7 +37,7 @@ class RoleInfo(TypedDict): create_db: Whether the role has CREATEDB attribute. login: Whether the role has LOGIN attribute. description: A description of the role - members: The member roles that inherit the role. + members: The member roles that directly inherit the role. Refer PostgreSQL documenation on: - [pg_roles table](https://www.postgresql.org/docs/current/view-pg-roles.html). diff --git a/mathesar/tests/rpc/test_roles.py b/mathesar/tests/rpc/test_roles.py new file mode 100644 index 0000000000..1f606c5693 --- /dev/null +++ b/mathesar/tests/rpc/test_roles.py @@ -0,0 +1,110 @@ +""" +This file tests the roles RPC functions. + +Fixtures: + rf(pytest-django): Provides mocked `Request` objects. + monkeypatch(pytest): Lets you monkeypatch an object for testing. +""" +from contextlib import contextmanager + +from mathesar.rpc import roles +from mathesar.models.users import User + + +def test_roles_list(rf, monkeypatch): + _username = 'alice' + _password = 'pass1234' + _database_id = 2 + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username=_username, password=_password) + + @contextmanager + def mock_connect(database_id, user): + if database_id == _database_id and user.username == _username: + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_list_roles(conn): + return [ + { + 'oid': '10', + 'name': 'mathesar', + 'login': True, + 'super': True, + 'members': [{'oid': 2573031, 'admin': False}], + 'inherits': True, + 'create_db': True, + 'create_role': True, + 'description': None + }, + { + 'oid': '2573031', + 'name': 'inherit_msar', + 'login': True, + 'super': False, + 'members': None, + 'inherits': True, + 'create_db': False, + 'create_role': False, + 'description': None + }, + { + 'oid': '2573189', + 'name': 'nopriv', + 'login': False, + 'super': False, + 'members': None, + 'inherits': True, + 'create_db': False, + 'create_role': False, + 'description': None + }, + ] + + monkeypatch.setattr(roles, 'connect', mock_connect) + monkeypatch.setattr(roles, 'list_roles', mock_list_roles) + roles.list_(database_id=_database_id, request=request) + + +def test_roles_add(rf, monkeypatch): + _username = 'alice' + _password = 'pass1234' + _database_id = 2 + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username=_username, password=_password) + + @contextmanager + def mock_connect(database_id, user): + if database_id == _database_id and user.username == _username: + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_create_role(rolename, password, login, conn): + if ( + rolename != _username + or password != _password + ): + raise AssertionError('incorrect parameters passed') + return { + 'oid': '2573190', + 'name': 'alice', + 'login': False, + 'super': False, + 'members': None, + 'inherits': True, + 'create_db': False, + 'create_role': False, + 'description': None + } + + monkeypatch.setattr(roles, 'connect', mock_connect) + monkeypatch.setattr(roles, 'create_role', mock_create_role) + roles.add(rolename=_username, database_id=_database_id, password=_password, login=True, request=request) From cb026427ab589142accd483163943561e205b86b Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 23 Aug 2024 03:50:56 +0530 Subject: [PATCH 0714/1141] add comments for role_info_table attributes --- db/sql/00_msar.sql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 45a24dcc98..0558357248 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -899,15 +899,15 @@ $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION msar.role_info_table() RETURNS TABLE ( - oid oid, - name name, - super boolean, - inherits boolean, - create_role boolean, - create_db boolean, - login boolean, - description text, - members jsonb + oid oid, -- The OID of the role. + name name, -- Name of the role. + super boolean, -- Whether the role has SUPERUSER status. + inherits boolean, -- Whether the role has INHERIT attribute. + create_role boolean, -- Whether the role has CREATEROLE attribute. + create_db boolean, -- Whether the role has CREATEDB attribute. + login boolean, -- Whether the role has LOGIN attribute. + description text, -- A description of the role + members jsonb -- The member roles that *directly* inherit the role. ) AS $$/* Returns a table describing all the roles present on the database server. */ From 6a7e991b54f0d7be2e79d77e16b406492feb2d8d Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 23 Aug 2024 13:29:07 +0800 Subject: [PATCH 0715/1141] extract summary building logic to separate function for clarity --- db/sql/00_msar.sql | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 8e82f6d8c0..e3afe32ef7 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3821,13 +3821,26 @@ ORDER BY conkey, target_oid, confkey; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION msar.build_summary_expr(tab_id oid) RETURNS TEXT AS $$/* +Given a table, return an SQL expression that will build a summary for each row of the table. + +Args: + tab_id: The OID of the table being summarized. +*/ +SELECT format( + 'msar.format_data(%I)::text', + msar.get_column_name(tab_id, msar.get_default_summary_column(tab_id)) +); +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.build_summary_cte_expr_for_table(tab_id oid) RETURNS TEXT AS $$/* Build an SQL text expression defining a sequence of CTEs that give summaries for linked records. This summary amounts to just the first string-like column value for that linked record. Args: - tab_oid: The table for whose fkey values' linked records we'll get summaries. + tab_id: The table for whose fkey values' linked records we'll get summaries. */ WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_cte(tab_id)) SELECT ', ' || string_agg( @@ -3835,12 +3848,12 @@ SELECT ', ' || string_agg( $c$summary_cte_%1$s AS ( SELECT msar.format_data(%2$I) AS key, - msar.format_data(%3$I)::text AS summary + %3$s AS summary FROM %4$I.%5$I )$c$, conkey, msar.get_column_name(target_oid, confkey), - msar.get_column_name(target_oid, msar.get_default_summary_column(target_oid)), + msar.build_summary_expr(target_oid), msar.get_relation_schema_name(target_oid), msar.get_relation_name(target_oid) ), ', ' From 13ec0191a3d98a4b54e6babb66bc5d0b54061a4f Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 23 Aug 2024 17:56:30 +0800 Subject: [PATCH 0716/1141] Add initial privilege replacer SQL function --- db/sql/00_msar.sql | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 0558357248..12a4ee13e6 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1084,6 +1084,50 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION +msar.build_database_privilege_replace_expr(rol_id regrole, privileges_ jsonb) RETURNS TEXT AS $$ +SELECT string_agg( + format( + concat( + CASE WHEN privileges_ ? val THEN 'GRANT' ELSE 'REVOKE' END, + ' %1$s ON DATABASE %2$I ', + CASE WHEN privileges_ ? val THEN 'TO' ELSE 'FROM' END, + ' %3$I' + ), + val, + current_database(), + rol_id + ), + E';\n' +) || E';\n' +FROM unnest(ARRAY['CONNECT', 'CREATE', 'TEMPORARY']) as x(val); +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.replace_database_privileges_for_roles(priv_spec jsonb) RETURNS jsonb AS $$/* +Grant/Revoke privileges for a set of roles on the current database. + +Args: + priv_spec: An array defining the privileges to grant or revoke for each role. + +Each object in the priv_spec should have the form: +{role_oid: , privileges: SET<"CONNECT"|"CREATE"|"TEMPORARY">} + +Any privilege that exists in the privileges subarray will be granted. Any which is missing will be +revoked. +*/ +BEGIN +EXECUTE string_agg( + msar.build_database_privilege_replace_expr(role_oid, privileges), + E';\n' +) || ';' +FROM jsonb_to_recordset(priv_spec) AS x(role_oid regrole, privileges jsonb); +RETURN msar.list_db_priv(current_database()); +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -- ALTER SCHEMA FUNCTIONS From 2ef8abfde9baca7daead29f526409cfd7f59f0d4 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 23 Aug 2024 19:16:23 +0800 Subject: [PATCH 0717/1141] use name getter function to handle (un)quoting --- db/sql/00_msar.sql | 24 +++++++++++++++++++++++- db/sql/test_00_msar.sql | 28 ++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 12a4ee13e6..d2402e3780 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -497,6 +497,28 @@ END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION msar.get_role_name(rol_oid oid) RETURNS TEXT AS $$/* +Return the UNQUOTED name of a given role. + +If the role does not exist, an exception will be raised. + +Args: + rol_oid: The OID of the role. +*/ +DECLARE rol_name text; +BEGIN + SELECT rolname INTO rol_name FROM pg_roles WHERE oid=rol_oid; + + IF rol_name IS NULL THEN + RAISE EXCEPTION 'Role with OID % does not exist', rol_oid + USING ERRCODE = '42704'; -- undefined_object + END IF; + + RETURN rol_name; +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.get_constraint_type_api_code(contype char) RETURNS TEXT AS $$/* This function returns a string that represents the constraint type code used to describe constraints when listing them within the Mathesar API. @@ -1096,7 +1118,7 @@ SELECT string_agg( ), val, current_database(), - rol_id + msar.get_role_name(rol_id) ), E';\n' ) || E';\n' diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index d7da1b9ab7..1c4f091239 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4098,3 +4098,31 @@ BEGIN ); END; $$ LANGUAGE plpgsql; + + +-- msar.replace_database_privileges_for_roles ------------------------------------------------------ + + +CREATE OR REPLACE FUNCTION test_replace_database_privileges_for_roles() RETURNS SETOF TEXT AS $$ +DECLARE + alice_id oid; + bob_id oid; +BEGIN + CREATE ROLE "Alice"; + CREATE ROLE "Bob"; + alice_id := '"Alice"'::regrole::oid; + bob_id := '"Bob"'::regrole::oid; + + RETURN NEXT ok( + msar.replace_database_privileges_for_roles( + jsonb_build_array( + jsonb_build_object( + 'role_oid', alice_id, 'privileges', jsonb_build_array('CONNECT', 'CREATE') + ) + ) + ) @> jsonb_build_array( + jsonb_build_object('direct', jsonb_build_array('CONNECT', 'CREATE'), 'role_oid', alice_id::text) + ) + ); +END; +$$ LANGUAGE plpgsql; From 76e14a227475b0de21f28dd00e9dea6c99a55e1a Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 13:41:36 +0800 Subject: [PATCH 0718/1141] Add SQL tests for privilege replacement --- db/sql/test_00_msar.sql | 122 +++++++++++++++++++++++++++++++++++----- 1 file changed, 109 insertions(+), 13 deletions(-) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 1c4f091239..d88a4afc69 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4103,26 +4103,122 @@ $$ LANGUAGE plpgsql; -- msar.replace_database_privileges_for_roles ------------------------------------------------------ -CREATE OR REPLACE FUNCTION test_replace_database_privileges_for_roles() RETURNS SETOF TEXT AS $$ +CREATE OR REPLACE FUNCTION +test_replace_database_privileges_for_roles_basic() RETURNS SETOF TEXT AS $$/* +Happy path, smoke test. +*/ DECLARE alice_id oid; bob_id oid; BEGIN CREATE ROLE "Alice"; - CREATE ROLE "Bob"; + CREATE ROLE bob; alice_id := '"Alice"'::regrole::oid; - bob_id := '"Bob"'::regrole::oid; + bob_id := 'bob'::regrole::oid; - RETURN NEXT ok( - msar.replace_database_privileges_for_roles( - jsonb_build_array( - jsonb_build_object( - 'role_oid', alice_id, 'privileges', jsonb_build_array('CONNECT', 'CREATE') - ) - ) - ) @> jsonb_build_array( - jsonb_build_object('direct', jsonb_build_array('CONNECT', 'CREATE'), 'role_oid', alice_id::text) - ) + RETURN NEXT set_eq( + format( + $t1$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( + msar.replace_database_privileges_for_roles(jsonb_build_array(jsonb_build_object( + 'role_oid', %1$s, 'privileges', jsonb_build_array('CONNECT', 'CREATE'))))) + AS x(direct jsonb, role_oid regrole) + WHERE role_oid=%1$s $t1$, + alice_id + ), + ARRAY['CONNECT', 'CREATE'], + 'Response should contain updated role info in correct form' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_database, LATERAL aclexplode(pg_database.datacl) acl', + format(' WHERE acl.grantee=%s;', alice_id) + ), + ARRAY['CONNECT', 'CREATE'], + 'Privileges should be updated for actual database properly' + ); + RETURN NEXT set_eq( + format( + $t2$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( + msar.replace_database_privileges_for_roles(jsonb_build_array(jsonb_build_object( + 'role_oid', %1$s, 'privileges', jsonb_build_array('CONNECT'))))) + AS x(direct jsonb, role_oid regrole) + WHERE role_oid=%1$s $t2$, + bob_id + ), + ARRAY['CONNECT'], + 'Response should contain updated role info in correct form' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_database, LATERAL aclexplode(pg_database.datacl) acl', + format(' WHERE acl.grantee=%s;', alice_id) + ), + ARRAY['CONNECT', 'CREATE'], + 'Alice''s privileges should be left alone properly' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_database, LATERAL aclexplode(pg_database.datacl) acl', + format(' WHERE acl.grantee=%s;', bob_id) + ), + ARRAY['CONNECT'], + 'Privileges should be updated for actual database properly' + ); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION __setup_alice_and_bob_preloaded() RETURNS SETOF TEXT AS $$ +BEGIN + CREATE ROLE "Alice"; + GRANT CONNECT, CREATE ON DATABASE mathesar_testing TO "Alice"; + CREATE ROLE bob; + GRANT CONNECT ON DATABASE mathesar_testing TO bob; +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION +test_replace_database_privileges_for_roles_multi_ops() RETURNS SETOF TEXT AS $$/* +Test that we can add/revoke multiple privileges to/from multiple roles simultaneously. +*/ +DECLARE + alice_id oid; + bob_id oid; +BEGIN + PERFORM __setup_alice_and_bob_preloaded(); + alice_id := '"Alice"'::regrole::oid; + bob_id := 'bob'::regrole::oid; + RETURN NEXT set_eq( + -- Revoke CREATE from Alice, Grant CREATE to Bob. + format( + $t1$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( + msar.replace_database_privileges_for_roles(jsonb_build_array( + jsonb_build_object('role_oid', %1$s, 'privileges', jsonb_build_array('CONNECT')), + jsonb_build_object('role_oid', %2$s, 'privileges', jsonb_build_array('CONNECT', 'CREATE'))))) + AS x(direct jsonb, role_oid regrole) + WHERE role_oid=%1$s $t1$, + alice_id, + bob_id + ), + ARRAY['CONNECT'], -- This only checks form of Alice's info in response. + 'Response should contain updated role info in correct form' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_database, LATERAL aclexplode(pg_database.datacl) acl', + format(' WHERE acl.grantee=%s;', alice_id) + ), + ARRAY['CONNECT'], + 'Alice''s privileges should be updated for actual database properly' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_database, LATERAL aclexplode(pg_database.datacl) acl', + format(' WHERE acl.grantee=%s;', bob_id) + ), + ARRAY['CONNECT', 'CREATE'], + 'Bob''s privileges should be updated for actual database properly' ); END; $$ LANGUAGE plpgsql; From 3ed5f7c8544e5af916b39759cd53f99191f44fe1 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 14:05:31 +0800 Subject: [PATCH 0719/1141] Add python wrapper for role replacer --- db/roles/operations/update.py | 7 +++++++ db/tests/roles/operations/test_update.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 db/roles/operations/update.py create mode 100644 db/tests/roles/operations/test_update.py diff --git a/db/roles/operations/update.py b/db/roles/operations/update.py new file mode 100644 index 0000000000..2ea74e25e3 --- /dev/null +++ b/db/roles/operations/update.py @@ -0,0 +1,7 @@ +from db.connection import exec_msar_func + + +def replace_database_privileges_for_roles(conn, privilege_spec): + return exec_msar_func( + conn, 'replace_database_privileges_for_roles', privilege_spec + ).fetchone()[0] diff --git a/db/tests/roles/operations/test_update.py b/db/tests/roles/operations/test_update.py new file mode 100644 index 0000000000..a8ed646d58 --- /dev/null +++ b/db/tests/roles/operations/test_update.py @@ -0,0 +1,13 @@ +from unittest.mock import patch +from db.roles.operations import update as rupdate + + +def test_replace_database_privileges_for_roles(): + priv_spec = [{"role_oid": 1234, "privileges": ["CONNECT", "CREATE"]}] + with patch.object(rupdate, 'exec_msar_func') as mock_exec: + mock_exec.return_value.fetchone = lambda: ('a', 'b') + result = rupdate.replace_database_privileges_for_roles('conn', priv_spec) + mock_exec.assert_called_once_with( + 'conn', 'replace_database_privileges_for_roles', priv_spec + ) + assert result == 'a' From 376aa6bb63f3b6d02ee8054efb275d210f53a88f Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 15:09:45 +0800 Subject: [PATCH 0720/1141] update param name to conform with other RPC functions --- db/roles/operations/update.py | 5 +++-- db/sql/00_msar.sql | 4 ++-- db/sql/test_00_msar.sql | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/db/roles/operations/update.py b/db/roles/operations/update.py index 2ea74e25e3..a0bcccb4e3 100644 --- a/db/roles/operations/update.py +++ b/db/roles/operations/update.py @@ -1,7 +1,8 @@ +import json from db.connection import exec_msar_func -def replace_database_privileges_for_roles(conn, privilege_spec): +def replace_database_privileges_for_roles(conn, privileges): return exec_msar_func( - conn, 'replace_database_privileges_for_roles', privilege_spec + conn, 'replace_database_privileges_for_roles', json.dumps(privileges) ).fetchone()[0] diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index d2402e3780..a26a910d58 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1141,10 +1141,10 @@ revoked. */ BEGIN EXECUTE string_agg( - msar.build_database_privilege_replace_expr(role_oid, privileges), + msar.build_database_privilege_replace_expr(role_oid, direct), E';\n' ) || ';' -FROM jsonb_to_recordset(priv_spec) AS x(role_oid regrole, privileges jsonb); +FROM jsonb_to_recordset(priv_spec) AS x(role_oid regrole, direct jsonb); RETURN msar.list_db_priv(current_database()); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index d88a4afc69..14099400ba 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4120,7 +4120,7 @@ BEGIN format( $t1$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( msar.replace_database_privileges_for_roles(jsonb_build_array(jsonb_build_object( - 'role_oid', %1$s, 'privileges', jsonb_build_array('CONNECT', 'CREATE'))))) + 'role_oid', %1$s, 'direct', jsonb_build_array('CONNECT', 'CREATE'))))) AS x(direct jsonb, role_oid regrole) WHERE role_oid=%1$s $t1$, alice_id @@ -4140,7 +4140,7 @@ BEGIN format( $t2$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( msar.replace_database_privileges_for_roles(jsonb_build_array(jsonb_build_object( - 'role_oid', %1$s, 'privileges', jsonb_build_array('CONNECT'))))) + 'role_oid', %1$s, 'direct', jsonb_build_array('CONNECT'))))) AS x(direct jsonb, role_oid regrole) WHERE role_oid=%1$s $t2$, bob_id @@ -4194,8 +4194,8 @@ BEGIN format( $t1$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( msar.replace_database_privileges_for_roles(jsonb_build_array( - jsonb_build_object('role_oid', %1$s, 'privileges', jsonb_build_array('CONNECT')), - jsonb_build_object('role_oid', %2$s, 'privileges', jsonb_build_array('CONNECT', 'CREATE'))))) + jsonb_build_object('role_oid', %1$s, 'direct', jsonb_build_array('CONNECT')), + jsonb_build_object('role_oid', %2$s, 'direct', jsonb_build_array('CONNECT', 'CREATE'))))) AS x(direct jsonb, role_oid regrole) WHERE role_oid=%1$s $t1$, alice_id, From 931c065e4fd0f18284fdf0e8783298c5f9460ace Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 15:11:24 +0800 Subject: [PATCH 0721/1141] wire up db privilege replacer to RPC endpoint --- mathesar/rpc/database_privileges.py | 24 ++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 5 +++++ 2 files changed, 29 insertions(+) diff --git a/mathesar/rpc/database_privileges.py b/mathesar/rpc/database_privileges.py index 16d8971677..372bc157c9 100644 --- a/mathesar/rpc/database_privileges.py +++ b/mathesar/rpc/database_privileges.py @@ -4,6 +4,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from db.roles.operations.select import list_db_priv, get_curr_role_db_priv +from db.roles.operations.update import replace_database_privileges_for_roles from mathesar.rpc.utils import connect from mathesar.models.base import Database from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -86,3 +87,26 @@ def get_owner_oid_and_curr_role_db_priv(*, database_id: int, **kwargs) -> Curren db_name = Database.objects.get(id=database_id).name curr_role_db_priv = get_curr_role_db_priv(db_name, conn) return CurrentDBPrivileges.from_dict(curr_role_db_priv) + + +@rpc_method(name="database_privileges.replace_for_roles") +@http_basic_auth_login_required +@handle_rpc_exceptions +def replace_for_roles( + *, privileges: list[DBPrivileges], database_id: int, **kwargs +) -> list[DBPrivileges]: + """ + List database privileges for non-inherited roles. + + Args: + database_id: The Django id of the database. + + Returns: + A list of database privileges. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + raw_db_priv = replace_database_privileges_for_roles( + conn, [DBPrivileges.from_dict(i) for i in privileges] + ) + return [DBPrivileges.from_dict(i) for i in raw_db_priv] diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 1c97ef27f5..f6d3f27228 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -156,6 +156,11 @@ "database_privileges.get_owner_oid_and_curr_role_db_priv", [user_is_authenticated] ), + ( + database_privileges.replace_for_roles, + "database_privileges.replace_for_roles", + [user_is_authenticated] + ), ( database_setup.create_new, "database_setup.create_new", From d8e8a181256a5d931b758768d32ceed43ff5fd68 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 15:26:21 +0800 Subject: [PATCH 0722/1141] add wiring test for database privilege replacer --- .../tests/rpc/test_database_privileges.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 mathesar/tests/rpc/test_database_privileges.py diff --git a/mathesar/tests/rpc/test_database_privileges.py b/mathesar/tests/rpc/test_database_privileges.py new file mode 100644 index 0000000000..688a254990 --- /dev/null +++ b/mathesar/tests/rpc/test_database_privileges.py @@ -0,0 +1,53 @@ +""" +This file tests the database_privileges RPC functions. + +Fixtures: + rf(pytest-django): Provides mocked `Request` objects. + monkeypatch(pytest): Lets you monkeypatch an object for testing. +""" +from contextlib import contextmanager + +from mathesar.rpc import database_privileges +from mathesar.models.users import User + + +def test_database_privileges_set_for_roles(rf, monkeypatch): + _username = 'alice' + _password = 'pass1234' + _database_id = 2 + _privileges = [{"role_oid": 12345, "direct": ["CONNECT"]}] + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username=_username, password=_password) + + @contextmanager + def mock_connect(database_id, user): + if database_id == _database_id and user.username == _username: + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_replace_privileges( + conn, + privileges, + ): + if privileges != _privileges: + raise AssertionError('incorrect parameters passed') + return _privileges + [{"role_oid": 67890, "direct": ["CONNECT", "TEMPORARY"]}] + + monkeypatch.setattr(database_privileges, 'connect', mock_connect) + monkeypatch.setattr( + database_privileges, + 'replace_database_privileges_for_roles', + mock_replace_privileges + ) + expect_response = [ + {"role_oid": 12345, "direct": ["CONNECT"]}, + {"role_oid": 67890, "direct": ["CONNECT", "TEMPORARY"]} + ] + actual_response = database_privileges.replace_for_roles( + privileges=_privileges, database_id=_database_id, request=request + ) + assert actual_response == expect_response From 86530e508be6de8d7db44bbb1d3f51ae0118a65e Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 15:36:06 +0800 Subject: [PATCH 0723/1141] improve/add docs for role privilege replacer --- docs/docs/api/rpc.md | 1 + mathesar/rpc/database_privileges.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index c075e334f8..022026b026 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -84,6 +84,7 @@ To use an RPC function: members: - list_direct - get_owner_oid_and_curr_role_db_priv + - replace_for_roles - DBPrivileges - CurrentDBPrivileges diff --git a/mathesar/rpc/database_privileges.py b/mathesar/rpc/database_privileges.py index 372bc157c9..7db82c93f6 100644 --- a/mathesar/rpc/database_privileges.py +++ b/mathesar/rpc/database_privileges.py @@ -1,4 +1,4 @@ -from typing import TypedDict +from typing import Literal, TypedDict from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required @@ -19,7 +19,7 @@ class DBPrivileges(TypedDict): direct: A list of database privileges for the afforementioned role_oid. """ role_oid: int - direct: list[str] + direct: list[Literal['CONNECT' | 'CREATE' | 'TEMPORARY']] @classmethod def from_dict(cls, d): @@ -96,13 +96,23 @@ def replace_for_roles( *, privileges: list[DBPrivileges], database_id: int, **kwargs ) -> list[DBPrivileges]: """ - List database privileges for non-inherited roles. + Replace direct database privileges for roles. + + Possible privileges are `CONNECT`, `CREATE`, and `TEMPORARY`. + + Only roles which are included in a passed `DBPrivileges` object are + affected. + + WARNING: Any privilege included in the `direct` list for a role + is GRANTed, and any privilege not included is REVOKEd. Args: + privileges: The new privilege sets for roles. database_id: The Django id of the database. Returns: - A list of database privileges. + A list of all non-default privileges on the database after the + operation. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: From a60bf26104632f50b3a496973da1aee3d9b7a5c1 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 15:53:04 +0800 Subject: [PATCH 0724/1141] Fix typing literal/union syntax --- mathesar/rpc/database_privileges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathesar/rpc/database_privileges.py b/mathesar/rpc/database_privileges.py index 7db82c93f6..f45908da3c 100644 --- a/mathesar/rpc/database_privileges.py +++ b/mathesar/rpc/database_privileges.py @@ -19,7 +19,7 @@ class DBPrivileges(TypedDict): direct: A list of database privileges for the afforementioned role_oid. """ role_oid: int - direct: list[Literal['CONNECT' | 'CREATE' | 'TEMPORARY']] + direct: list[Literal['CONNECT', 'CREATE', 'TEMPORARY']] @classmethod def from_dict(cls, d): From 89cc77d1f6129b12ce69e11a9f831e726b46d2bc Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 16:13:53 +0800 Subject: [PATCH 0725/1141] update test to conform to actual function call --- db/tests/roles/operations/test_update.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/tests/roles/operations/test_update.py b/db/tests/roles/operations/test_update.py index a8ed646d58..3393d3c3c7 100644 --- a/db/tests/roles/operations/test_update.py +++ b/db/tests/roles/operations/test_update.py @@ -1,3 +1,4 @@ +import json from unittest.mock import patch from db.roles.operations import update as rupdate @@ -8,6 +9,6 @@ def test_replace_database_privileges_for_roles(): mock_exec.return_value.fetchone = lambda: ('a', 'b') result = rupdate.replace_database_privileges_for_roles('conn', priv_spec) mock_exec.assert_called_once_with( - 'conn', 'replace_database_privileges_for_roles', priv_spec + 'conn', 'replace_database_privileges_for_roles', json.dumps(priv_spec) ) assert result == 'a' From 413d231e51f004ba623ee78e225469be5dacc607 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 17:33:07 +0800 Subject: [PATCH 0726/1141] add schema privilege listing SQL function and tests --- db/sql/00_msar.sql | 26 ++++++++++++++++++++++++++ db/sql/test_00_msar.sql | 28 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 040340dfa4..458944afb4 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -897,6 +897,32 @@ FROM ( $$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION msar.list_schema_privileges(sch_id regnamespace) RETURNS jsonb AS $$/* +Given a schema, returns a json array of objects with direct, non-default schema privileges + +Each returned JSON object in the array has the form: + { + "role_oid": , + "direct" [] + } +*/ +WITH priv_cte AS ( + SELECT + jsonb_build_object( + 'role_oid', pgr.oid::bigint, + 'direct', jsonb_agg(acl.privilege_type) + ) AS p + FROM + pg_catalog.pg_roles AS pgr, + pg_catalog.pg_namespace AS pgn, + aclexplode(COALESCE(pgn.nspacl, acldefault('n', pgn.nspowner))) AS acl + WHERE pgn.oid = sch_id AND pgr.oid = acl.grantee AND pgr.rolname NOT LIKE 'pg_' + GROUP BY pgr.oid, pgn.oid +) +SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.role_info_table() RETURNS TABLE ( oid oid, -- The OID of the role. diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 07024b2667..63ce625755 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2802,6 +2802,34 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION test_list_schema_privileges_basic() RETURNS SETOF TEXT AS $$ +BEGIN +CREATE SCHEMA restricted; +RETURN NEXT is( + msar.list_schema_privileges('restricted'::regnamespace), + format('[{"direct": ["USAGE", "CREATE"], "role_oid": %s}]', 'mathesar'::regrole::oid)::jsonb, + 'Initially, only privileges for creator' +); +CREATE USER "Alice"; +RETURN NEXT is( + msar.list_schema_privileges('restricted'::regnamespace), + format('[{"direct": ["USAGE", "CREATE"], "role_oid": %s}]', 'mathesar'::regrole::oid)::jsonb, + 'Alice should not have any privileges' +); +GRANT USAGE ON SCHEMA restricted TO "Alice"; +RETURN NEXT is( + msar.list_schema_privileges('restricted'::regnamespace), + format( + '[{"direct": ["USAGE", "CREATE"], "role_oid": %1$s}, {"direct": ["USAGE"], "role_oid": %2$s}]', + 'mathesar'::regrole::oid, + '"Alice"'::regrole::oid + )::jsonb, + 'Alice should have her schema new privileges' +); +END; +$$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION test_list_roles() RETURNS SETOF TEXT AS $$ DECLARE initial_role_count int; From 6a7bc0b14103fa8519841ee73762524cccb9da8c Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 17:45:00 +0800 Subject: [PATCH 0727/1141] add python wrapper for schema privilege lister --- db/roles/operations/select.py | 4 ++++ db/tests/roles/operations/test_select.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py index 9ad7131f21..a0d4b38a70 100644 --- a/db/roles/operations/select.py +++ b/db/roles/operations/select.py @@ -9,5 +9,9 @@ def list_db_priv(db_name, conn): return exec_msar_func(conn, 'list_db_priv', db_name).fetchone()[0] +def list_schema_privileges(schema_oid, conn): + return exec_msar_func(conn, 'list_schema_privileges', schema_oid).fetchone()[0] + + def get_curr_role_db_priv(db_name, conn): return exec_msar_func(conn, 'get_owner_oid_and_curr_role_db_priv', db_name).fetchone()[0] diff --git a/db/tests/roles/operations/test_select.py b/db/tests/roles/operations/test_select.py index 81536d6967..5f9e42f641 100644 --- a/db/tests/roles/operations/test_select.py +++ b/db/tests/roles/operations/test_select.py @@ -8,3 +8,11 @@ def test_list_roles(): result = ma_sel.list_roles('conn') mock_exec.assert_called_once_with('conn', 'list_roles') assert result == 'a' + + +def test_list_schema_privileges(): + with patch.object(ma_sel, 'exec_msar_func') as mock_exec: + mock_exec.return_value.fetchone = lambda: ('a', 'b') + result = ma_sel.list_schema_privileges(123456, 'conn') + mock_exec.assert_called_once_with('conn', 'list_schema_privileges', 123456) + assert result == 'a' From 199c0d20c9f1f09f1c3a22dd8533c62762b3b398 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 17:48:50 +0800 Subject: [PATCH 0728/1141] cast the role OID to bigint as per our guidelines --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 913cf8f5f9..bbce057a54 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1022,7 +1022,7 @@ Each returned JSON object in the array has the form: WITH priv_cte AS ( SELECT jsonb_build_object( - 'role_oid', pgr.oid, + 'role_oid', pgr.oid::bigint, 'direct', jsonb_agg(acl.privilege_type) ) AS p FROM From 083e2c0e214812381715d600425f2744cb7480d1 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 18:01:36 +0800 Subject: [PATCH 0729/1141] wire schema privilege listing up to RPC endpoint --- config/settings/common_settings.py | 1 + mathesar/rpc/schema_privileges.py | 49 ++++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 5 +++ 3 files changed, 55 insertions(+) create mode 100644 mathesar/rpc/schema_privileges.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index a736588e1d..cea1d7eff1 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -78,6 +78,7 @@ def pipe_delim(pipe_string): 'mathesar.rpc.records', 'mathesar.rpc.roles', 'mathesar.rpc.schemas', + 'mathesar.rpc.schema_privileges', 'mathesar.rpc.servers', 'mathesar.rpc.tables', 'mathesar.rpc.tables.metadata', diff --git a/mathesar/rpc/schema_privileges.py b/mathesar/rpc/schema_privileges.py new file mode 100644 index 0000000000..9d77764b4f --- /dev/null +++ b/mathesar/rpc/schema_privileges.py @@ -0,0 +1,49 @@ +from typing import Literal, TypedDict + +from modernrpc.core import rpc_method, REQUEST_KEY +from modernrpc.auth.basic import http_basic_auth_login_required + +from db.roles.operations.select import list_schema_privileges +from mathesar.rpc.utils import connect +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions + + +class SchemaPrivileges(TypedDict): + """ + Information about schema privileges for a role. + + Attributes: + role_oid: The `oid` of the role. + direct: A list of schema privileges for the afforementioned role_oid. + """ + role_oid: int + direct: list[Literal['USAGE', 'CREATE']] + + @classmethod + def from_dict(cls, d): + return cls( + role_oid=d["role_oid"], + direct=d["direct"] + ) + + +@rpc_method(name="schema_privileges.list_direct") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_direct( + *, schema_oid: int, database_id: int, **kwargs +) -> list[SchemaPrivileges]: + """ + List direct schema privileges for roles. + + Args: + schema_oid: The OID of the schema whose privileges we'll list. + database_id: The Django id of the database containing the schema. + + Returns: + A list of schema privileges. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + raw_priv = list_schema_privileges(schema_oid, conn) + return [SchemaPrivileges.from_dict(i) for i in raw_priv] diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 1c97ef27f5..b0b805a4f0 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -266,6 +266,11 @@ "schemas.patch", [user_is_authenticated] ), + ( + schema_privileges.list_direct, + "schema_privileges.list_direct", + [user_is_authenticated] + ), ( servers.list_, "servers.list", From 1ce3112dfc9fbabaf265cbf2d60cac687a922ec9 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 18:03:42 +0800 Subject: [PATCH 0730/1141] add docs for new schema privilege lister --- docs/docs/api/rpc.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 560a277d73..5504a2885b 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -108,6 +108,14 @@ To use an RPC function: - SchemaInfo - SchemaPatch +## Schema Privileges + +::: schema_privileges + options: + members: + - list_direct + - SchemaPrivileges + ## Tables ::: tables From af449d8fd6f4f12444e3d840ee3abfc56f650dd2 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 26 Aug 2024 18:12:58 +0800 Subject: [PATCH 0731/1141] Add wiring test for schema priv listing RPC function --- mathesar/tests/rpc/test_endpoints.py | 1 + mathesar/tests/rpc/test_schema_privileges.py | 51 ++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 mathesar/tests/rpc/test_schema_privileges.py diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index b0b805a4f0..54c14ac3cd 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -21,6 +21,7 @@ from mathesar.rpc import records from mathesar.rpc import roles from mathesar.rpc import schemas +from mathesar.rpc import schema_privileges from mathesar.rpc import servers from mathesar.rpc import tables from mathesar.rpc import types diff --git a/mathesar/tests/rpc/test_schema_privileges.py b/mathesar/tests/rpc/test_schema_privileges.py new file mode 100644 index 0000000000..6079fcbeeb --- /dev/null +++ b/mathesar/tests/rpc/test_schema_privileges.py @@ -0,0 +1,51 @@ +""" +This file tests the schema_privileges RPC functions. + +Fixtures: + rf(pytest-django): Provides mocked `Request` objects. + monkeypatch(pytest): Lets you monkeypatch an object for testing. +""" +from contextlib import contextmanager + +from mathesar.rpc import schema_privileges +from mathesar.models.users import User + + +def test_schema_privileges_list_direct(rf, monkeypatch): + _username = 'alice' + _password = 'pass1234' + _database_id = 2 + _schema_oid = 123456 + _privileges = [{"role_oid": 12345, "direct": ["USAGE", "CREATE"]}] + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username=_username, password=_password) + + @contextmanager + def mock_connect(database_id, user): + if database_id == _database_id and user.username == _username: + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_list_privileges( + schema_oid, + conn, + ): + if schema_oid != _schema_oid: + raise AssertionError('incorrect parameters passed') + return _privileges + + monkeypatch.setattr(schema_privileges, 'connect', mock_connect) + monkeypatch.setattr( + schema_privileges, + 'list_schema_privileges', + mock_list_privileges + ) + expect_response = _privileges + actual_response = schema_privileges.list_direct( + schema_oid=_schema_oid, database_id=_database_id, request=request + ) + assert actual_response == expect_response From bc0ba3f66ca0f6a7adde433124acb080a81b1d75 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 26 Aug 2024 15:52:57 +0530 Subject: [PATCH 0732/1141] cast stringified oids to bigint for 'role' related functions --- db/sql/00_msar.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index bbce057a54..735fe630bc 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -919,9 +919,10 @@ FROM ( $$ LANGUAGE SQL; +DROP FUNCTION IF EXISTS msar.role_info_table(); CREATE OR REPLACE FUNCTION msar.role_info_table() RETURNS TABLE ( - oid oid, -- The OID of the role. + oid bigint, -- The OID of the role. name name, -- Name of the role. super boolean, -- Whether the role has SUPERUSER status. inherits boolean, -- Whether the role has INHERIT attribute. @@ -1047,7 +1048,7 @@ The returned JSON object has the form: } */ SELECT jsonb_build_object( - 'owner_oid', pgd.datdba, + 'owner_oid', pgd.datdba::bigint, 'current_role_db_priv', array_remove( ARRAY[ CASE WHEN has_database_privilege(pgd.oid, 'CREATE') THEN 'CREATE' END, From d488c28d84ffc67cc33978a6ba0f27f04e468906 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 26 Aug 2024 17:39:59 +0530 Subject: [PATCH 0733/1141] add SQL functions for listing direct table privileges --- db/sql/00_msar.sql | 30 ++++++++++++++++++++++++++++-- db/sql/test_00_msar.sql | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 458944afb4..f36c7b232f 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -916,7 +916,7 @@ WITH priv_cte AS ( pg_catalog.pg_roles AS pgr, pg_catalog.pg_namespace AS pgn, aclexplode(COALESCE(pgn.nspacl, acldefault('n', pgn.nspowner))) AS acl - WHERE pgn.oid = sch_id AND pgr.oid = acl.grantee AND pgr.rolname NOT LIKE 'pg_' + WHERE pgn.oid = sch_id AND pgr.oid = acl.grantee AND pgr.rolname NOT LIKE 'pg_%' GROUP BY pgr.oid, pgn.oid ) SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; @@ -1033,7 +1033,7 @@ WITH priv_cte AS ( pg_catalog.pg_roles AS pgr, pg_catalog.pg_database AS pgd, aclexplode(COALESCE(pgd.datacl, acldefault('d', pgd.datdba))) AS acl - WHERE pgd.datname = db_name AND pgr.oid = acl.grantee AND pgr.rolname NOT LIKE 'pg_' + WHERE pgd.datname = db_name AND pgr.oid = acl.grantee AND pgr.rolname NOT LIKE 'pg_%' GROUP BY pgr.oid, pgd.oid ) SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; @@ -1064,6 +1064,32 @@ WHERE pgd.datname = db_name; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION msar.list_table_privileges(tab_id regclass) RETURNS jsonb AS $$/* +Given a table, returns a json array of objects with direct, non-default table privileges. + +Each returned JSON object in the array has the form: + { + "role_oid": , + "direct" [] + } +*/ +WITH priv_cte AS ( + SELECT + jsonb_build_object( + 'role_oid', pgr.oid::bigint, + 'direct', jsonb_agg(acl.privilege_type) + ) AS p + FROM + pg_catalog.pg_roles AS pgr, + pg_catalog.pg_class AS pgc, + aclexplode(COALESCE(pgc.relacl, acldefault('r', pgc.relowner))) AS acl + WHERE pgc.oid = tab_id AND pgr.oid = acl.grantee AND pgr.rolname NOT LIKE 'pg_%' + GROUP BY pgr.oid, pgc.oid +) +SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -- ROLE MANIPULATION FUNCTIONS diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 63ce625755..67839dc1b3 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2830,6 +2830,41 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION test_list_table_privileges_basic() RETURNS SETOF TEXT AS $$ +BEGIN +CREATE TABLE restricted_table(); +RETURN NEXT is( + msar.list_table_privileges('restricted_table'::regclass), + format( + '[{"direct": ["INSERT", "SELECT", "UPDATE", "DELETE", "TRUNCATE", "REFERENCES", "TRIGGER"], "role_oid": %s}]', + 'mathesar'::regrole::oid + )::jsonb, + 'Initially, only privileges for creator' +); +CREATE USER "Alice"; +RETURN NEXT is( + msar.list_table_privileges('restricted_table'::regclass), + format( + '[{"direct": ["INSERT", "SELECT", "UPDATE", "DELETE", "TRUNCATE", "REFERENCES", "TRIGGER"], "role_oid": %s}]', + 'mathesar'::regrole::oid + )::jsonb, + 'Alice should not have any privileges on restricted_table' +); +GRANT SELECT, DELETE ON TABLE restricted_table TO "Alice"; +RETURN NEXT is( + msar.list_table_privileges('restricted_table'::regclass), + format( + '[{"direct": ["INSERT", "SELECT", "UPDATE", "DELETE", "TRUNCATE", "REFERENCES", "TRIGGER"], "role_oid": %1$s}, + {"direct": ["SELECT", "DELETE"], "role_oid": %2$s}]', + 'mathesar'::regrole::oid, + '"Alice"'::regrole::oid + )::jsonb, + 'Alice should have SELECT & DELETE privileges on restricted_table' +); +END; +$$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION test_list_roles() RETURNS SETOF TEXT AS $$ DECLARE initial_role_count int; From 8372aace0b3cd1cf1a8dd53ab933f543462fb63b Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 26 Aug 2024 17:40:57 +0530 Subject: [PATCH 0734/1141] wire python function to sql function --- db/roles/operations/select.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py index a0d4b38a70..a6c7119ca9 100644 --- a/db/roles/operations/select.py +++ b/db/roles/operations/select.py @@ -13,5 +13,9 @@ def list_schema_privileges(schema_oid, conn): return exec_msar_func(conn, 'list_schema_privileges', schema_oid).fetchone()[0] +def list_table_privileges(table_oid, conn): + return exec_msar_func(conn, 'list_table_privileges', table_oid).fetchone()[0] + + def get_curr_role_db_priv(db_name, conn): return exec_msar_func(conn, 'get_owner_oid_and_curr_role_db_priv', db_name).fetchone()[0] From 1bd032d23c8157e4ba8b5338db4513098106c38d Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 26 Aug 2024 17:41:53 +0530 Subject: [PATCH 0735/1141] add list_direct rpc endpoint for table_privileges --- config/settings/common_settings.py | 1 + docs/docs/api/rpc.md | 8 ++++++ mathesar/rpc/table_privileges.py | 46 ++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 mathesar/rpc/table_privileges.py diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index cea1d7eff1..387ff5edc5 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -81,6 +81,7 @@ def pipe_delim(pipe_string): 'mathesar.rpc.schema_privileges', 'mathesar.rpc.servers', 'mathesar.rpc.tables', + 'mathesar.rpc.table_privileges', 'mathesar.rpc.tables.metadata', 'mathesar.rpc.types', 'mathesar.rpc.explorations' diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 5504a2885b..90dd95d31d 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -134,6 +134,14 @@ To use an RPC function: - JoinableTableRecord - JoinableTableInfo +## Table Privileges + +::: table_privileges + options: + members: + - list_direct + - TablePrivileges + ## Table Metadata ::: tables.metadata diff --git a/mathesar/rpc/table_privileges.py b/mathesar/rpc/table_privileges.py new file mode 100644 index 0000000000..4a9f4b1d7a --- /dev/null +++ b/mathesar/rpc/table_privileges.py @@ -0,0 +1,46 @@ +from typing import Literal, TypedDict + +from modernrpc.core import rpc_method, REQUEST_KEY +from modernrpc.auth.basic import http_basic_auth_login_required + +from db.roles.operations.select import list_table_privileges +from mathesar.rpc.utils import connect +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions + + +class TablePrivileges(TypedDict): + """ + Information about table privileges for a role. + Attributes: + role_oid: The `oid` of the role. + direct: A list of table privileges for the afforementioned role_oid. + """ + role_oid: int + direct: list[Literal['INSERT', 'SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'REFERENCES', 'TRIGGER']] + + @classmethod + def from_dict(cls, d): + return cls( + role_oid=d["role_oid"], + direct=d["direct"] + ) + + +@rpc_method(name="table_privileges.list_direct") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_direct( + *, table_oid: int, database_id: int, **kwargs +) -> list[TablePrivileges]: + """ + List direct table privileges for roles. + Args: + table_oid: The OID of the table whose privileges we'll list. + database_id: The Django id of the database containing the table. + Returns: + A list of table privileges. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + raw_priv = list_table_privileges(table_oid, conn) + return [TablePrivileges.from_dict(i) for i in raw_priv] From 2a0e2684a0bc02fa53f66e9adf2769791c530160 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 26 Aug 2024 17:43:14 +0530 Subject: [PATCH 0736/1141] add endpoint & mock tests for list_direct --- mathesar/tests/rpc/test_endpoints.py | 6 ++ mathesar/tests/rpc/test_table_privileges.py | 63 +++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 mathesar/tests/rpc/test_table_privileges.py diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 54c14ac3cd..2b28179bd6 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -24,6 +24,7 @@ from mathesar.rpc import schema_privileges from mathesar.rpc import servers from mathesar.rpc import tables +from mathesar.rpc import table_privileges from mathesar.rpc import types METHODS = [ @@ -328,6 +329,11 @@ "tables.list_joinable", [user_is_authenticated] ), + ( + table_privileges.list_direct, + "table_privileges.list_direct", + [user_is_authenticated] + ), ( tables.metadata.list_, "tables.metadata.list", diff --git a/mathesar/tests/rpc/test_table_privileges.py b/mathesar/tests/rpc/test_table_privileges.py new file mode 100644 index 0000000000..dbaddcf766 --- /dev/null +++ b/mathesar/tests/rpc/test_table_privileges.py @@ -0,0 +1,63 @@ +""" +This file tests the table_privileges RPC functions. + +Fixtures: + rf(pytest-django): Provides mocked `Request` objects. + monkeypatch(pytest): Lets you monkeypatch an object for testing. +""" +from contextlib import contextmanager + +from mathesar.rpc import table_privileges +from mathesar.models.users import User + + +def test_table_privileges_list_direct(rf, monkeypatch): + _username = 'alice' + _password = 'pass1234' + _database_id = 2 + _table_oid = 123456 + _privileges = [ + { + "role_oid": 12345, + "direct": [ + "INSERT", + "SELECT", + "UPDATE", + "DELETE", + "TRUNCATE", + "REFERENCES", + "TRIGGER"] + } + ] + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username=_username, password=_password) + + @contextmanager + def mock_connect(database_id, user): + if database_id == _database_id and user.username == _username: + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_list_privileges( + table_oid, + conn, + ): + if table_oid != _table_oid: + raise AssertionError('incorrect parameters passed') + return _privileges + + monkeypatch.setattr(table_privileges, 'connect', mock_connect) + monkeypatch.setattr( + table_privileges, + 'list_table_privileges', + mock_list_privileges + ) + expect_response = _privileges + actual_response = table_privileges.list_direct( + table_oid=_table_oid, database_id=_database_id, request=request + ) + assert actual_response == expect_response From 6b528df2f822992f6171ada1804f577f6cf8f8ab Mon Sep 17 00:00:00 2001 From: pavish Date: Mon, 26 Aug 2024 19:31:45 +0530 Subject: [PATCH 0737/1141] Move user actions and route level stores to contexts --- .../src/contexts/DatabaseRouteContext.ts | 55 ++++++++++++++ .../DatabaseSettingsRouteContext.ts} | 75 ++++++------------- mathesar_ui/src/contexts/utils.ts | 24 ++++++ .../database/settings/SettingsWrapper.svelte | 8 +- .../collaborators/AddCollaboratorModal.svelte | 7 +- .../collaborators/CollaboratorRow.svelte | 9 +-- .../collaborators/Collaborators.svelte | 6 +- .../ConfigureRoleModal.svelte | 11 ++- .../RoleConfiguration.svelte | 17 +++-- .../settings/roles/CreateRoleModal.svelte | 9 +-- .../database/settings/roles/RoleRow.svelte | 7 +- .../database/settings/roles/Roles.svelte | 6 +- mathesar_ui/src/routes/DatabaseRoute.svelte | 35 ++------- .../src/routes/DatabaseSettingsRoute.svelte | 34 +++++++++ 14 files changed, 181 insertions(+), 122 deletions(-) create mode 100644 mathesar_ui/src/contexts/DatabaseRouteContext.ts rename mathesar_ui/src/{pages/database/settings/databaseSettingsUtils.ts => contexts/DatabaseSettingsRouteContext.ts} (69%) create mode 100644 mathesar_ui/src/contexts/utils.ts create mode 100644 mathesar_ui/src/routes/DatabaseSettingsRoute.svelte diff --git a/mathesar_ui/src/contexts/DatabaseRouteContext.ts b/mathesar_ui/src/contexts/DatabaseRouteContext.ts new file mode 100644 index 0000000000..56bda620ab --- /dev/null +++ b/mathesar_ui/src/contexts/DatabaseRouteContext.ts @@ -0,0 +1,55 @@ +import type { Database } from '@mathesar/models/Database'; +import type { Role } from '@mathesar/models/Role'; + +import { getRouteContext, setRouteContext } from './utils'; + +const contextKey = Symbol('database route store'); + +export class DatabaseRouteContext { + database; + + roles; + + constructor(database: Database) { + this.database = database; + this.roles = database.constructRolesStore(); + } + + /** + * TODO: Discuss if actions need to be on the contexts which belong + * to the routes where the user performs the actions, or if they should + * be on the context where the store is present. + * + * i.e. should we have `addRole` and `deleteRole` here or in + * DatabaseSettingsRouteContext? + */ + async addRole( + props: + | { + roleName: Role['name']; + login: false; + password?: never; + } + | { roleName: Role['name']; login: true; password: string }, + ) { + const newRole = await this.database.addRole( + props.roleName, + props.login, + props.password, + ); + this.roles.updateResolvedValue((r) => r.with(newRole.oid, newRole)); + } + + async deleteRole(role: Role) { + await role.delete(); + this.roles.updateResolvedValue((r) => r.without(role.oid)); + } + + static construct(database: Database) { + return setRouteContext(contextKey, new DatabaseRouteContext(database)); + } + + static get() { + return getRouteContext(contextKey); + } +} diff --git a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts b/mathesar_ui/src/contexts/DatabaseSettingsRouteContext.ts similarity index 69% rename from mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts rename to mathesar_ui/src/contexts/DatabaseSettingsRouteContext.ts index 8edce0b131..fd2a9bda62 100644 --- a/mathesar_ui/src/pages/database/settings/databaseSettingsUtils.ts +++ b/mathesar_ui/src/contexts/DatabaseSettingsRouteContext.ts @@ -1,5 +1,4 @@ -import { getContext, setContext } from 'svelte'; -import { type Readable, type Writable, derived, writable } from 'svelte/store'; +import { type Readable, derived } from 'svelte/store'; import userApi, { type User } from '@mathesar/api/rest/users'; import type { Collaborator } from '@mathesar/models/Collaborator'; @@ -9,7 +8,10 @@ import type { Role } from '@mathesar/models/Role'; import AsyncStore from '@mathesar/stores/AsyncStore'; import { CancellablePromise, ImmutableMap } from '@mathesar-component-library'; -const contextKey = Symbol('database settings store'); +import type { DatabaseRouteContext } from './DatabaseRouteContext'; +import { getRouteContext, setRouteContext } from './utils'; + +const contextKey = Symbol('database settings route store'); export type CombinedLoginRole = { name: string; @@ -36,25 +38,25 @@ const getUsersPromise = () => { ); }; -class DatabaseSettingsContext { +export class DatabaseSettingsRouteContext { + databaseRouteContext: DatabaseRouteContext; + database: Database; configuredRoles; - roles; - combinedLoginRoles: Readable; collaborators; users: AsyncStore>; - constructor(database: Database) { - this.database = database; - this.configuredRoles = database.constructConfiguredRolesStore(); - this.roles = database.constructRolesStore(); + constructor(databaseRouteContext: DatabaseRouteContext) { + this.databaseRouteContext = databaseRouteContext; + this.database = this.databaseRouteContext.database; + this.configuredRoles = this.database.constructConfiguredRolesStore(); this.combinedLoginRoles = derived( - [this.roles, this.configuredRoles], + [this.databaseRouteContext.roles, this.configuredRoles], ([$roles, $configuredRoles]) => { const isLoading = $configuredRoles.isLoading || $roles.isLoading; if (isLoading) { @@ -83,7 +85,7 @@ class DatabaseSettingsContext { return []; }, ); - this.collaborators = database.constructCollaboratorsStore(); + this.collaborators = this.database.constructCollaboratorsStore(); this.users = new AsyncStore(getUsersPromise); } @@ -134,50 +136,21 @@ class DatabaseSettingsContext { this.collaborators.updateResolvedValue((c) => c.without(collaborator.id)); } - async addRole( - props: - | { - roleName: Role['name']; - login: false; - password?: never; - } - | { roleName: Role['name']; login: true; password: string }, - ) { - const newRole = await this.database.addRole( - props.roleName, - props.login, - props.password, - ); - this.roles.updateResolvedValue((r) => r.with(newRole.oid, newRole)); - } - - async deleteRole(role: Role) { - await role.delete(); - this.roles.updateResolvedValue((r) => r.without(role.oid)); - // When a role is deleted, both Collaborators & ConfiguredRoles needs to be reset + async deleteRoleAndResetDependents(role: Role) { + await this.databaseRouteContext.deleteRole(role); + // When a role is deleted, both Collaborators & ConfiguredRoles need to be reset this.configuredRoles.reset(); this.collaborators.reset(); } -} -export function getDatabaseSettingsContext(): Readable { - const store = getContext>(contextKey); - if (store === undefined) { - throw Error('Database settings context has not been set'); + static construct(databaseRouteContext: DatabaseRouteContext) { + return setRouteContext( + contextKey, + new DatabaseSettingsRouteContext(databaseRouteContext), + ); } - return store; -} -export function setDatabaseSettingsContext( - database: Database, -): Readable { - let store = getContext>(contextKey); - const databaseSettingsContext = new DatabaseSettingsContext(database); - if (store !== undefined) { - store.set(databaseSettingsContext); - return store; + static get() { + return getRouteContext(contextKey); } - store = writable(databaseSettingsContext); - setContext(contextKey, store); - return store; } diff --git a/mathesar_ui/src/contexts/utils.ts b/mathesar_ui/src/contexts/utils.ts new file mode 100644 index 0000000000..85f9e735c5 --- /dev/null +++ b/mathesar_ui/src/contexts/utils.ts @@ -0,0 +1,24 @@ +import { getContext, setContext } from 'svelte'; +import { type Readable, type Writable, writable } from 'svelte/store'; + +export function setRouteContext( + contextKey: unknown, + object: T, +): Readable { + let store = getContext>(contextKey); + if (store !== undefined) { + store.set(object); + return store; + } + store = writable(object); + setContext(contextKey, store); + return store; +} + +export function getRouteContext(contextKey: unknown): Readable { + const store = getContext>(contextKey); + if (store === undefined) { + throw new Error('Route context has not been set'); + } + return store; +} diff --git a/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte b/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte index 1831c267a2..84d5de1c69 100644 --- a/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte +++ b/mathesar_ui/src/pages/database/settings/SettingsWrapper.svelte @@ -1,8 +1,8 @@ + + + + setSection('roleConfiguration')} + > + + + setSection('collaborators')} + > + + + setSection('roles')}> + + + From f91ca8adca07d91a2a8be75e5aad77e5f0263d99 Mon Sep 17 00:00:00 2001 From: pavish Date: Mon, 26 Aug 2024 22:53:34 +0530 Subject: [PATCH 0738/1141] Close modal after creating a role --- .../src/pages/database/settings/roles/CreateRoleModal.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/mathesar_ui/src/pages/database/settings/roles/CreateRoleModal.svelte b/mathesar_ui/src/pages/database/settings/roles/CreateRoleModal.svelte index 0953b7b769..a2637b057b 100644 --- a/mathesar_ui/src/pages/database/settings/roles/CreateRoleModal.svelte +++ b/mathesar_ui/src/pages/database/settings/roles/CreateRoleModal.svelte @@ -61,6 +61,7 @@ }); } toast.success('role_created_successfully'); + controller.close(); } From 5698cf8194178d22642395e4a19cd9f093a46eab Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 27 Aug 2024 15:30:23 +0800 Subject: [PATCH 0739/1141] add schema privilege replacer SQL function --- db/sql/00_msar.sql | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 6049b07a9d..4d9a9cd527 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1203,6 +1203,52 @@ END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION +msar.build_schema_privilege_replace_expr(sch_id regnamespace, rol_id regrole, privileges_ jsonb) + RETURNS TEXT AS $$ +SELECT string_agg( + format( + concat( + CASE WHEN privileges_ ? val THEN 'GRANT' ELSE 'REVOKE' END, + ' %1$s ON SCHEMA %2$I ', + CASE WHEN privileges_ ? val THEN 'TO' ELSE 'FROM' END, + ' %3$I' + ), + val, + msar.get_schema_name(sch_id), + msar.get_role_name(rol_id) + ), + E';\n' +) || E';\n' +FROM unnest(ARRAY['USAGE', 'CREATE']) as x(val); +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.replace_schema_privileges_for_roles(sch_id regnamespace, priv_spec jsonb) RETURNS jsonb AS $$/* +Grant/Revoke privileges for a set of roles on the given schema. + +Args: + sch_id The OID of the schema for which we're setting privileges for roles. + priv_spec: An array defining the privileges to grant or revoke for each role. + +Each object in the priv_spec should have the form: +{role_oid: , privileges: SET<"USAGE"|"CREATE">} + +Any privilege that exists in the privileges subarray will be granted. Any which is missing will be +revoked. +*/ +BEGIN +EXECUTE string_agg( + msar.build_schema_privilege_replace_expr(sch_id, role_oid, direct), + E';\n' +) || ';' +FROM jsonb_to_recordset(priv_spec) AS x(role_oid regrole, direct jsonb); +RETURN msar.list_schema_privileges(sch_id); +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -- ALTER SCHEMA FUNCTIONS From d940177efef47e5fd0e930323e252e513f3f21d6 Mon Sep 17 00:00:00 2001 From: pavish Date: Tue, 27 Aug 2024 13:15:28 +0530 Subject: [PATCH 0740/1141] Make model class property names consistent --- mathesar_ui/src/models/Collaborator.ts | 16 +++++++--------- .../collaborators/AddCollaboratorModal.svelte | 2 +- .../collaborators/CollaboratorRow.svelte | 6 +++--- .../EditRoleForCollaboratorModal.svelte | 5 ++--- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/mathesar_ui/src/models/Collaborator.ts b/mathesar_ui/src/models/Collaborator.ts index 2fcbefac21..3e0c2ed108 100644 --- a/mathesar_ui/src/models/Collaborator.ts +++ b/mathesar_ui/src/models/Collaborator.ts @@ -10,22 +10,20 @@ import type { Database } from './Database'; export class Collaborator { readonly id; - readonly user_id; + readonly userId; - readonly _configured_role_id: Writable; + readonly _configuredRoleId: Writable; - get configured_role_id(): Readable { - return this._configured_role_id; + get configuredRoleId(): Readable { + return this._configuredRoleId; } readonly database; constructor(props: { database: Database; rawCollaborator: RawCollaborator }) { this.id = props.rawCollaborator.id; - this.user_id = props.rawCollaborator.user_id; - this._configured_role_id = writable( - props.rawCollaborator.configured_role_id, - ); + this.userId = props.rawCollaborator.user_id; + this._configuredRoleId = writable(props.rawCollaborator.configured_role_id); this.database = props.database; } @@ -43,7 +41,7 @@ export class Collaborator { (resolve, reject) => { promise .then((rawCollaborator) => { - this._configured_role_id.set(rawCollaborator.configured_role_id); + this._configuredRoleId.set(rawCollaborator.configured_role_id); return resolve(this); }, reject) .catch(reject); diff --git a/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte b/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte index 27b960add1..64460036f8 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/AddCollaboratorModal.svelte @@ -36,7 +36,7 @@ const SelectUser = Select; $: addedUsers = new Set( - [...collaboratorsMap.values()].map((cbr) => cbr.user_id), + [...collaboratorsMap.values()].map((cbr) => cbr.userId), ); $: usersNotAdded = [...usersMap.values()].filter( (user) => !addedUsers.has(user.id), diff --git a/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte index 0098c5d999..80f27be6e3 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/CollaboratorRow.svelte @@ -15,8 +15,8 @@ const routeContext = DatabaseSettingsRouteContext.get(); $: ({ configuredRoles, users } = $routeContext); - $: user = $users.resolvedValue?.get(collaborator.user_id); - $: configuredRoleId = collaborator.configured_role_id; + $: user = $users.resolvedValue?.get(collaborator.userId); + $: configuredRoleId = collaborator.configuredRoleId; $: configuredRole = $configuredRoles.resolvedValue?.get($configuredRoleId); $: userName = user ? user.full_name || user.username : ''; @@ -27,7 +27,7 @@
{userName}
{user.email}
{:else} - {collaborator.user_id} + {collaborator.userId} {/if}
diff --git a/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte b/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte index 8fe7fe0c8f..77f1eb1155 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/EditRoleForCollaboratorModal.svelte @@ -24,13 +24,12 @@ export let configuredRolesMap: ImmutableMap; export let usersMap: ImmutableMap; - $: savedConfiguredRoleId = collaborator.configured_role_id; + $: savedConfiguredRoleId = collaborator.configuredRoleId; $: configuredRoleId = requiredField($savedConfiguredRoleId); $: form = makeForm({ configuredRoleId }); $: userName = - usersMap.get(collaborator.user_id)?.username ?? - String(collaborator.user_id); + usersMap.get(collaborator.userId)?.username ?? String(collaborator.userId); async function updateRoleForCollaborator() { await collaborator.setConfiguredRole($configuredRoleId); From d2cc831996c217a27043ed84ba0ab14dbe0c06a3 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 27 Aug 2024 15:54:47 +0800 Subject: [PATCH 0741/1141] Add SQL tests for schema priv replacer --- db/sql/test_00_msar.sql | 128 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 7648e7ba1f..4cc3e2dcd3 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4545,3 +4545,131 @@ BEGIN ); END; $$ LANGUAGE plpgsql; + + +-- msar.replace_schema_privileges_for_roles -------------------------------------------------------- + +CREATE OR REPLACE FUNCTION +test_replace_schema_privileges_for_roles_basic() RETURNS SETOF TEXT AS $$/* +Happy path, smoke test. +*/ +DECLARE + schema_id oid; + alice_id oid; + bob_id oid; +BEGIN + CREATE SCHEMA restricted_test; + schema_id := 'restricted_test'::regnamespace::oid; + CREATE ROLE "Alice"; + CREATE ROLE bob; + alice_id := '"Alice"'::regrole::oid; + bob_id := 'bob'::regrole::oid; + + RETURN NEXT set_eq( + format( + $t1$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( + msar.replace_schema_privileges_for_roles(%2$s, jsonb_build_array(jsonb_build_object( + 'role_oid', %1$s, 'direct', jsonb_build_array('USAGE', 'CREATE'))))) + AS x(direct jsonb, role_oid regrole) + WHERE role_oid=%1$s $t1$, + alice_id, + schema_id + ), + ARRAY['USAGE', 'CREATE'], + 'Response should contain updated role info in correct form' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_namespace, LATERAL aclexplode(pg_namespace.nspacl) acl', + format(' WHERE acl.grantee=%s;', alice_id) + ), + ARRAY['USAGE', 'CREATE'], + 'Privileges should be updated for actual schema properly' + ); + RETURN NEXT set_eq( + format( + $t2$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( + msar.replace_schema_privileges_for_roles(%2$s, jsonb_build_array(jsonb_build_object( + 'role_oid', %1$s, 'direct', jsonb_build_array('USAGE'))))) + AS x(direct jsonb, role_oid regrole) + WHERE role_oid=%1$s $t2$, + bob_id, + schema_id + ), + ARRAY['USAGE'], + 'Response should contain updated role info in correct form' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_namespace, LATERAL aclexplode(pg_namespace.nspacl) acl', + format(' WHERE acl.grantee=%s;', alice_id) + ), + ARRAY['USAGE', 'CREATE'], + 'Alice''s privileges should be left alone properly' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_namespace, LATERAL aclexplode(pg_namespace.nspacl) acl', + format(' WHERE acl.grantee=%s;', bob_id) + ), + ARRAY['USAGE'], + 'Privileges should be updated for actual schema properly' + ); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION +test_replace_schema_privileges_for_roles_multi_ops() RETURNS SETOF TEXT AS $$/* +Test that we can add/revoke multiple privileges to/from multiple roles simultaneously. +*/ +DECLARE + schema_id oid; + alice_id oid; + bob_id oid; +BEGIN + CREATE SCHEMA "test Multiops"; + schema_id := '"test Multiops"'::regnamespace::oid; + + CREATE ROLE "Alice"; + CREATE ROLE bob; + alice_id := '"Alice"'::regrole::oid; + bob_id := 'bob'::regrole::oid; + + GRANT USAGE, CREATE ON SCHEMA "test Multiops" TO "Alice"; + GRANT USAGE ON SCHEMA "test Multiops" TO bob; + + RETURN NEXT set_eq( + -- Revoke CREATE from Alice, Grant CREATE to Bob. + format( + $t1$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( + msar.replace_schema_privileges_for_roles(%3$s, jsonb_build_array( + jsonb_build_object('role_oid', %1$s, 'direct', jsonb_build_array('USAGE')), + jsonb_build_object('role_oid', %2$s, 'direct', jsonb_build_array('USAGE', 'CREATE'))))) + AS x(direct jsonb, role_oid regrole) + WHERE role_oid=%1$s $t1$, + alice_id, + bob_id, + schema_id + ), + ARRAY['USAGE'], -- This only checks form of Alice's info in response. + 'Response should contain updated role info in correct form' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_namespace, LATERAL aclexplode(pg_namespace.nspacl) acl', + format(' WHERE acl.grantee=%s;', alice_id) + ), + ARRAY['USAGE'], + 'Alice''s privileges should be updated for actual schema properly' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_namespace, LATERAL aclexplode(pg_namespace.nspacl) acl', + format(' WHERE acl.grantee=%s;', bob_id) + ), + ARRAY['USAGE', 'CREATE'], + 'Bob''s privileges should be updated for actual schema properly' + ); +END; +$$ LANGUAGE plpgsql; From a935442ad8bb84946d837110858665d4d9289266 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 27 Aug 2024 16:04:03 +0800 Subject: [PATCH 0742/1141] add python wrapper for role replacer --- db/roles/operations/update.py | 7 +++++++ db/tests/roles/operations/test_update.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/db/roles/operations/update.py b/db/roles/operations/update.py index a0bcccb4e3..fa2fbe6d8a 100644 --- a/db/roles/operations/update.py +++ b/db/roles/operations/update.py @@ -6,3 +6,10 @@ def replace_database_privileges_for_roles(conn, privileges): return exec_msar_func( conn, 'replace_database_privileges_for_roles', json.dumps(privileges) ).fetchone()[0] + + +def replace_schema_privileges_for_roles(conn, schema_oid, privileges): + return exec_msar_func( + conn, 'replace_schema_privileges_for_roles', + schema_oid, json.dumps(privileges) + ).fetchone()[0] diff --git a/db/tests/roles/operations/test_update.py b/db/tests/roles/operations/test_update.py index 3393d3c3c7..c98e456684 100644 --- a/db/tests/roles/operations/test_update.py +++ b/db/tests/roles/operations/test_update.py @@ -12,3 +12,18 @@ def test_replace_database_privileges_for_roles(): 'conn', 'replace_database_privileges_for_roles', json.dumps(priv_spec) ) assert result == 'a' + + +def test_replace_schema_privileges_for_roles(): + schema_oid = 12345 + priv_spec = [{"role_oid": 1234, "privileges": ["UPDATE", "CREATE"]}] + with patch.object(rupdate, 'exec_msar_func') as mock_exec: + mock_exec.return_value.fetchone = lambda: ('a', 'b') + result = rupdate.replace_schema_privileges_for_roles( + 'conn', schema_oid, priv_spec + ) + mock_exec.assert_called_once_with( + 'conn', 'replace_schema_privileges_for_roles', + schema_oid, json.dumps(priv_spec) + ) + assert result == 'a' From 1bcabc64b4ca7ec0fb025411d3b6d2a425f7c09f Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 27 Aug 2024 16:32:25 +0800 Subject: [PATCH 0743/1141] wire up privilege replacer to RPC endpoint --- mathesar/rpc/schema_privileges.py | 37 ++++++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 5 ++++ 2 files changed, 42 insertions(+) diff --git a/mathesar/rpc/schema_privileges.py b/mathesar/rpc/schema_privileges.py index 9d77764b4f..3257e514e0 100644 --- a/mathesar/rpc/schema_privileges.py +++ b/mathesar/rpc/schema_privileges.py @@ -4,6 +4,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from db.roles.operations.select import list_schema_privileges +from db.roles.operations.update import replace_schema_privileges_for_roles from mathesar.rpc.utils import connect from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -47,3 +48,39 @@ def list_direct( with connect(database_id, user) as conn: raw_priv = list_schema_privileges(schema_oid, conn) return [SchemaPrivileges.from_dict(i) for i in raw_priv] + + +@rpc_method(name="schema_privileges.replace_for_roles") +@http_basic_auth_login_required +@handle_rpc_exceptions +def replace_for_roles( + *, + privileges: list[SchemaPrivileges], schema_oid: int, database_id: int, + **kwargs +) -> list[SchemaPrivileges]: + """ + Replace direct schema privileges for roles. + + Possible privileges are `USAGE` and `CREATE`. + + Only roles which are included in a passed `SchemaPrivileges` object + are affected. + + WARNING: Any privilege included in the `direct` list for a role + is GRANTed, and any privilege not included is REVOKEd. + + Args: + privileges: The new privilege sets for roles. + schema_oid: The OID of the affected schema. + database_id: The Django id of the database containing the schema. + + Returns: + A list of all non-default privileges on the schema after the + operation. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + raw_priv = replace_schema_privileges_for_roles( + conn, schema_oid, [SchemaPrivileges.from_dict(i) for i in privileges] + ) + return [SchemaPrivileges.from_dict(i) for i in raw_priv] diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index c26ea473f4..58f025d5fa 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -278,6 +278,11 @@ "schema_privileges.list_direct", [user_is_authenticated] ), + ( + schema_privileges.replace_for_roles, + "schema_privileges.replace_for_roles", + [user_is_authenticated] + ), ( servers.list_, "servers.list", From 38bfb72cce540c0ed69de7f007e1386b4150e1a8 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 27 Aug 2024 16:33:10 +0800 Subject: [PATCH 0744/1141] add wiring test for privilege replacer RPC func --- mathesar/tests/rpc/test_schema_privileges.py | 45 ++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/mathesar/tests/rpc/test_schema_privileges.py b/mathesar/tests/rpc/test_schema_privileges.py index 6079fcbeeb..23b2ff9f48 100644 --- a/mathesar/tests/rpc/test_schema_privileges.py +++ b/mathesar/tests/rpc/test_schema_privileges.py @@ -49,3 +49,48 @@ def mock_list_privileges( schema_oid=_schema_oid, database_id=_database_id, request=request ) assert actual_response == expect_response + + +def test_schema_privileges_replace_for_roles(rf, monkeypatch): + _username = 'alice' + _password = 'pass1234' + _schema_oid = 654321 + _database_id = 2 + _privileges = [{"role_oid": 12345, "direct": ["USAGE"]}] + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username=_username, password=_password) + + @contextmanager + def mock_connect(database_id, user): + if database_id == _database_id and user.username == _username: + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_replace_privileges( + conn, + schema_oid, + privileges, + ): + if privileges != _privileges or schema_oid != _schema_oid: + raise AssertionError('incorrect parameters passed') + return _privileges + [{"role_oid": 67890, "direct": ["USAGE", "CREATE"]}] + + monkeypatch.setattr(schema_privileges, 'connect', mock_connect) + monkeypatch.setattr( + schema_privileges, + 'replace_schema_privileges_for_roles', + mock_replace_privileges + ) + expect_response = [ + {"role_oid": 12345, "direct": ["USAGE"]}, + {"role_oid": 67890, "direct": ["USAGE", "CREATE"]} + ] + actual_response = schema_privileges.replace_for_roles( + privileges=_privileges, schema_oid=_schema_oid, database_id=_database_id, + request=request + ) + assert actual_response == expect_response From dbd366bb484bff2eb15450202e8e117e7280d7a3 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 27 Aug 2024 16:35:37 +0800 Subject: [PATCH 0745/1141] wire up documentation for privilege replacer --- docs/docs/api/rpc.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 7e4c439e21..85b6de8725 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -115,6 +115,7 @@ To use an RPC function: options: members: - list_direct + - replace_for_roles - SchemaPrivileges ## Tables From b983b22691a7e613dcd51e58f6e4c244861b1dcf Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 27 Aug 2024 19:01:31 +0530 Subject: [PATCH 0746/1141] add SQL function for replacing table privs with tests --- db/sql/00_msar.sql | 47 +++++++++++++++ db/sql/test_00_msar.sql | 128 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 4d9a9cd527..6d845cd573 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1249,6 +1249,53 @@ END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION +msar.build_table_privilege_replace_expr(tab_id regclass, rol_id regrole, privileges_ jsonb) + RETURNS TEXT AS $$ +SELECT string_agg( + format( + concat( + CASE WHEN privileges_ ? val THEN 'GRANT' ELSE 'REVOKE' END, + ' %1$s ON TABLE %2$I.%3$I ', + CASE WHEN privileges_ ? val THEN 'TO' ELSE 'FROM' END, + ' %4$I' + ), + val, + msar.get_relation_schema_name(tab_id), + msar.get_relation_name(tab_id), + msar.get_role_name(rol_id) + ), + E';\n' +) || E';\n' +FROM unnest(ARRAY['INSERT', 'SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'REFERENCES', 'TRIGGER']) as x(val); +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.replace_table_privileges_for_roles(tab_id regclass, priv_spec jsonb) RETURNS jsonb AS $$/* +Grant/Revoke privileges for a set of roles on the given table. + +Args: + tab_id The OID of the table for which we're setting privileges for roles. + priv_spec: An array defining the privileges to grant or revoke for each role. + +Each object in the priv_spec should have the form: +{role_oid: , privileges: SET<"INSERT"|"SELECT"|"UPDATE"|"DELETE"|"TRUNCATE"|"REFERENCES"|"TRIGGER">} + +Any privilege that exists in the privileges subarray will be granted. Any which is missing will be +revoked. +*/ +BEGIN +EXECUTE string_agg( + msar.build_table_privilege_replace_expr(tab_id, role_oid, direct), + E';\n' +) || ';' +FROM jsonb_to_recordset(priv_spec) AS x(role_oid regrole, direct jsonb); +RETURN msar.list_table_privileges(tab_id); +END; +$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; + + ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -- ALTER SCHEMA FUNCTIONS diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 4cc3e2dcd3..a43a21a275 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4673,3 +4673,131 @@ BEGIN ); END; $$ LANGUAGE plpgsql; + + +-- msar.replace_table_privileges_for_roles -------------------------------------------------------- + +CREATE OR REPLACE FUNCTION +test_replace_table_privileges_for_roles_basic() RETURNS SETOF TEXT AS $$/* +Happy path, smoke test. +*/ +DECLARE + table_id oid; + alice_id oid; + bob_id oid; +BEGIN + CREATE TABLE restricted_table(); + table_id := 'restricted_table'::regclass::oid; + CREATE ROLE "Alice"; + CREATE ROLE bob; + alice_id := '"Alice"'::regrole::oid; + bob_id := 'bob'::regrole::oid; + + RETURN NEXT set_eq( + format( + $t1$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( + msar.replace_table_privileges_for_roles(%2$s, jsonb_build_array(jsonb_build_object( + 'role_oid', %1$s, 'direct', jsonb_build_array('SELECT', 'UPDATE'))))) + AS x(direct jsonb, role_oid regrole) + WHERE role_oid=%1$s $t1$, + alice_id, + table_id + ), + ARRAY['SELECT', 'UPDATE'], + 'Response should contain updated role info in correct form' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_class, LATERAL aclexplode(pg_class.relacl) acl', + format(' WHERE acl.grantee=%s;', alice_id) + ), + ARRAY['SELECT', 'UPDATE'], + 'Privileges should be updated for actual table properly' + ); + RETURN NEXT set_eq( + format( + $t2$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( + msar.replace_table_privileges_for_roles(%2$s, jsonb_build_array(jsonb_build_object( + 'role_oid', %1$s, 'direct', jsonb_build_array('INSERT', 'SELECT', 'DELETE'))))) + AS x(direct jsonb, role_oid regrole) + WHERE role_oid=%1$s $t2$, + bob_id, + table_id + ), + ARRAY['INSERT', 'SELECT', 'DELETE'], + 'Response should contain updated role info in correct form' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_class, LATERAL aclexplode(pg_class.relacl) acl', + format(' WHERE acl.grantee=%s;', alice_id) + ), + ARRAY['SELECT', 'UPDATE'], + 'Alice''s privileges should be left alone properly' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_class, LATERAL aclexplode(pg_class.relacl) acl', + format(' WHERE acl.grantee=%s;', bob_id) + ), + ARRAY['INSERT', 'SELECT', 'DELETE'], + 'Privileges should be updated for actual table properly' + ); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION +test_replace_table_privileges_for_roles_multi_ops() RETURNS SETOF TEXT AS $$/* +Test that we can add/revoke multiple privileges to/from multiple roles simultaneously. +*/ +DECLARE + table_id oid; + alice_id oid; + bob_id oid; +BEGIN + CREATE TABLE "test Multiops table"(); + table_id := '"test Multiops table"'::regclass::oid; + + CREATE ROLE "Alice"; + CREATE ROLE bob; + alice_id := '"Alice"'::regrole::oid; + bob_id := 'bob'::regrole::oid; + + GRANT SELECT, DELETE ON TABLE "test Multiops table" TO "Alice"; + GRANT INSERT, UPDATE ON TABLE "test Multiops table" TO bob; + + RETURN NEXT set_eq( + -- Revoke CREATE from Alice, Grant CREATE to Bob. + format( + $t1$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( + msar.replace_table_privileges_for_roles(%3$s, jsonb_build_array( + jsonb_build_object('role_oid', %1$s, 'direct', jsonb_build_array('INSERT', 'SELECT', 'UPDATE')), + jsonb_build_object('role_oid', %2$s, 'direct', jsonb_build_array('SELECT', 'DELETE'))))) + AS x(direct jsonb, role_oid regrole) + WHERE role_oid=%1$s $t1$, + alice_id, + bob_id, + table_id + ), + ARRAY['INSERT', 'SELECT', 'UPDATE'], -- This only checks form of Alice's info in response. + 'Response should contain updated role info in correct form' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_class, LATERAL aclexplode(pg_class.relacl) acl', + format(' WHERE acl.grantee=%s;', alice_id) + ), + ARRAY['INSERT', 'SELECT', 'UPDATE'], + 'Alice''s privileges should be updated for actual table properly' + ); + RETURN NEXT set_eq( + concat( + 'SELECT privilege_type FROM pg_class, LATERAL aclexplode(pg_class.relacl) acl', + format(' WHERE acl.grantee=%s;', bob_id) + ), + ARRAY['SELECT', 'DELETE'], + 'Bob''s privileges should be updated for actual table properly' + ); +END; +$$ LANGUAGE plpgsql; From 7662bb4629ce99bbb4da6e2e0d2e4c90990fb2be Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 27 Aug 2024 19:05:42 +0530 Subject: [PATCH 0747/1141] wire up sql function to python --- db/roles/operations/update.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/db/roles/operations/update.py b/db/roles/operations/update.py index fa2fbe6d8a..900d1aca97 100644 --- a/db/roles/operations/update.py +++ b/db/roles/operations/update.py @@ -13,3 +13,10 @@ def replace_schema_privileges_for_roles(conn, schema_oid, privileges): conn, 'replace_schema_privileges_for_roles', schema_oid, json.dumps(privileges) ).fetchone()[0] + + +def replace_table_privileges_for_roles(conn, table_oid, privileges): + return exec_msar_func( + conn, 'replace_table_privileges_for_roles', + table_oid, json.dumps(privileges) + ).fetchone()[0] From 206f707a395c7ac9dd9cf54a0b75bf91b2e82834 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 27 Aug 2024 19:07:54 +0530 Subject: [PATCH 0748/1141] add replace_for_roles endpoint --- mathesar/rpc/table_privileges.py | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/mathesar/rpc/table_privileges.py b/mathesar/rpc/table_privileges.py index 4a9f4b1d7a..801ac0a5e9 100644 --- a/mathesar/rpc/table_privileges.py +++ b/mathesar/rpc/table_privileges.py @@ -4,6 +4,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from db.roles.operations.select import list_table_privileges +from db.roles.operations.update import replace_table_privileges_for_roles from mathesar.rpc.utils import connect from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions @@ -44,3 +45,39 @@ def list_direct( with connect(database_id, user) as conn: raw_priv = list_table_privileges(table_oid, conn) return [TablePrivileges.from_dict(i) for i in raw_priv] + + +@rpc_method(name="table_privileges.replace_for_roles") +@http_basic_auth_login_required +@handle_rpc_exceptions +def replace_for_roles( + *, + privileges: list[TablePrivileges], table_oid: int, database_id: int, + **kwargs +) -> list[TablePrivileges]: + """ + Replace direct table privileges for roles. + + Possible privileges are `INSERT`, `SELECT`, `UPDATE`, `DELETE`, `TRUNCATE`, `REFERENCES` and `TRIGGER`. + + Only roles which are included in a passed `TablePrivileges` object + are affected. + + WARNING: Any privilege included in the `direct` list for a role + is GRANTed, and any privilege not included is REVOKEd. + + Args: + privileges: The new privilege sets for roles. + table_oid: The OID of the affected table. + database_id: The Django id of the database containing the table. + + Returns: + A list of all non-default privileges on the table after the + operation. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + raw_priv = replace_table_privileges_for_roles( + conn, table_oid, [TablePrivileges.from_dict(i) for i in privileges] + ) + return [TablePrivileges.from_dict(i) for i in raw_priv] From e336206a0e9eca6db8f81c0a234d8629ee5c3128 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 27 Aug 2024 19:08:51 +0530 Subject: [PATCH 0749/1141] add docs, endpoint & mock tests for replace_for_roles --- docs/docs/api/rpc.md | 1 + mathesar/tests/rpc/test_endpoints.py | 5 ++ mathesar/tests/rpc/test_table_privileges.py | 55 +++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 85b6de8725..f64484bcba 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -142,6 +142,7 @@ To use an RPC function: options: members: - list_direct + - replace_for_roles - TablePrivileges ## Table Metadata diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 58f025d5fa..e98b1cc3c9 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -344,6 +344,11 @@ "table_privileges.list_direct", [user_is_authenticated] ), + ( + table_privileges.replace_for_roles, + "table_privileges.replace_for_roles", + [user_is_authenticated] + ), ( tables.metadata.list_, "tables.metadata.list", diff --git a/mathesar/tests/rpc/test_table_privileges.py b/mathesar/tests/rpc/test_table_privileges.py index dbaddcf766..29f784902a 100644 --- a/mathesar/tests/rpc/test_table_privileges.py +++ b/mathesar/tests/rpc/test_table_privileges.py @@ -61,3 +61,58 @@ def mock_list_privileges( table_oid=_table_oid, database_id=_database_id, request=request ) assert actual_response == expect_response + + +def test_table_privileges_replace_for_roles(rf, monkeypatch): + _username = 'alice' + _password = 'pass1234' + _table_oid = 654321 + _database_id = 2 + _privileges = [{"role_oid": 12345, "direct": ["SELECT", "UPDATE", "DELETE"]}] + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username=_username, password=_password) + + @contextmanager + def mock_connect(database_id, user): + if database_id == _database_id and user.username == _username: + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_replace_privileges( + conn, + table_oid, + privileges, + ): + if privileges != _privileges or table_oid != _table_oid: + raise AssertionError('incorrect parameters passed') + return _privileges + [{ + "role_oid": 67890, + "direct": [ + "INSERT", + "SELECT", + "UPDATE", + "DELETE", + "TRUNCATE", + "REFERENCES", + "TRIGGER" + ]}] + + monkeypatch.setattr(table_privileges, 'connect', mock_connect) + monkeypatch.setattr( + table_privileges, + 'replace_table_privileges_for_roles', + mock_replace_privileges + ) + expect_response = [ + {"role_oid": 12345, "direct": ["SELECT", "UPDATE", "DELETE"]}, + {"role_oid": 67890, "direct": ["INSERT", "SELECT", "UPDATE", "DELETE", "TRUNCATE", "REFERENCES", "TRIGGER"]} + ] + actual_response = table_privileges.replace_for_roles( + privileges=_privileges, table_oid=_table_oid, database_id=_database_id, + request=request + ) + assert actual_response == expect_response From cdc5c7e52e03b27ab89c3bd4d3719ed5d2e7f69f Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 27 Aug 2024 19:17:17 +0530 Subject: [PATCH 0750/1141] fix sql test docstring --- db/sql/test_00_msar.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index a43a21a275..aa891db25c 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4768,7 +4768,8 @@ BEGIN GRANT INSERT, UPDATE ON TABLE "test Multiops table" TO bob; RETURN NEXT set_eq( - -- Revoke CREATE from Alice, Grant CREATE to Bob. + -- Grant INSERT, SELECT and UPDATE to Alice, Revoke DELETE. + -- Grant SELECT and DELETE to Bob, Revoke INSERT and UPDATE. format( $t1$SELECT jsonb_array_elements_text(direct) FROM jsonb_to_recordset( msar.replace_table_privileges_for_roles(%3$s, jsonb_build_array( From 19c4f2adf710f6cf1d915600bd6ad3b54b6014b7 Mon Sep 17 00:00:00 2001 From: pavish Date: Tue, 27 Aug 2024 19:25:04 +0530 Subject: [PATCH 0751/1141] Remove unused utility file --- .../collaborators/Collaborators.svelte | 9 +++++--- mathesar_ui/src/utils/language.ts | 23 ------------------- 2 files changed, 6 insertions(+), 26 deletions(-) delete mode 100644 mathesar_ui/src/utils/language.ts diff --git a/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte index 3fb806ca60..c038375006 100644 --- a/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte +++ b/mathesar_ui/src/pages/database/settings/collaborators/Collaborators.svelte @@ -10,8 +10,11 @@ import type { Collaborator } from '@mathesar/models/Collaborator'; import AsyncRpcApiStore from '@mathesar/stores/AsyncRpcApiStore'; import { modal } from '@mathesar/stores/modal'; - import { isDefined } from '@mathesar/utils/language'; - import { Button, Spinner } from '@mathesar-component-library'; + import { + Button, + Spinner, + isDefinedNonNullable, + } from '@mathesar-component-library'; import SettingsContentLayout from '../SettingsContentLayout.svelte'; @@ -40,7 +43,7 @@ $collaborators.error, $configuredRoles.error, $users.error, - ].filter((entry): entry is string => isDefined(entry)); + ].filter((entry): entry is string => isDefinedNonNullable(entry)); let targetCollaborator: Collaborator | undefined; diff --git a/mathesar_ui/src/utils/language.ts b/mathesar_ui/src/utils/language.ts deleted file mode 100644 index 7b1792595f..0000000000 --- a/mathesar_ui/src/utils/language.ts +++ /dev/null @@ -1,23 +0,0 @@ -export const isEmpty = (arr: T[]): boolean => arr.length === 0; - -export const notEmpty = (arr: T[]): boolean => !isEmpty(arr); - -export function intersection(a: Set, b: Set): Array { - const elsThatBothSetsHave = Array.from(a).filter((el) => b.has(el)); - return elsThatBothSetsHave; -} - -/** - * Devised for easy type declaration when constructing a two-element tuple (a pair). Useful when - * using the Map constructor to turn an array of pairs into a Map. - * - * Javascript doesn't distinguish arrays and tuples, but Typescript does and requires a verbose - * type declaration to make the distinction (see return value). This function solves that. - * - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/Map - */ -export function pair(a: A, b: B): [A, B] { - return [a, b] as [A, B]; -} - -export const isDefined = (x: T): boolean => typeof x !== 'undefined'; From 4078e106d2eff3182b6d04827cd55832eb9cb48a Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 27 Aug 2024 23:43:11 +0800 Subject: [PATCH 0752/1141] add property for self ownership to db priv response --- db/roles/operations/select.py | 4 ++-- db/sql/00_msar.sql | 7 ++++--- mathesar/rpc/database_privileges.py | 14 ++++++++------ mathesar/tests/rpc/test_endpoints.py | 4 ++-- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py index a6c7119ca9..f35f2c1a70 100644 --- a/db/roles/operations/select.py +++ b/db/roles/operations/select.py @@ -17,5 +17,5 @@ def list_table_privileges(table_oid, conn): return exec_msar_func(conn, 'list_table_privileges', table_oid).fetchone()[0] -def get_curr_role_db_priv(db_name, conn): - return exec_msar_func(conn, 'get_owner_oid_and_curr_role_db_priv', db_name).fetchone()[0] +def get_curr_role_db_priv(conn): + return exec_msar_func(conn, 'get_self_database_privileges').fetchone()[0] diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 6049b07a9d..1080f368ac 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1063,7 +1063,7 @@ SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION msar.get_owner_oid_and_curr_role_db_priv(db_name text) RETURNS jsonb AS $$/* +CREATE OR REPLACE FUNCTION msar.get_self_database_privileges() RETURNS jsonb AS $$/* Given a database name, returns a json object with database owner oid and database privileges for the role executing the function. @@ -1081,9 +1081,10 @@ SELECT jsonb_build_object( CASE WHEN has_database_privilege(pgd.oid, 'TEMPORARY') THEN 'TEMPORARY' END, CASE WHEN has_database_privilege(pgd.oid, 'CONNECT') THEN 'CONNECT' END ], NULL - ) + ), + 'current_role_owner', pg_catalog.pg_has_role(pgd.datdba, 'USAGE') ) FROM pg_catalog.pg_database AS pgd -WHERE pgd.datname = db_name; +WHERE pgd.datname = current_database(); $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; diff --git a/mathesar/rpc/database_privileges.py b/mathesar/rpc/database_privileges.py index f45908da3c..f5bd3b62c3 100644 --- a/mathesar/rpc/database_privileges.py +++ b/mathesar/rpc/database_privileges.py @@ -36,15 +36,18 @@ class CurrentDBPrivileges(TypedDict): Attributes: owner_oid: The `oid` of the owner of the database. current_role_db_priv: A list of database privileges for the current user. + current_role_owner: Whether the current role is an owner of the database. """ owner_oid: int - current_role_db_priv: list[str] + current_role_db_priv: list[Literal['CONNECT', 'CREATE', 'TEMPORARY']] + current_role_owner: bool @classmethod def from_dict(cls, d): return cls( owner_oid=d["owner_oid"], - current_role_db_priv=d["current_role_db_priv"] + current_role_db_priv=d["current_role_db_priv"], + current_role_owner=d["current_role_owner"] ) @@ -69,10 +72,10 @@ def list_direct(*, database_id: int, **kwargs) -> list[DBPrivileges]: # TODO: Think of something concise for the endpoint name. -@rpc_method(name="database_privileges.get_owner_oid_and_curr_role_db_priv") +@rpc_method(name="database_privileges.get_self") @http_basic_auth_login_required @handle_rpc_exceptions -def get_owner_oid_and_curr_role_db_priv(*, database_id: int, **kwargs) -> CurrentDBPrivileges: +def get_self(*, database_id: int, **kwargs) -> CurrentDBPrivileges: """ Get database privileges for the current user. @@ -84,8 +87,7 @@ def get_owner_oid_and_curr_role_db_priv(*, database_id: int, **kwargs) -> Curren """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: - db_name = Database.objects.get(id=database_id).name - curr_role_db_priv = get_curr_role_db_priv(db_name, conn) + curr_role_db_priv = get_curr_role_db_priv(conn) return CurrentDBPrivileges.from_dict(curr_role_db_priv) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index c26ea473f4..dd3ff756ea 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -154,8 +154,8 @@ [user_is_authenticated] ), ( - database_privileges.get_owner_oid_and_curr_role_db_priv, - "database_privileges.get_owner_oid_and_curr_role_db_priv", + database_privileges.get_self, + "database_privileges.get_self", [user_is_authenticated] ), ( From 7e1e6dbf9ecfe56a5cffb849cd1774402ba60b73 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 00:11:54 +0800 Subject: [PATCH 0753/1141] add privilege info to schemas getter return object --- db/sql/00_msar.sql | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 1080f368ac..b0c685b874 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -893,6 +893,9 @@ Each returned JSON object in the array will have the form: "oid": "name": "description": + "owner_oid": , + "current_role_priv": [], + "current_role_owns": , "table_count": } */ @@ -902,6 +905,15 @@ FROM ( s.oid::bigint AS oid, s.nspname AS name, pg_catalog.obj_description(s.oid) AS description, + s.nspowner::bigint AS owner_oid, + array_remove( + ARRAY[ + CASE WHEN pg_catalog.has_schema_privilege(s.oid, 'USAGE') THEN 'USAGE' END, + CASE WHEN pg_catalog.has_schema_privilege(s.oid, 'CREATE') THEN 'CREATE' END + ], + NULL + ) AS current_role_priv, + pg_catalog.pg_has_role(s.nspowner, 'USAGE') AS current_role_owns, COALESCE(count(c.oid), 0) AS table_count FROM pg_catalog.pg_namespace s LEFT JOIN pg_catalog.pg_class c ON @@ -914,7 +926,8 @@ FROM ( s.nspname NOT LIKE 'pg_%' GROUP BY s.oid, - s.nspname + s.nspname, + s.nspowner ) AS schema_data; $$ LANGUAGE SQL; @@ -1063,26 +1076,27 @@ SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION msar.get_self_database_privileges() RETURNS jsonb AS $$/* +CREATE OR REPLACE FUNCTION msar.get_current_role_database_privileges() RETURNS jsonb AS $$/* Given a database name, returns a json object with database owner oid and database privileges for the role executing the function. The returned JSON object has the form: { "owner_oid": , - "current_role_db_priv" [] + "current_role_priv": [], + "current_role_owner": } */ SELECT jsonb_build_object( 'owner_oid', pgd.datdba::bigint, - 'current_role_db_priv', array_remove( + 'current_role_priv', array_remove( ARRAY[ CASE WHEN has_database_privilege(pgd.oid, 'CREATE') THEN 'CREATE' END, CASE WHEN has_database_privilege(pgd.oid, 'TEMPORARY') THEN 'TEMPORARY' END, CASE WHEN has_database_privilege(pgd.oid, 'CONNECT') THEN 'CONNECT' END ], NULL ), - 'current_role_owner', pg_catalog.pg_has_role(pgd.datdba, 'USAGE') + 'current_role_owns', pg_catalog.pg_has_role(pgd.datdba, 'USAGE') ) FROM pg_catalog.pg_database AS pgd WHERE pgd.datname = current_database(); $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; From 8d133924465209044d35003bdd6bb9078539dff9 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 00:13:47 +0800 Subject: [PATCH 0754/1141] add new schema privilege info to RPC lister --- mathesar/rpc/schemas.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mathesar/rpc/schemas.py b/mathesar/rpc/schemas.py index a372f3b8e0..733f908ac3 100644 --- a/mathesar/rpc/schemas.py +++ b/mathesar/rpc/schemas.py @@ -1,7 +1,7 @@ """ Classes and functions exposed to the RPC endpoint for managing schemas. """ -from typing import Optional, TypedDict +from typing import Literal, Optional, TypedDict from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required @@ -23,11 +23,19 @@ class SchemaInfo(TypedDict): oid: The OID of the schema name: The name of the schema description: A description of the schema + owner_oid: The OID of the owner of the schema + current_role_priv: All privileges available to the calling role + on the schema. + current_role_owns: Whether the current role is the owner of the + schema (even indirectly). table_count: The number of tables in the schema """ oid: int name: str description: Optional[str] + owner_oid: int + current_role_priv: list[Literal['USAGE', 'CREATE']] + current_role_owns: bool table_count: int From 9de56a18914163b1c70aa368c3b8385dba155671 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 00:32:29 +0800 Subject: [PATCH 0755/1141] Add table privilege info to SQL lister and getter --- db/sql/00_msar.sql | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index b0c685b874..60c74f20a5 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -844,6 +844,20 @@ SELECT jsonb_build_object( 'name', relname, 'schema', relnamespace::bigint, 'description', msar.obj_description(oid, 'pg_class') + 'owner_oid', relowner, + 'current_role_priv', array_remove( + ARRAY[ + CASE WHEN pg_catalog.has_table_privilege(oid, 'SELECT') THEN 'SELECT' END, + CASE WHEN pg_catalog.has_table_privilege(oid, 'INSERT') THEN 'INSERT' END, + CASE WHEN pg_catalog.has_table_privilege(oid, 'UPDATE') THEN 'UPDATE' END, + CASE WHEN pg_catalog.has_table_privilege(oid, 'DELETE') THEN 'DELETE' END, + CASE WHEN pg_catalog.has_table_privilege(oid, 'TRUNCATE') THEN 'TRUNCATE' END, + CASE WHEN pg_catalog.has_table_privilege(oid, 'REFERENCES') THEN 'REFERENCES' END, + CASE WHEN pg_catalog.has_table_privilege(oid, 'TRIGGER') THEN 'TRIGGER' END + ], + NULL + ), + 'current_role_owns', pg_catalog.pg_has_role(relowner, 'USAGE') ) FROM pg_catalog.pg_class WHERE oid = tab_id; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -856,7 +870,10 @@ Each returned JSON object in the array will have the form: "oid": , "name": , "schema": , - "description": + "description": , + "owner_oid": , + "current_role_priv": [], + "current_role_owns": } Args: @@ -868,7 +885,21 @@ SELECT coalesce( 'oid', pgc.oid::bigint, 'name', pgc.relname, 'schema', pgc.relnamespace::bigint, - 'description', msar.obj_description(pgc.oid, 'pg_class') + 'description', msar.obj_description(pgc.oid, 'pg_class'), + 'owner_oid', pgc.relowner, + 'current_role_priv', array_remove( + ARRAY[ + CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'SELECT') THEN 'SELECT' END, + CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'INSERT') THEN 'INSERT' END, + CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'UPDATE') THEN 'UPDATE' END, + CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'DELETE') THEN 'DELETE' END, + CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'TRUNCATE') THEN 'TRUNCATE' END, + CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'REFERENCES') THEN 'REFERENCES' END, + CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'TRIGGER') THEN 'TRIGGER' END + ], + NULL + ), + 'current_role_owns', pg_catalog.pg_has_role(pgc.relowner, 'USAGE') ) ), '[]'::jsonb From 9d562d9d3ac59eb4cf6f79e92ef38a3af48391b2 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 00:34:05 +0800 Subject: [PATCH 0756/1141] remove extraneous and out-of-date docstring snippets --- db/tables/operations/select.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/db/tables/operations/select.py b/db/tables/operations/select.py index a25fca1d43..fea5b168b4 100644 --- a/db/tables/operations/select.py +++ b/db/tables/operations/select.py @@ -22,15 +22,6 @@ def get_table(table, conn): The `table` can be given as either a "qualified name", or an OID. The OID is the preferred identifier, since it's much more robust. - The returned dictionary is of the following form: - - { - "oid": , - "name": , - "schema": , - "description": - } - Args: table: The table for which we want table info. """ @@ -44,15 +35,6 @@ def get_table_info(schema, conn): The `schema` can be given as either a "qualified name", or an OID. The OID is the preferred identifier, since it's much more robust. - The returned list contains dictionaries of the following form: - - { - "oid": , - "name": , - "schema": , - "description": - } - Args: schema: The schema for which we want table info. """ From 12a6b63ac38b5b4ed159da5d224023b71ae56e27 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 00:39:42 +0800 Subject: [PATCH 0757/1141] wire up table privilege info to RPC function --- db/sql/00_msar.sql | 5 ++++- mathesar/rpc/tables/base.py | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 60c74f20a5..55c12b80c8 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -833,7 +833,10 @@ Each returned JSON object will have the form: "oid": , "name": , "schema": , - "description": + "description": , + "owner_oid": , + "current_role_priv": [], + "current_role_owns": } Args: diff --git a/mathesar/rpc/tables/base.py b/mathesar/rpc/tables/base.py index 9ee9b4e73e..b0132e480f 100644 --- a/mathesar/rpc/tables/base.py +++ b/mathesar/rpc/tables/base.py @@ -1,7 +1,7 @@ """ Classes and functions exposed to the RPC endpoint for managing tables in a database. """ -from typing import Optional, TypedDict +from typing import Literal, Optional, TypedDict from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required @@ -28,11 +28,27 @@ class TableInfo(TypedDict): name: The name of the table. schema: The `oid` of the schema where the table lives. description: The description of the table. + owner_oid: The OID of the direct owner of the table. + current_role_priv: The privileges held by the user on the table. + current_role_owns: Whether the current role owns the table. """ oid: int name: str schema: int description: Optional[str] + owner_oid: int + current_role_priv: list[ + Literal[ + 'SELECT', + 'INSERT', + 'UPDATE', + 'DELETE', + 'TRUNCATE', + 'REFERENCES', + 'TRIGGER' + ] + ] + current_role_owns: bool class SettableTableInfo(TypedDict): From 3ae677a4eb0937e53941735be26253b9bc82977b Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 00:50:26 +0800 Subject: [PATCH 0758/1141] add privilege info to column response object --- db/columns/operations/select.py | 1 + db/sql/00_msar.sql | 10 ++++++++++ mathesar/rpc/columns/base.py | 1 + 3 files changed, 12 insertions(+) diff --git a/db/columns/operations/select.py b/db/columns/operations/select.py index a79e63e1c8..5ef5f1f508 100644 --- a/db/columns/operations/select.py +++ b/db/columns/operations/select.py @@ -33,6 +33,7 @@ def get_column_info_for_table(table, conn): "valid_target_types": [, , ..., ] "default": {"value": , "is_dynamic": }, "has_dependents": , + "current_role_priv": [, , ...], "description": } diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 55c12b80c8..f603cd0d5e 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -776,6 +776,7 @@ Each returned JSON object in the array will have the form: "default": {"value": , "is_dynamic": }, "has_dependents": , "description": , + "current_role_priv": [, , ...], "valid_target_types": [, , ...] } @@ -808,6 +809,15 @@ SELECT jsonb_agg( ), 'has_dependents', msar.has_dependents(tab_id, attnum), 'description', msar.col_description(tab_id, attnum), + 'current_role_priv', array_remove( + ARRAY[ + CASE WHEN pg_catalog.has_column_privilege(tab_id, attnum, 'SELECT') THEN 'SELECT' END, + CASE WHEN pg_catalog.has_column_privilege(tab_id, attnum, 'INSERT') THEN 'INSERT' END, + CASE WHEN pg_catalog.has_column_privilege(tab_id, attnum, 'UPDATE') THEN 'UPDATE' END, + CASE WHEN pg_catalog.has_column_privilege(tab_id, attnum, 'REFERENCES') THEN 'REFERENCES' END + ], + NULL + ), 'valid_target_types', msar.get_valid_target_type_strings(atttypid) ) ) diff --git a/mathesar/rpc/columns/base.py b/mathesar/rpc/columns/base.py index 1b0f5e954d..cb1ef972d2 100644 --- a/mathesar/rpc/columns/base.py +++ b/mathesar/rpc/columns/base.py @@ -186,6 +186,7 @@ def from_dict(cls, col_info): default=ColumnDefault.from_dict(col_info.get("default")), has_dependents=col_info["has_dependents"], description=col_info.get("description"), + current_role_priv=col_info["current_role_priv"], valid_target_types=col_info.get("valid_target_types") ) From 7723c4261a0d61b66cd570e3c9f99beff0f16a28 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 01:05:02 +0800 Subject: [PATCH 0759/1141] Modify test for new column lister response --- db/sql/00_msar.sql | 2 +- mathesar/rpc/columns/base.py | 4 +++- mathesar/tests/rpc/columns/test_c_base.py | 10 ++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index f603cd0d5e..df3d765222 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -856,7 +856,7 @@ SELECT jsonb_build_object( 'oid', oid::bigint, 'name', relname, 'schema', relnamespace::bigint, - 'description', msar.obj_description(oid, 'pg_class') + 'description', msar.obj_description(oid, 'pg_class'), 'owner_oid', relowner, 'current_role_priv', array_remove( ARRAY[ diff --git a/mathesar/rpc/columns/base.py b/mathesar/rpc/columns/base.py index cb1ef972d2..82251e1b9e 100644 --- a/mathesar/rpc/columns/base.py +++ b/mathesar/rpc/columns/base.py @@ -1,7 +1,7 @@ """ Classes and functions exposed to the RPC endpoint for managing table columns. """ -from typing import Optional, TypedDict +from typing import Literal, Optional, TypedDict from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required @@ -160,6 +160,7 @@ class ColumnInfo(TypedDict): default: The default value and whether it's dynamic. has_dependents: Whether the column has dependent objects. description: The description of the column. + current_role_priv: The privileges of the user for the column. valid_target_types: A list of all types to which the column can be cast. """ @@ -172,6 +173,7 @@ class ColumnInfo(TypedDict): default: ColumnDefault has_dependents: bool description: str + current_role_priv: list[Literal['SELECT', 'INSERT', 'UPDATE', 'REFERENCES']] valid_target_types: list[str] @classmethod diff --git a/mathesar/tests/rpc/columns/test_c_base.py b/mathesar/tests/rpc/columns/test_c_base.py index 88f7db64f5..69d7bc767c 100644 --- a/mathesar/tests/rpc/columns/test_c_base.py +++ b/mathesar/tests/rpc/columns/test_c_base.py @@ -37,6 +37,7 @@ def mock_column_info(_table_oid, conn): 'nullable': False, 'description': None, 'primary_key': True, 'type_options': None, 'has_dependents': True, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': ['text'] }, { 'id': 2, 'name': 'numcol', 'type': 'numeric', @@ -46,6 +47,7 @@ def mock_column_info(_table_oid, conn): 'primary_key': False, 'type_options': {'scale': None, 'precision': None}, 'has_dependents': False, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': ['text'] }, { 'id': 4, 'name': 'numcolmod', 'type': 'numeric', @@ -53,6 +55,7 @@ def mock_column_info(_table_oid, conn): 'nullable': True, 'description': None, 'primary_key': False, 'type_options': {'scale': 3, 'precision': 5}, 'has_dependents': False, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': ['text'] }, { 'id': 8, 'name': 'ivlcolmod', 'type': 'interval', @@ -60,6 +63,7 @@ def mock_column_info(_table_oid, conn): 'nullable': True, 'description': None, 'primary_key': False, 'type_options': {'fields': 'day to second'}, 'has_dependents': False, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': ['text'] }, { 'id': 10, 'name': 'arrcol', 'type': '_array', @@ -67,6 +71,7 @@ def mock_column_info(_table_oid, conn): 'nullable': True, 'description': None, 'primary_key': False, 'type_options': {'item_type': 'character varying', 'length': 3}, 'has_dependents': False, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': None }, ] @@ -80,6 +85,7 @@ def mock_column_info(_table_oid, conn): 'nullable': False, 'description': None, 'primary_key': True, 'type_options': None, 'has_dependents': True, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': ['text'] }, { 'id': 2, 'name': 'numcol', 'type': 'numeric', @@ -89,6 +95,7 @@ def mock_column_info(_table_oid, conn): 'primary_key': False, 'type_options': None, 'has_dependents': False, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': ['text'] }, { 'id': 4, 'name': 'numcolmod', 'type': 'numeric', @@ -96,6 +103,7 @@ def mock_column_info(_table_oid, conn): 'nullable': True, 'description': None, 'primary_key': False, 'type_options': {'scale': 3, 'precision': 5}, 'has_dependents': False, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': ['text'] }, { 'id': 8, 'name': 'ivlcolmod', 'type': 'interval', @@ -103,6 +111,7 @@ def mock_column_info(_table_oid, conn): 'nullable': True, 'description': None, 'primary_key': False, 'type_options': {'fields': 'day to second'}, 'has_dependents': False, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': ['text'] }, { 'id': 10, 'name': 'arrcol', 'type': '_array', @@ -110,6 +119,7 @@ def mock_column_info(_table_oid, conn): 'nullable': True, 'description': None, 'primary_key': False, 'type_options': {'item_type': 'character varying', 'length': 3}, 'has_dependents': False, + 'current_role_priv': ['SELECT', 'INSERT', 'UPDATE'], 'valid_target_types': None } ] From 22ff7fb12291fd06695b0f40d0d048d9c31443da Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 01:12:24 +0800 Subject: [PATCH 0760/1141] update test for new column info response type --- db/sql/test_00_msar.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 7648e7ba1f..2b802e65d1 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2597,6 +2597,7 @@ BEGIN "primary_key": true, "type_options": null, "has_dependents": true, + "current_role_priv": ["SELECT", "INSERT", "UPDATE", "REFERENCES"], "valid_target_types": null }, { @@ -2612,6 +2613,7 @@ BEGIN "precision": null }, "has_dependents": false, + "current_role_priv": ["SELECT", "INSERT", "UPDATE", "REFERENCES"], "valid_target_types": null }, { @@ -2626,6 +2628,7 @@ BEGIN "length": 128 }, "has_dependents": false, + "current_role_priv": ["SELECT", "INSERT", "UPDATE", "REFERENCES"], "valid_target_types": null }, { @@ -2641,6 +2644,7 @@ BEGIN "primary_key": false, "type_options": null, "has_dependents": false, + "current_role_priv": ["SELECT", "INSERT", "UPDATE", "REFERENCES"], "valid_target_types": ["numeric", "text"] }, { @@ -2658,6 +2662,7 @@ BEGIN "precision": null }, "has_dependents": false, + "current_role_priv": ["SELECT", "INSERT", "UPDATE", "REFERENCES"], "valid_target_types": null }, { @@ -2672,6 +2677,7 @@ BEGIN "item_type": "integer" }, "has_dependents": false, + "current_role_priv": ["SELECT", "INSERT", "UPDATE", "REFERENCES"], "valid_target_types": null }, { @@ -2688,6 +2694,7 @@ BEGIN "precision": 15 }, "has_dependents": false, + "current_role_priv": ["SELECT", "INSERT", "UPDATE", "REFERENCES"], "valid_target_types": null } ]$j$::jsonb From 09742176882240e18a2480c042256e2f8507177d Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 28 Aug 2024 01:31:30 +0530 Subject: [PATCH 0761/1141] add SQL func for getting info for current_role --- db/sql/00_msar.sql | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 6049b07a9d..09d7b46cda 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -919,7 +919,6 @@ FROM ( $$ LANGUAGE SQL; -DROP FUNCTION IF EXISTS msar.role_info_table(); CREATE OR REPLACE FUNCTION msar.list_schema_privileges(sch_id regnamespace) RETURNS jsonb AS $$/* Given a schema, returns a json array of objects with direct, non-default schema privileges @@ -946,6 +945,7 @@ SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; +DROP FUNCTION IF EXISTS msar.role_info_table(); CREATE OR REPLACE FUNCTION msar.role_info_table() RETURNS TABLE ( oid bigint, -- The OID of the role. @@ -1037,6 +1037,25 @@ WHERE role_data.name = rolename; $$ LANGUAGE SQL STABLE; +CREATE OR REPLACE FUNCTION +msar.get_current_role() RETURNS TEXT AS $$/* +Returns a JSON object describing the current_role and the parent role(s) it inherits from. +*/ +SELECT jsonb_build_object( + 'current_role', msar.get_role(current_role), + 'parent_roles', array_remove( + array_agg( + CASE WHEN pg_has_role(role_data.name, current_role, 'MEMBER') + THEN msar.get_role(role_data.name) END + ), NULL + ) +) +FROM msar.role_info_table() AS role_data +WHERE role_data.name NOT LIKE 'pg_%' +AND role_data.name != current_role; +$$ LANGUAGE SQL STABLE; + + CREATE OR REPLACE FUNCTION msar.list_db_priv(db_name text) RETURNS jsonb AS $$/* Given a database name, returns a json array of objects with database privileges for non-inherited roles. From 20d3e61f144a28d39bf8e1bab09b64ad0b9f72bf Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 28 Aug 2024 01:31:52 +0530 Subject: [PATCH 0762/1141] wire up sql func to python --- db/roles/operations/select.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py index a6c7119ca9..2186039b45 100644 --- a/db/roles/operations/select.py +++ b/db/roles/operations/select.py @@ -5,6 +5,10 @@ def list_roles(conn): return exec_msar_func(conn, 'list_roles').fetchone()[0] +def get_current_role_from_db(conn): + return exec_msar_func(conn, 'get_current_role').fetchone()[0] + + def list_db_priv(db_name, conn): return exec_msar_func(conn, 'list_db_priv', db_name).fetchone()[0] From d55dea568b56b479df3940d51b84055d93aa9ba3 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 28 Aug 2024 01:32:50 +0530 Subject: [PATCH 0763/1141] add get_current_role rpc endpoint --- mathesar/rpc/roles.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/mathesar/rpc/roles.py b/mathesar/rpc/roles.py index 4f3303243d..c6fb4ccc3a 100644 --- a/mathesar/rpc/roles.py +++ b/mathesar/rpc/roles.py @@ -8,7 +8,7 @@ from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect -from db.roles.operations.select import list_roles +from db.roles.operations.select import list_roles, get_current_role_from_db from db.roles.operations.create import create_role @@ -116,3 +116,26 @@ def add( with connect(database_id, user) as conn: role = create_role(rolename, password, login, conn) return RoleInfo.from_dict(role) + + +@rpc_method(name="roles.get_current_role") +@http_basic_auth_login_required +@handle_rpc_exceptions +def get_current_role(*, database_id: int, **kwargs) -> dict: + """ + Get information about the current role and all the parent role(s) it inherits from. + Requires a database id inorder to connect to the server. + + Args: + database_id: The Django id of the database. + + Returns: + A dict describing the current role. + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + current_role = get_current_role_from_db(conn) + return { + "current_role": RoleInfo.from_dict(current_role["current_role"]), + "parent_roles": [RoleInfo.from_dict(role) for role in current_role["parent_roles"]] + } From eb0b89f3db49211bb764a85212e21ceb99a3c336 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 28 Aug 2024 01:33:48 +0530 Subject: [PATCH 0764/1141] add docs along with endpoint & mock tests --- docs/docs/api/rpc.md | 1 + mathesar/tests/rpc/test_endpoints.py | 5 +++ mathesar/tests/rpc/test_roles.py | 49 ++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 7e4c439e21..9ebf7d7645 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -249,6 +249,7 @@ To use an RPC function: members: - list_ - add + - get_current_role - RoleInfo - RoleMember diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index c26ea473f4..722246f9b9 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -253,6 +253,11 @@ "roles.add", [user_is_authenticated] ), + ( + roles.get_current_role, + "roles.get_current_role", + [user_is_authenticated] + ), ( schemas.add, "schemas.add", diff --git a/mathesar/tests/rpc/test_roles.py b/mathesar/tests/rpc/test_roles.py index 1f606c5693..b5af4f5d06 100644 --- a/mathesar/tests/rpc/test_roles.py +++ b/mathesar/tests/rpc/test_roles.py @@ -108,3 +108,52 @@ def mock_create_role(rolename, password, login, conn): monkeypatch.setattr(roles, 'connect', mock_connect) monkeypatch.setattr(roles, 'create_role', mock_create_role) roles.add(rolename=_username, database_id=_database_id, password=_password, login=True, request=request) + + +def test_get_current_role(rf, monkeypatch): + _username = 'alice' + _password = 'pass1234' + _database_id = 2 + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username=_username, password=_password) + + @contextmanager + def mock_connect(database_id, user): + if database_id == _database_id and user.username == _username: + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_get_current_role(conn): + return { + "current_role": { + 'oid': '2573031', + 'name': 'inherit_msar', + 'login': True, + 'super': False, + 'members': None, + 'inherits': True, + 'create_db': False, + 'create_role': False, + 'description': None + }, + "parent_roles": [ + { + 'oid': '10', + 'name': 'mathesar', + 'login': True, + 'super': True, + 'members': [{'oid': 2573031, 'admin': False}], + 'inherits': True, + 'create_db': True, + 'create_role': True, + 'description': None + } + ] + } + monkeypatch.setattr(roles, 'connect', mock_connect) + monkeypatch.setattr(roles, 'get_current_role_from_db', mock_get_current_role) + roles.get_current_role(database_id=_database_id, request=request) From 402baeb13cc390e4aab3edc28c005ca2ca608b4f Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 13:32:51 +0800 Subject: [PATCH 0765/1141] tidy and extract privilege listing logic --- db/sql/00_msar.sql | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index df3d765222..5b06361222 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -835,6 +835,20 @@ SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid=tab_id AND attname=col_ $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION +msar.list_table_privileges_for_current_role(tab_id regclass) RETURNS jsonb AS $$/* +Return a JSONB array of all privileges current_user holds on the passed table. +*/ +SELECT coalesce(jsonb_agg(privilege), '[]'::jsonb) +FROM + unnest( + ARRAY['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'TRUNCATE', 'REFERENCES', 'TRIGGER'] + ) AS x(privilege), + has_table_privilege(tab_id, privilege) as has_privilege +WHERE has_privilege; +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.get_table(tab_id regclass) RETURNS jsonb AS $$/* Given a table identifier, return a JSON object describing the table. @@ -858,18 +872,7 @@ SELECT jsonb_build_object( 'schema', relnamespace::bigint, 'description', msar.obj_description(oid, 'pg_class'), 'owner_oid', relowner, - 'current_role_priv', array_remove( - ARRAY[ - CASE WHEN pg_catalog.has_table_privilege(oid, 'SELECT') THEN 'SELECT' END, - CASE WHEN pg_catalog.has_table_privilege(oid, 'INSERT') THEN 'INSERT' END, - CASE WHEN pg_catalog.has_table_privilege(oid, 'UPDATE') THEN 'UPDATE' END, - CASE WHEN pg_catalog.has_table_privilege(oid, 'DELETE') THEN 'DELETE' END, - CASE WHEN pg_catalog.has_table_privilege(oid, 'TRUNCATE') THEN 'TRUNCATE' END, - CASE WHEN pg_catalog.has_table_privilege(oid, 'REFERENCES') THEN 'REFERENCES' END, - CASE WHEN pg_catalog.has_table_privilege(oid, 'TRIGGER') THEN 'TRIGGER' END - ], - NULL - ), + 'current_role_priv', msar.list_table_privileges_for_current_role(tab_id), 'current_role_owns', pg_catalog.pg_has_role(relowner, 'USAGE') ) FROM pg_catalog.pg_class WHERE oid = tab_id; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; @@ -900,18 +903,7 @@ SELECT coalesce( 'schema', pgc.relnamespace::bigint, 'description', msar.obj_description(pgc.oid, 'pg_class'), 'owner_oid', pgc.relowner, - 'current_role_priv', array_remove( - ARRAY[ - CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'SELECT') THEN 'SELECT' END, - CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'INSERT') THEN 'INSERT' END, - CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'UPDATE') THEN 'UPDATE' END, - CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'DELETE') THEN 'DELETE' END, - CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'TRUNCATE') THEN 'TRUNCATE' END, - CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'REFERENCES') THEN 'REFERENCES' END, - CASE WHEN pg_catalog.has_table_privilege(pgc.oid, 'TRIGGER') THEN 'TRIGGER' END - ], - NULL - ), + 'current_role_priv', msar.list_table_privileges_for_current_role(pgc.oid), 'current_role_owns', pg_catalog.pg_has_role(pgc.relowner, 'USAGE') ) ), From 8fb3975bbbe6afc67c0ab7843127e822b88b6899 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 14:53:26 +0800 Subject: [PATCH 0766/1141] extract privilege listing for databases, tidy up naming --- db/roles/operations/select.py | 7 +++- db/sql/00_msar.sql | 59 +++++++++++++++++++---------- mathesar/rpc/database_privileges.py | 4 +- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py index f35f2c1a70..4b182ad31f 100644 --- a/db/roles/operations/select.py +++ b/db/roles/operations/select.py @@ -18,4 +18,9 @@ def list_table_privileges(table_oid, conn): def get_curr_role_db_priv(conn): - return exec_msar_func(conn, 'get_self_database_privileges').fetchone()[0] + db_info = exec_msar_func(conn, 'get_current_database_info').fetchone()[0] + return { + "owner_oid": db_info["owner_oid"], + "current_role_priv": db_info["current_role_priv"], + "current_role_owns": db_info["current_role_owns"], + } diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 5b06361222..0466af6767 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -844,7 +844,7 @@ FROM unnest( ARRAY['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'TRUNCATE', 'REFERENCES', 'TRIGGER'] ) AS x(privilege), - has_table_privilege(tab_id, privilege) as has_privilege + pg_catalog.has_table_privilege(tab_id, privilege) as has_privilege WHERE has_privilege; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; @@ -915,6 +915,20 @@ WHERE pgc.relnamespace = sch_id AND pgc.relkind = 'r'; $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION +msar.list_schema_privileges_for_current_role(sch_id regnamespace) RETURNS jsonb AS $$/* +Return a JSONB array of all privileges current_user holds on the passed schema. +*/ +SELECT coalesce(jsonb_agg(privilege), '[]'::jsonb) +FROM + unnest( + ARRAY['USAGE', 'CREATE'] + ) AS x(privilege), + pg_catalog.has_schema_privilege(sch_id, privilege) as has_privilege +WHERE has_privilege; +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.get_schemas() RETURNS jsonb AS $$/* Return a json array of objects describing the user-defined schemas in the database. @@ -942,13 +956,7 @@ FROM ( s.nspname AS name, pg_catalog.obj_description(s.oid) AS description, s.nspowner::bigint AS owner_oid, - array_remove( - ARRAY[ - CASE WHEN pg_catalog.has_schema_privilege(s.oid, 'USAGE') THEN 'USAGE' END, - CASE WHEN pg_catalog.has_schema_privilege(s.oid, 'CREATE') THEN 'CREATE' END - ], - NULL - ) AS current_role_priv, + msar.list_schema_privileges_for_current_role(s.oid) AS current_role_priv, pg_catalog.pg_has_role(s.nspowner, 'USAGE') AS current_role_owns, COALESCE(count(c.oid), 0) AS table_count FROM pg_catalog.pg_namespace s @@ -968,7 +976,6 @@ FROM ( $$ LANGUAGE SQL; -DROP FUNCTION IF EXISTS msar.role_info_table(); CREATE OR REPLACE FUNCTION msar.list_schema_privileges(sch_id regnamespace) RETURNS jsonb AS $$/* Given a schema, returns a json array of objects with direct, non-default schema privileges @@ -995,6 +1002,7 @@ SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; +DROP FUNCTION IF EXISTS msar.role_info_table(); CREATE OR REPLACE FUNCTION msar.role_info_table() RETURNS TABLE ( oid bigint, -- The OID of the role. @@ -1112,26 +1120,37 @@ SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION msar.get_current_role_database_privileges() RETURNS jsonb AS $$/* -Given a database name, returns a json object with database owner oid and database privileges -for the role executing the function. +CREATE OR REPLACE FUNCTION +msar.list_database_privileges_for_current_role(dat_id oid) RETURNS jsonb AS $$/* +Return a JSONB array of all privileges current_user holds on the passed database. +*/ +SELECT coalesce(jsonb_agg(privilege), '[]'::jsonb) +FROM + unnest( + ARRAY['CONNECT', 'CREATE', 'TEMPORARY'] + ) AS x(privilege), + pg_catalog.has_database_privilege(dat_id, privilege) as has_privilege +WHERE has_privilege; +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION msar.get_current_database_info() RETURNS jsonb AS $$/* +Return information about the current database. The returned JSON object has the form: { - "owner_oid": , + "oid": , + "name": , + "owner_oid": , "current_role_priv": [], "current_role_owner": } */ SELECT jsonb_build_object( + 'oid', pgd.oid::bigint, + 'name', pgd.datname, 'owner_oid', pgd.datdba::bigint, - 'current_role_priv', array_remove( - ARRAY[ - CASE WHEN has_database_privilege(pgd.oid, 'CREATE') THEN 'CREATE' END, - CASE WHEN has_database_privilege(pgd.oid, 'TEMPORARY') THEN 'TEMPORARY' END, - CASE WHEN has_database_privilege(pgd.oid, 'CONNECT') THEN 'CONNECT' END - ], NULL - ), + 'current_role_priv', msar.list_database_privileges_for_current_role(pgd.oid), 'current_role_owns', pg_catalog.pg_has_role(pgd.datdba, 'USAGE') ) FROM pg_catalog.pg_database AS pgd WHERE pgd.datname = current_database(); diff --git a/mathesar/rpc/database_privileges.py b/mathesar/rpc/database_privileges.py index f5bd3b62c3..13d1d09076 100644 --- a/mathesar/rpc/database_privileges.py +++ b/mathesar/rpc/database_privileges.py @@ -46,8 +46,8 @@ class CurrentDBPrivileges(TypedDict): def from_dict(cls, d): return cls( owner_oid=d["owner_oid"], - current_role_db_priv=d["current_role_db_priv"], - current_role_owner=d["current_role_owner"] + current_role_priv=d["current_role_priv"], + current_role_owns=d["current_role_owns"] ) From ad91464a6c06ee10dd5aa909ec8a059cd25777a9 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 15:48:58 +0800 Subject: [PATCH 0767/1141] extract column privileges getter function --- db/sql/00_msar.sql | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 0466af6767..452662efd2 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -762,6 +762,18 @@ SELECT EXISTS ( $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION +msar.list_column_privileges_for_current_role(tab_id regclass, attnum smallint) RETURNS jsonb AS $$/* +Return a JSONB array of all privileges current_user holds on the passed table. +*/ +SELECT coalesce(jsonb_agg(privilege), '[]'::jsonb) +FROM + unnest(ARRAY['SELECT', 'INSERT', 'UPDATE', 'REFERENCES']) AS x(privilege), + pg_catalog.has_column_privilege(tab_id, attnum, privilege) as has_privilege +WHERE has_privilege; +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.get_column_info(tab_id regclass) RETURNS jsonb AS $$/* Given a table identifier, return an array of objects describing the columns of the table. @@ -809,15 +821,7 @@ SELECT jsonb_agg( ), 'has_dependents', msar.has_dependents(tab_id, attnum), 'description', msar.col_description(tab_id, attnum), - 'current_role_priv', array_remove( - ARRAY[ - CASE WHEN pg_catalog.has_column_privilege(tab_id, attnum, 'SELECT') THEN 'SELECT' END, - CASE WHEN pg_catalog.has_column_privilege(tab_id, attnum, 'INSERT') THEN 'INSERT' END, - CASE WHEN pg_catalog.has_column_privilege(tab_id, attnum, 'UPDATE') THEN 'UPDATE' END, - CASE WHEN pg_catalog.has_column_privilege(tab_id, attnum, 'REFERENCES') THEN 'REFERENCES' END - ], - NULL - ), + 'current_role_priv', msar.list_column_privileges_for_current_role(tab_id, attnum), 'valid_target_types', msar.get_valid_target_type_strings(atttypid) ) ) From 9b4fea82f32835e421ca55dd2134504f41c5532c Mon Sep 17 00:00:00 2001 From: Ghislaine Guerin Date: Wed, 28 Aug 2024 09:59:36 +0200 Subject: [PATCH 0768/1141] modal and forms consistency and visual updates --- mathesar_ui/src/App.svelte | 12 ++++++++-- .../checkbox-group/CheckboxGroup.svelte | 2 ++ .../collapsible/Collapsible.scss | 3 ++- .../fieldset-group/FieldsetGroup.scss | 2 ++ .../labeled-input/LabeledInput.scss | 1 + .../src/component-library/window/Window.scss | 7 ++++-- .../NameAndDescInputModalForm.svelte | 9 +++++--- .../src/components/NameWithIcon.svelte | 4 ++-- .../components/message-boxes/ErrorBox.svelte | 2 +- .../components/message-boxes/InfoBox.svelte | 2 +- .../message-boxes/MessageBox.svelte | 2 +- .../message-boxes/OutcomeBox.svelte | 2 +- .../message-boxes/WarningBox.svelte | 2 +- mathesar_ui/src/i18n/languages/en/dict.json | 14 ++++++------ .../InstallationSchemaSelector.svelte | 13 +++-------- .../systems/schemas/AddEditSchemaModal.svelte | 15 ++++++++++--- .../ConstraintCollapseHeader.svelte | 2 +- .../constraints/ConstraintTypeSection.svelte | 8 +++---- .../ForeignKeyConstraintDetails.svelte | 1 + .../constraints/NewFkConstraint.svelte | 17 ++++++++++---- .../constraints/NewUniqueConstraint.svelte | 9 +++++--- .../constraints/TableConstraintsModal.svelte | 1 - .../link-table/LinkTableForm.svelte | 7 ++++++ .../link-table/LinkTableModal.svelte | 2 +- .../link-table/LinkTablePill.svelte | 5 +++-- .../table-view/link-table/NewColumn.svelte | 2 +- .../link-table/SelectLinkType.svelte | 1 + .../ExtractColumnsModal.svelte | 22 ++++++++++--------- 28 files changed, 107 insertions(+), 62 deletions(-) diff --git a/mathesar_ui/src/App.svelte b/mathesar_ui/src/App.svelte index d4317163e0..3a14e90ebf 100644 --- a/mathesar_ui/src/App.svelte +++ b/mathesar_ui/src/App.svelte @@ -53,7 +53,7 @@ --color-contrast-light: var(--color-blue-light); --color-link: var(--color-blue-dark); --color-text: #171717; - --color-text-muted: #6b7280; + --color-text-muted: #5e6471; --color-substring-match: rgb(254, 221, 72); --color-substring-match-light: rgba(254, 221, 72, 0.2); --text-size-xx-small: var(--size-xx-small); @@ -65,7 +65,10 @@ --text-size-xx-large: var(--size-xx-large); --text-size-ultra-large: var(--size-ultra-large); --text-size-super-ultra-large: var(--size-super-ultra-large); - + --font-weight-light: 300; + --font-weight-normal: 400; + --font-weight-medium: 500; + --font-weight-bold: 600; --modal-z-index: 50; --modal-record-selector-z-index: 50; @@ -141,6 +144,10 @@ height: 100vh; } + p { + line-height: 1.5; + } + h1 { margin: 0 0 1rem 0; font-size: var(--size-xx-large); @@ -221,4 +228,5 @@ justify-content: center; display: flex; } + diff --git a/mathesar_ui/src/component-library/checkbox-group/CheckboxGroup.svelte b/mathesar_ui/src/component-library/checkbox-group/CheckboxGroup.svelte index 9ed775b439..743d13e50c 100644 --- a/mathesar_ui/src/component-library/checkbox-group/CheckboxGroup.svelte +++ b/mathesar_ui/src/component-library/checkbox-group/CheckboxGroup.svelte @@ -21,6 +21,7 @@ ) => string | ComponentWithProps | undefined = () => undefined; export let getCheckboxDisabled: (value: Option | undefined) => boolean = () => false; + export let boxed = false; /** * By default, options will be compared by equality. If you're using objects as * options, you can supply a custom function here to compare them. @@ -53,6 +54,7 @@ {label} {ariaLabel} {disabled} + {boxed} let:option let:disabled={innerDisabled} on:change diff --git a/mathesar_ui/src/component-library/collapsible/Collapsible.scss b/mathesar_ui/src/component-library/collapsible/Collapsible.scss index 9a3a827c74..59e00806a9 100644 --- a/mathesar_ui/src/component-library/collapsible/Collapsible.scss +++ b/mathesar_ui/src/component-library/collapsible/Collapsible.scss @@ -14,7 +14,8 @@ .collapsible-header-title { flex-grow: 1; overflow: hidden; - font-weight: 600; + font-weight: var(--font-weight-bold); + padding-left: 0.25em; } svg { flex-shrink: 0; diff --git a/mathesar_ui/src/component-library/fieldset-group/FieldsetGroup.scss b/mathesar_ui/src/component-library/fieldset-group/FieldsetGroup.scss index 75baec4fce..65b687afda 100644 --- a/mathesar_ui/src/component-library/fieldset-group/FieldsetGroup.scss +++ b/mathesar_ui/src/component-library/fieldset-group/FieldsetGroup.scss @@ -6,6 +6,8 @@ legend { padding: 0; + font-weight: var(--font-weight-medium); + margin-bottom: 0.5rem; } .options { padding-left: 0; diff --git a/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss b/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss index 0a433abab8..ced2535658 100644 --- a/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss +++ b/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss @@ -5,6 +5,7 @@ .label { display: inline-block; color: var(--slate-800); + font-weight: var(--font-weight-medium); } .input { display: block; diff --git a/mathesar_ui/src/component-library/window/Window.scss b/mathesar_ui/src/component-library/window/Window.scss index dffb8641f0..f5a0a506b6 100644 --- a/mathesar_ui/src/component-library/window/Window.scss +++ b/mathesar_ui/src/component-library/window/Window.scss @@ -3,6 +3,7 @@ border-radius: 0.4em; min-height: var(--window-min-height, 7em); width: var(--window-width, 100%); + min-width: var(--window-min-width, 40em); overflow: hidden; display: grid; grid-template: auto 1fr auto / auto; @@ -15,14 +16,16 @@ display: flex; justify-content: flex-end; align-items: flex-start; + padding: 1rem; + border-bottom: 1px solid var(--slate-200); .title { &:empty { display: none; } flex: 1 1 100%; - padding: 1rem 1rem 0.5rem 1rem; + margin: auto; font-weight: 500; - font-size: var(--text-size-large); + font-size: var(--text-size-x-large); } } diff --git a/mathesar_ui/src/components/NameAndDescInputModalForm.svelte b/mathesar_ui/src/components/NameAndDescInputModalForm.svelte index 300e8be269..7e752b17ed 100644 --- a/mathesar_ui/src/components/NameAndDescInputModalForm.svelte +++ b/mathesar_ui/src/components/NameAndDescInputModalForm.svelte @@ -29,6 +29,9 @@ export let getInitialName: () => string = () => ''; export let getInitialDescription: () => string = () => ''; export let save: (name: string, description: string) => Promise; + //export name and description placeholders + export let namePlaceholder = $_('name'); + export let descriptionPlaceholder = $_('description'); /** * NOTE: This is NOT a feature @@ -100,7 +103,7 @@ nameHasChanged = true; }} disabled={isSubmitting} - placeholder={$_('name')} + placeholder={namePlaceholder} id="name" /> {#if nameHasChanged && nameValidationErrors.length} @@ -118,7 +121,7 @@ bind:value={description} aria-label={$_('description')} disabled={isSubmitting} - placeholder={$_('description')} + placeholder={descriptionPlaceholder} />
@@ -146,7 +149,7 @@ flex-direction: column; > :global(* + *) { - margin-top: 1rem; + margin-top: 0.25rem; } } diff --git a/mathesar_ui/src/components/NameWithIcon.svelte b/mathesar_ui/src/components/NameWithIcon.svelte index 9817367802..fc2c54e170 100644 --- a/mathesar_ui/src/components/NameWithIcon.svelte +++ b/mathesar_ui/src/components/NameWithIcon.svelte @@ -44,7 +44,7 @@ .icon { color: var(--icon-color, currentcolor); opacity: var(--NameWithIcon__icon-opacity, 0.75); - vertical-align: middle; + vertical-align: bottom; } .icon > :global(.fa-icon + .fa-icon) { margin-left: 0.2em; @@ -61,6 +61,6 @@ } .name { color: var(--name-color, currentcolor); - vertical-align: middle; + vertical-align: bottom; } diff --git a/mathesar_ui/src/components/message-boxes/ErrorBox.svelte b/mathesar_ui/src/components/message-boxes/ErrorBox.svelte index d07c970419..5f8a3226e2 100644 --- a/mathesar_ui/src/components/message-boxes/ErrorBox.svelte +++ b/mathesar_ui/src/components/message-boxes/ErrorBox.svelte @@ -15,7 +15,7 @@ diff --git a/mathesar_ui/src/components/message-boxes/MessageBox.svelte b/mathesar_ui/src/components/message-boxes/MessageBox.svelte index 16757983da..4cab9e64e2 100644 --- a/mathesar_ui/src/components/message-boxes/MessageBox.svelte +++ b/mathesar_ui/src/components/message-boxes/MessageBox.svelte @@ -31,7 +31,7 @@ border-radius: var(--border-radius-m); margin: var(--MessageBox__margin); background: var(--MessageBox__background); - border: var(--MessageBox__border); + border-left: var(--MessageBox__border); } .message-box:not(.full-width) { max-width: max-content; diff --git a/mathesar_ui/src/components/message-boxes/OutcomeBox.svelte b/mathesar_ui/src/components/message-boxes/OutcomeBox.svelte index 15993406cc..0d9e9c03df 100644 --- a/mathesar_ui/src/components/message-boxes/OutcomeBox.svelte +++ b/mathesar_ui/src/components/message-boxes/OutcomeBox.svelte @@ -17,7 +17,7 @@ diff --git a/mathesar_ui/src/components/message-boxes/WarningBox.svelte b/mathesar_ui/src/components/message-boxes/WarningBox.svelte index 1fbae58ef2..6c6d9e8b98 100644 --- a/mathesar_ui/src/components/message-boxes/WarningBox.svelte +++ b/mathesar_ui/src/components/message-boxes/WarningBox.svelte @@ -15,7 +15,7 @@ diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index 4f6eab083f..84049bfd43 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -74,15 +74,15 @@ "column_name_cannot_be_empty": "Column name cannot be empty.", "column_names": "Column Names", "column_number_name": "Column {number} Name", - "column_references_target_table": "Column in this table which references the target table", + "column_references_target_table": "The column in this table which references the target table.", "column_will_allow_duplicates": "Column {columnName} will allow duplicates.", "column_will_allow_null": "Column {columnName} will allow NULL", "column_will_not_allow_duplicates": "Column {columnName} will no longer allow duplicates.", "column_will_not_allow_null": "Column {columnName} will no longer allow NULL", "columns": "Columns", "columns_moved_to_table": "Columns {columnNames} have been moved to table {tableName}", - "columns_removed_from_table_added_to_new_table": "{count, plural, one {The column above will be removed from [tableName] and added to the new table} other {The columns above will be removed from [tableName] and added to the new table}}", - "columns_removed_from_table_added_to_target": "{count, plural, one {The column above will be removed from [tableName] and added to [targetTableName]} other {The columns above will be removed from [tableName] and added to [targetTableName]}}", + "columns_removed_from_table_added_to_new_table": "{count, plural, one {The selected column will be moved from [tableName] to the new table} other {The selected columns will be moved from [tableName] to the new table}}.", + "columns_removed_from_table_added_to_target": "{count, plural, one {The selected column will be moved from [tableName] to [targetTableName]} other {The selected columns will be moved from [tableName] to [targetTableName]}}.", "columns_to_extract": "Columns to Extract", "columns_to_move": "Columns to Move", "configure_in_mathesar": "Configure in Mathesar", @@ -106,7 +106,7 @@ "constraint_name_unique_help": "At the database level, each constraint must have a unique name across all the constraints, tables, views, and indexes within the schema.", "constraint_name_unique_mathesar_relevance_help": "In Mathesar however, the name of the constraint will likely never be relevant, — so we recommend allowing Mathesar to automatically generate constraint names when adding new constraints.", "constraints": "Constraints", - "constraints_info_help": "Constraints are used to define relationships between records in different tables or to ensure that records in a column are unique. Constraints can be applied to a single column or a combination of columns.", + "constraints_info_help": "Use constraints to define relationships between records in different tables or ensure uniqueness within a column.", "constraints_info_help_mini": "Constraints are rules that apply to the data in a column to ensure that it is valid.", "content": "Content", "continue": "Continue", @@ -290,7 +290,7 @@ "learn_implications_deleting_mathesar_schemas": "Learn more about the implications of deleting Mathesar schemas.", "link_copied": "Link copied", "link_successfully_regenerated": "The link has been successfully regenerated", - "link_table_to": "Link [tablePill] to", + "link_table_to": "Create link between [tablePill] and", "linked_from_base_table": "Linked from Base Table", "linked_record_summary": "Linked Record Summary", "linked_table": "Linked Table", @@ -300,7 +300,7 @@ "linked_via_foreign_key_column": "It is linked via a foreign key column", "linking_table": "Linking Table", "links": "Links", - "links_info": "Links are stored in the database as foreign key constraints, which you may add to existing columns via the \"Advanced\" section of the table inspector.", + "links_info": "Existing columns can be turned into links by adding foreign key constraints through the \"Advanced\" section of the table inspector.", "loading": "Loading", "loading_release_data": "Loading release data", "loading_spinner_no_progress_bar": "You will see a loading spinner but no progress bar.", @@ -337,7 +337,7 @@ "navigate_to_table_record": "Navigate to a {tableName} record", "needs_import_confirmation": "Needs Import Confirmation", "new_column": "New Column", - "new_column_added_to_table": "A new column will be added to [tableName]", + "new_column_added_to_table": "The new column that will be added to [tableName]", "new_foreign_key_constraint": "New Foreign Key Constraint", "new_link_wont_work_once_regenerated": "Once you regenerate a new link, the old link will no longer work.", "new_password": "New Password", diff --git a/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte b/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte index 276a3cf05c..28a2c88245 100644 --- a/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte +++ b/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte @@ -29,8 +29,9 @@ export let installationSchemas: RequiredField; -
+
- + diff --git a/mathesar_ui/src/systems/schemas/AddEditSchemaModal.svelte b/mathesar_ui/src/systems/schemas/AddEditSchemaModal.svelte index 140593c671..7e2434ee45 100644 --- a/mathesar_ui/src/systems/schemas/AddEditSchemaModal.svelte +++ b/mathesar_ui/src/systems/schemas/AddEditSchemaModal.svelte @@ -60,12 +60,14 @@ getInitialName={() => schema?.name ?? ''} getInitialDescription={() => schema?.description ?? ''} saveButtonLabel={schema ? $_('save') : $_('create_new_schema')} + namePlaceholder='Eg. Personal Finances, Movies' > + {#if !schema} - - {$_('name_your_schema_help')} - + + Use schemas to organize related tables into logical groups within your database. + {/if} @@ -81,3 +83,10 @@ {/if} + + diff --git a/mathesar_ui/src/systems/table-view/constraints/ConstraintCollapseHeader.svelte b/mathesar_ui/src/systems/table-view/constraints/ConstraintCollapseHeader.svelte index 3e60be951a..9246057343 100644 --- a/mathesar_ui/src/systems/table-view/constraints/ConstraintCollapseHeader.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/ConstraintCollapseHeader.svelte @@ -45,7 +45,7 @@ font-size: var(--text-size-small); background-color: var(--slate-200); border-radius: var(--border-radius-xl); - padding: 0.285rem 0.428rem; + padding: 0.25rem 0.75rem; margin-bottom: var(--size-super-ultra-small); } diff --git a/mathesar_ui/src/systems/table-view/constraints/ConstraintTypeSection.svelte b/mathesar_ui/src/systems/table-view/constraints/ConstraintTypeSection.svelte index 52c7b02da1..2794f9920f 100644 --- a/mathesar_ui/src/systems/table-view/constraints/ConstraintTypeSection.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/ConstraintTypeSection.svelte @@ -137,7 +137,7 @@ } :global(.collapsible-content) { - padding-left: 1rem; + padding-left: 2rem; } .null { @@ -150,14 +150,14 @@ justify-content: space-between; align-items: center; font-size: var(--text-size-large); - + font-weight: var(--font-weight-medium); border-bottom: 1px solid var(--slate-200); - padding: 0.25rem; + min-height: 2.5rem; margin-bottom: 0.5rem; } .add-constraint { - padding: 0.5rem; + padding: 1rem; border: 1px solid var(--slate-300); border-radius: var(--border-radius-m); } diff --git a/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte b/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte index 86d35a139d..96fa900b45 100644 --- a/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/ForeignKeyConstraintDetails.svelte @@ -89,6 +89,7 @@ .target-details { display: flex; + align-items: center; > :global(* + *) { margin-left: 0.25rem; diff --git a/mathesar_ui/src/systems/table-view/constraints/NewFkConstraint.svelte b/mathesar_ui/src/systems/table-view/constraints/NewFkConstraint.svelte index 9f478ce067..2ceb2b56da 100644 --- a/mathesar_ui/src/systems/table-view/constraints/NewFkConstraint.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/NewFkConstraint.svelte @@ -121,7 +121,6 @@
- {$_('new_foreign_key_constraint')} + label="Referencing Column" + + > + + {$_('column_references_target_table')} + +
@@ -202,5 +206,10 @@ > :global(* + *) { margin-top: 1rem; } + + .title { + font-weight: var(--font-weight-medium); + font-size: var(--text-size-large); + } } diff --git a/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte b/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte index 7f6769f498..8748583c88 100644 --- a/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/NewUniqueConstraint.svelte @@ -123,10 +123,13 @@
- {$_('new_unique_constraint')}
- + + + The columns in this table that should contain unique values. + + diff --git a/mathesar_ui/src/systems/table-view/constraints/TableConstraintsModal.svelte b/mathesar_ui/src/systems/table-view/constraints/TableConstraintsModal.svelte index df84f81fa2..67b8befe9c 100644 --- a/mathesar_ui/src/systems/table-view/constraints/TableConstraintsModal.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/TableConstraintsModal.svelte @@ -22,7 +22,6 @@ diff --git a/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte b/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte index d288943939..8d8dcaa4a8 100644 --- a/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte +++ b/mathesar_ui/src/systems/table-view/link-table/LinkTableForm.svelte @@ -219,6 +219,10 @@ } +
+ Use Links to connect related tables in your database together. +
+
@@ -340,4 +344,7 @@ --target-fill: var(--base-fill); --target-stroke: var(--base-stroke); } + .description { + margin-bottom: 1rem; + } diff --git a/mathesar_ui/src/systems/table-view/link-table/LinkTableModal.svelte b/mathesar_ui/src/systems/table-view/link-table/LinkTableModal.svelte index 865d9a169c..a27b68f0b0 100644 --- a/mathesar_ui/src/systems/table-view/link-table/LinkTableModal.svelte +++ b/mathesar_ui/src/systems/table-view/link-table/LinkTableModal.svelte @@ -13,7 +13,7 @@ {#if $currentTable} - + controller.close()} /> {/if} diff --git a/mathesar_ui/src/systems/table-view/link-table/LinkTablePill.svelte b/mathesar_ui/src/systems/table-view/link-table/LinkTablePill.svelte index f6fb64885c..a51cc4060a 100644 --- a/mathesar_ui/src/systems/table-view/link-table/LinkTablePill.svelte +++ b/mathesar_ui/src/systems/table-view/link-table/LinkTablePill.svelte @@ -19,10 +19,11 @@ .table-pill { border-radius: 500px; padding: 0 0.5em; + margin: 0 0.1em; max-width: 100%; display: inline-block; - vertical-align: -17%; - line-height: 1.3em; + vertical-align: -20%; + line-height: 1.2em; } .base { background: var(--base-fill); diff --git a/mathesar_ui/src/systems/table-view/link-table/NewColumn.svelte b/mathesar_ui/src/systems/table-view/link-table/NewColumn.svelte index b503c8b8c5..962b28254a 100644 --- a/mathesar_ui/src/systems/table-view/link-table/NewColumn.svelte +++ b/mathesar_ui/src/systems/table-view/link-table/NewColumn.svelte @@ -9,7 +9,7 @@ import Pill from './LinkTablePill.svelte'; - const label = 'Column Name'; + const label = 'Name of New Column'; type Which = ComponentProps['which']; diff --git a/mathesar_ui/src/systems/table-view/link-table/SelectLinkType.svelte b/mathesar_ui/src/systems/table-view/link-table/SelectLinkType.svelte index 16289909fc..51fc4f1470 100644 --- a/mathesar_ui/src/systems/table-view/link-table/SelectLinkType.svelte +++ b/mathesar_ui/src/systems/table-view/link-table/SelectLinkType.svelte @@ -48,6 +48,7 @@ } legend { margin-bottom: 0.5rem; + font-weight: var(--font-weight-medium); } .options { display: grid; diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte index b522b7ec87..05e61bb4ea 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/column-extraction/ExtractColumnsModal.svelte @@ -300,10 +300,10 @@ text={targetTableName ? $_('columns_removed_from_table_added_to_target', { values: { count: $columns.length }, - }) + }) + ' A link column will be created to maintain the relationship between records.' : $_('columns_removed_from_table_added_to_new_table', { values: { count: $columns.length }, - })} + }) + ' A link column will be created to maintain the relationship between records.'} let:slotName > {#if slotName === 'tableName'} @@ -318,14 +318,16 @@

{#if $targetType === 'newTable'} -

- - {#if slotName === 'tableName'} - - {/if} - -

- + + + + + {#if slotName === 'tableName'} + + {/if} + + + {/if}
From 332b27fbdf0ee7375bdca7f021c9ca4c4be0967d Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 15:59:38 +0800 Subject: [PATCH 0769/1141] fix naming in db privilege class --- mathesar/rpc/database_privileges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mathesar/rpc/database_privileges.py b/mathesar/rpc/database_privileges.py index 13d1d09076..dfc661b3d4 100644 --- a/mathesar/rpc/database_privileges.py +++ b/mathesar/rpc/database_privileges.py @@ -39,8 +39,8 @@ class CurrentDBPrivileges(TypedDict): current_role_owner: Whether the current role is an owner of the database. """ owner_oid: int - current_role_db_priv: list[Literal['CONNECT', 'CREATE', 'TEMPORARY']] - current_role_owner: bool + current_role_priv: list[Literal['CONNECT', 'CREATE', 'TEMPORARY']] + current_role_owns: bool @classmethod def from_dict(cls, d): From b6852f3cf2b6353f3dffd036a5289e675feb5253 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 16:35:52 +0800 Subject: [PATCH 0770/1141] add tests for new SQL privilege lister functions --- db/sql/test_00_msar.sql | 134 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 2b802e65d1..c3d914ce76 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4552,3 +4552,137 @@ BEGIN ); END; $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_list_column_privileges_for_current_role() RETURNS SETOF TEXT AS $$ +DECLARE + tab_id oid; +BEGIN +CREATE TABLE mytab (col1 varchar, col2 varchar); +tab_id := 'mytab'::regclass::oid; +CREATE ROLE test_intern1; +CREATE ROLE test_intern2; +GRANT USAGE ON SCHEMA msar, __msar TO test_intern1, test_intern2; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA msar, __msar TO test_intern1, test_intern2; +GRANT SELECT, INSERT (col1) ON TABLE mytab TO test_intern1; +GRANT SELECT (col2) ON TABLE mytab TO test_intern1; +GRANT UPDATE (col1) ON TABLE mytab TO test_intern2; +GRANT UPDATE, REFERENCES (col2) ON TABLE mytab TO test_intern2; + +RETURN NEXT is( + msar.list_column_privileges_for_current_role(tab_id, 1::smallint), + '["SELECT", "INSERT", "UPDATE", "REFERENCES"]' +); +RETURN NEXT is( + msar.list_column_privileges_for_current_role(tab_id, 2::smallint), + '["SELECT", "INSERT", "UPDATE", "REFERENCES"]' +); + +SET ROLE test_intern1; +RETURN NEXT is( + msar.list_column_privileges_for_current_role(tab_id, 1::smallint), + '["SELECT", "INSERT"]' +); +RETURN NEXT is( + msar.list_column_privileges_for_current_role(tab_id, 2::smallint), + '["SELECT"]' +); + +SET ROLE test_intern2; +RETURN NEXT is( + msar.list_column_privileges_for_current_role(tab_id, 1::smallint), + '["UPDATE"]' +); +RETURN NEXT is( + msar.list_column_privileges_for_current_role(tab_id, 2::smallint), + '["UPDATE", "REFERENCES"]' +); + +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_list_schema_privileges_for_current_role() RETURNS SETOF TEXT AS $$ +DECLARE + sch_id oid; +BEGIN +CREATE SCHEMA restricted; +sch_id := 'restricted'::regnamespace::oid; +CREATE ROLE test_intern1; +CREATE ROLE test_intern2; +GRANT USAGE ON SCHEMA msar, __msar TO test_intern1, test_intern2; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA msar, __msar TO test_intern1, test_intern2; +GRANT USAGE ON SCHEMA restricted TO test_intern1; +GRANT USAGE, CREATE ON SCHEMA restricted TO test_intern2; + +RETURN NEXT is(msar.list_schema_privileges_for_current_role(sch_id), '["USAGE", "CREATE"]'); + +SET ROLE test_intern1; +RETURN NEXT is(msar.list_schema_privileges_for_current_role(sch_id), '["USAGE"]'); + +SET ROLE test_intern2; +RETURN NEXT is(msar.list_schema_privileges_for_current_role(sch_id), '["USAGE", "CREATE"]'); + +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_list_table_privileges_for_current_role() RETURNS SETOF TEXT AS $$ +DECLARE + tab_id oid; +BEGIN +CREATE TABLE mytab (col1 varchar); +tab_id := 'mytab'::regclass::oid; +CREATE ROLE test_intern1; +CREATE ROLE test_intern2; +GRANT USAGE ON SCHEMA msar, __msar TO test_intern1, test_intern2; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA msar, __msar TO test_intern1, test_intern2; + +GRANT SELECT, INSERT, UPDATE ON TABLE mytab TO test_intern1; +GRANT DELETE, TRUNCATE, REFERENCES, TRIGGER ON TABLE mytab TO test_intern2; + +RETURN NEXT is( + msar.list_table_privileges_for_current_role(tab_id), + '["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE", "REFERENCES", "TRIGGER"]' +); + +SET ROLE test_intern1; +RETURN NEXT is( + msar.list_table_privileges_for_current_role(tab_id), + '["SELECT", "INSERT", "UPDATE"]' +); + +SET ROLE test_intern2; +RETURN NEXT is( + msar.list_table_privileges_for_current_role(tab_id), + '["DELETE", "TRUNCATE", "REFERENCES", "TRIGGER"]' +); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_list_database_privileges_for_current_role() RETURNS SETOF TEXT AS $$ +DECLARE + dat_id oid := oid FROM pg_database WHERE datname=current_database(); +BEGIN +CREATE ROLE test_intern1; +CREATE ROLE test_intern2; +GRANT USAGE ON SCHEMA msar, __msar TO test_intern1, test_intern2; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA msar, __msar TO test_intern1, test_intern2; + +REVOKE ALL ON DATABASE mathesar_testing FROM PUBLIC; +GRANT CONNECT, CREATE ON DATABASE mathesar_testing TO test_intern1; +GRANT CONNECT, TEMPORARY ON DATABASE mathesar_testing TO test_intern2; + +RETURN NEXT is( + msar.list_database_privileges_for_current_role(dat_id), + '["CONNECT", "CREATE", "TEMPORARY"]' +); + +SET ROLE test_intern1; +RETURN NEXT is(msar.list_database_privileges_for_current_role(dat_id), '["CONNECT", "CREATE"]'); + +SET ROLE test_intern2; +RETURN NEXT is(msar.list_database_privileges_for_current_role(dat_id), '["CONNECT", "TEMPORARY"]'); +END; +$$ LANGUAGE plpgsql; From dfc50145af9d25d3442e77f72ee34cd863d4ee12 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 28 Aug 2024 16:39:44 +0800 Subject: [PATCH 0771/1141] update documentation to be more precise w.r.t. privileges --- mathesar/rpc/columns/base.py | 2 +- mathesar/rpc/database_privileges.py | 4 ++-- mathesar/rpc/tables/base.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mathesar/rpc/columns/base.py b/mathesar/rpc/columns/base.py index 82251e1b9e..16a7641e39 100644 --- a/mathesar/rpc/columns/base.py +++ b/mathesar/rpc/columns/base.py @@ -160,7 +160,7 @@ class ColumnInfo(TypedDict): default: The default value and whether it's dynamic. has_dependents: Whether the column has dependent objects. description: The description of the column. - current_role_priv: The privileges of the user for the column. + current_role_priv: The privileges available to the user for the column. valid_target_types: A list of all types to which the column can be cast. """ diff --git a/mathesar/rpc/database_privileges.py b/mathesar/rpc/database_privileges.py index dfc661b3d4..eb41e04a0c 100644 --- a/mathesar/rpc/database_privileges.py +++ b/mathesar/rpc/database_privileges.py @@ -35,8 +35,8 @@ class CurrentDBPrivileges(TypedDict): Attributes: owner_oid: The `oid` of the owner of the database. - current_role_db_priv: A list of database privileges for the current user. - current_role_owner: Whether the current role is an owner of the database. + current_role_priv: A list of privileges available to the user. + current_role_owns: Whether the user is an owner of the database. """ owner_oid: int current_role_priv: list[Literal['CONNECT', 'CREATE', 'TEMPORARY']] diff --git a/mathesar/rpc/tables/base.py b/mathesar/rpc/tables/base.py index b0132e480f..8b971ca9a3 100644 --- a/mathesar/rpc/tables/base.py +++ b/mathesar/rpc/tables/base.py @@ -29,7 +29,7 @@ class TableInfo(TypedDict): schema: The `oid` of the schema where the table lives. description: The description of the table. owner_oid: The OID of the direct owner of the table. - current_role_priv: The privileges held by the user on the table. + current_role_priv: The privileges available to the user on the table. current_role_owns: Whether the current role owns the table. """ oid: int From 706cb67668db5ab0cac667864df2edaf60cc65a3 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Wed, 28 Aug 2024 18:54:32 +0530 Subject: [PATCH 0772/1141] change logic from 'MEMBER'->'USAGE' and reflect in docstrings --- db/sql/00_msar.sql | 7 ++++--- mathesar/rpc/roles.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index c84a53cc23..2fb08a9fe8 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1038,14 +1038,15 @@ $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION -msar.get_current_role() RETURNS TEXT AS $$/* -Returns a JSON object describing the current_role and the parent role(s) it inherits from. +msar.get_current_role() RETURNS jsonb AS $$/* +Returns a JSON object describing the current_role and the parent role(s) whose +privileges are immediately available to current_role without doing SET ROLE. */ SELECT jsonb_build_object( 'current_role', msar.get_role(current_role), 'parent_roles', array_remove( array_agg( - CASE WHEN pg_has_role(role_data.name, current_role, 'MEMBER') + CASE WHEN pg_has_role(role_data.name, current_role, 'USAGE') THEN msar.get_role(role_data.name) END ), NULL ) diff --git a/mathesar/rpc/roles.py b/mathesar/rpc/roles.py index c6fb4ccc3a..9eb0de8402 100644 --- a/mathesar/rpc/roles.py +++ b/mathesar/rpc/roles.py @@ -123,8 +123,8 @@ def add( @handle_rpc_exceptions def get_current_role(*, database_id: int, **kwargs) -> dict: """ - Get information about the current role and all the parent role(s) it inherits from. - Requires a database id inorder to connect to the server. + Get information about the current role and all the parent role(s) whose + privileges are immediately available to current role without doing SET ROLE. Args: database_id: The Django id of the database. From 23a6751041009edf75dd8bc919fcf23dc9aae390 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 14:38:45 +0800 Subject: [PATCH 0773/1141] make initial move of files --- config/settings/common_settings.py | 19 +++---- mathesar/rpc/databases/base.py | 53 +++++++++---------- mathesar/rpc/databases/configured.py | 52 ++++++++++++++++++ .../privileges.py} | 48 ++--------------- .../{database_setup.py => databases/setup.py} | 10 ++-- mathesar/rpc/roles/__init__.py | 1 + mathesar/rpc/{roles.py => roles/base.py} | 0 .../configured.py} | 8 +-- mathesar/rpc/schemas/__init__.py | 1 + mathesar/rpc/{schemas.py => schemas/base.py} | 0 .../privileges.py} | 4 +- mathesar/rpc/servers/__init__.py | 0 .../rpc/{servers.py => servers/configured.py} | 2 +- .../privileges.py} | 4 +- mathesar/views.py | 4 +- 15 files changed, 108 insertions(+), 98 deletions(-) create mode 100644 mathesar/rpc/databases/configured.py rename mathesar/rpc/{database_privileges.py => databases/privileges.py} (62%) rename mathesar/rpc/{database_setup.py => databases/setup.py} (92%) create mode 100644 mathesar/rpc/roles/__init__.py rename mathesar/rpc/{roles.py => roles/base.py} (100%) rename mathesar/rpc/{configured_roles.py => roles/configured.py} (94%) create mode 100644 mathesar/rpc/schemas/__init__.py rename mathesar/rpc/{schemas.py => schemas/base.py} (100%) rename mathesar/rpc/{schema_privileges.py => schemas/privileges.py} (96%) create mode 100644 mathesar/rpc/servers/__init__.py rename mathesar/rpc/{servers.py => servers/configured.py} (95%) rename mathesar/rpc/{table_privileges.py => tables/privileges.py} (96%) diff --git a/config/settings/common_settings.py b/config/settings/common_settings.py index 387ff5edc5..0aeab18afc 100644 --- a/config/settings/common_settings.py +++ b/config/settings/common_settings.py @@ -66,25 +66,26 @@ def pipe_delim(pipe_string): MODERNRPC_METHODS_MODULES = [ 'mathesar.rpc.collaborators', - 'mathesar.rpc.configured_roles', - 'mathesar.rpc.connections', - 'mathesar.rpc.constraints', 'mathesar.rpc.columns', 'mathesar.rpc.columns.metadata', + 'mathesar.rpc.connections', + 'mathesar.rpc.constraints', 'mathesar.rpc.data_modeling', - 'mathesar.rpc.database_privileges', - 'mathesar.rpc.database_setup', 'mathesar.rpc.databases', + 'mathesar.rpc.databases.configured', + 'mathesar.rpc.databases.privileges', + 'mathesar.rpc.databases.setup', + 'mathesar.rpc.explorations', 'mathesar.rpc.records', 'mathesar.rpc.roles', + 'mathesar.rpc.roles.configured', 'mathesar.rpc.schemas', - 'mathesar.rpc.schema_privileges', - 'mathesar.rpc.servers', + 'mathesar.rpc.schemas.privileges', + 'mathesar.rpc.servers.configured', 'mathesar.rpc.tables', - 'mathesar.rpc.table_privileges', 'mathesar.rpc.tables.metadata', + 'mathesar.rpc.tables.privileges', 'mathesar.rpc.types', - 'mathesar.rpc.explorations' ] TEMPLATES = [ diff --git a/mathesar/rpc/databases/base.py b/mathesar/rpc/databases/base.py index abc9d43b89..42c8415559 100644 --- a/mathesar/rpc/databases/base.py +++ b/mathesar/rpc/databases/base.py @@ -1,52 +1,49 @@ -from typing import TypedDict +from typing import Literal, TypedDict -from modernrpc.core import rpc_method +from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required -from mathesar.models.base import Database +from mathesar.rpc.utils import connect +from db.roles.operations.select import get_curr_role_db_priv from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -class DatabaseInfo(TypedDict): +class CurrentDBPrivileges(TypedDict): """ - Information about a database. + Information about database privileges for current user. Attributes: - id: the Django ID of the database model instance. - name: The name of the database on the server. - server_id: the Django ID of the server model instance for the database. + owner_oid: The `oid` of the owner of the database. + current_role_priv: A list of privileges available to the user. + current_role_owns: Whether the user is an owner of the database. """ - id: int - name: str - server_id: int + owner_oid: int + current_role_priv: list[Literal['CONNECT', 'CREATE', 'TEMPORARY']] + current_role_owns: bool @classmethod - def from_model(cls, model): + def from_dict(cls, d): return cls( - id=model.id, - name=model.name, - server_id=model.server.id + owner_oid=d["owner_oid"], + current_role_priv=d["current_role_priv"], + current_role_owns=d["current_role_owns"] ) -@rpc_method(name="databases.list") +@rpc_method(name="databases.get") @http_basic_auth_login_required @handle_rpc_exceptions -def list_(*, server_id: int = None, **kwargs) -> list[DatabaseInfo]: +def get(*, database_id: int, **kwargs) -> CurrentDBPrivileges: """ - List information about databases for a server. Exposed as `list`. - - If called with no `server_id`, all databases for all servers are listed. + Get database privileges for the current user. Args: - server_id: The Django id of the server containing the databases. + database_id: The Django id of the database. Returns: - A list of database details. + A dict describing current user's database privilege. """ - if server_id is not None: - database_qs = Database.objects.filter(server__id=server_id) - else: - database_qs = Database.objects.all() - - return [DatabaseInfo.from_model(db_model) for db_model in database_qs] + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + curr_role_db_priv = get_curr_role_db_priv(conn) + return CurrentDBPrivileges.from_dict(curr_role_db_priv) diff --git a/mathesar/rpc/databases/configured.py b/mathesar/rpc/databases/configured.py new file mode 100644 index 0000000000..d37e2cd732 --- /dev/null +++ b/mathesar/rpc/databases/configured.py @@ -0,0 +1,52 @@ +from typing import TypedDict + +from modernrpc.core import rpc_method +from modernrpc.auth.basic import http_basic_auth_login_required + +from mathesar.models.base import Database +from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions + + +class DatabaseInfo(TypedDict): + """ + Information about a database. + + Attributes: + id: the Django ID of the database model instance. + name: The name of the database on the server. + server_id: the Django ID of the server model instance for the database. + """ + id: int + name: str + server_id: int + + @classmethod + def from_model(cls, model): + return cls( + id=model.id, + name=model.name, + server_id=model.server.id + ) + + +@rpc_method(name="databases.configured.list") +@http_basic_auth_login_required +@handle_rpc_exceptions +def list_(*, server_id: int = None, **kwargs) -> list[DatabaseInfo]: + """ + List information about databases for a server. Exposed as `list`. + + If called with no `server_id`, all databases for all servers are listed. + + Args: + server_id: The Django id of the server containing the databases. + + Returns: + A list of database details. + """ + if server_id is not None: + database_qs = Database.objects.filter(server__id=server_id) + else: + database_qs = Database.objects.all() + + return [DatabaseInfo.from_model(db_model) for db_model in database_qs] diff --git a/mathesar/rpc/database_privileges.py b/mathesar/rpc/databases/privileges.py similarity index 62% rename from mathesar/rpc/database_privileges.py rename to mathesar/rpc/databases/privileges.py index eb41e04a0c..265284e345 100644 --- a/mathesar/rpc/database_privileges.py +++ b/mathesar/rpc/databases/privileges.py @@ -3,7 +3,7 @@ from modernrpc.core import rpc_method, REQUEST_KEY from modernrpc.auth.basic import http_basic_auth_login_required -from db.roles.operations.select import list_db_priv, get_curr_role_db_priv +from db.roles.operations.select import list_db_priv from db.roles.operations.update import replace_database_privileges_for_roles from mathesar.rpc.utils import connect from mathesar.models.base import Database @@ -29,29 +29,7 @@ def from_dict(cls, d): ) -class CurrentDBPrivileges(TypedDict): - """ - Information about database privileges for current user. - - Attributes: - owner_oid: The `oid` of the owner of the database. - current_role_priv: A list of privileges available to the user. - current_role_owns: Whether the user is an owner of the database. - """ - owner_oid: int - current_role_priv: list[Literal['CONNECT', 'CREATE', 'TEMPORARY']] - current_role_owns: bool - - @classmethod - def from_dict(cls, d): - return cls( - owner_oid=d["owner_oid"], - current_role_priv=d["current_role_priv"], - current_role_owns=d["current_role_owns"] - ) - - -@rpc_method(name="database_privileges.list_direct") +@rpc_method(name="databases.privileges.list_direct") @http_basic_auth_login_required @handle_rpc_exceptions def list_direct(*, database_id: int, **kwargs) -> list[DBPrivileges]: @@ -71,27 +49,7 @@ def list_direct(*, database_id: int, **kwargs) -> list[DBPrivileges]: return [DBPrivileges.from_dict(i) for i in raw_db_priv] -# TODO: Think of something concise for the endpoint name. -@rpc_method(name="database_privileges.get_self") -@http_basic_auth_login_required -@handle_rpc_exceptions -def get_self(*, database_id: int, **kwargs) -> CurrentDBPrivileges: - """ - Get database privileges for the current user. - - Args: - database_id: The Django id of the database. - - Returns: - A dict describing current user's database privilege. - """ - user = kwargs.get(REQUEST_KEY).user - with connect(database_id, user) as conn: - curr_role_db_priv = get_curr_role_db_priv(conn) - return CurrentDBPrivileges.from_dict(curr_role_db_priv) - - -@rpc_method(name="database_privileges.replace_for_roles") +@rpc_method(name="databases.privileges.replace_for_roles") @http_basic_auth_login_required @handle_rpc_exceptions def replace_for_roles( diff --git a/mathesar/rpc/database_setup.py b/mathesar/rpc/databases/setup.py similarity index 92% rename from mathesar/rpc/database_setup.py rename to mathesar/rpc/databases/setup.py index d81b936d41..92f61dfa66 100644 --- a/mathesar/rpc/database_setup.py +++ b/mathesar/rpc/databases/setup.py @@ -8,9 +8,9 @@ from mathesar.utils import permissions from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -from mathesar.rpc.servers import ServerInfo -from mathesar.rpc.databases import DatabaseInfo -from mathesar.rpc.configured_roles import ConfiguredRoleInfo +from mathesar.rpc.servers.configured import ServerInfo +from mathesar.rpc.databases.configured import DatabaseInfo +from mathesar.rpc.roles.configured import ConfiguredRoleInfo class DatabaseConnectionResult(TypedDict): @@ -38,7 +38,7 @@ def from_model(cls, model): ) -@rpc_method(name='database_setup.create_new') +@rpc_method(name='databases.setup.create_new') @http_basic_auth_superuser_required @handle_rpc_exceptions def create_new( @@ -66,7 +66,7 @@ def create_new( return DatabaseConnectionResult.from_model(result) -@rpc_method(name='database_setup.connect_existing') +@rpc_method(name='databases.setup.connect_existing') @http_basic_auth_superuser_required @handle_rpc_exceptions def connect_existing( diff --git a/mathesar/rpc/roles/__init__.py b/mathesar/rpc/roles/__init__.py new file mode 100644 index 0000000000..44c8b96840 --- /dev/null +++ b/mathesar/rpc/roles/__init__.py @@ -0,0 +1 @@ +from .base import * # noqa diff --git a/mathesar/rpc/roles.py b/mathesar/rpc/roles/base.py similarity index 100% rename from mathesar/rpc/roles.py rename to mathesar/rpc/roles/base.py diff --git a/mathesar/rpc/configured_roles.py b/mathesar/rpc/roles/configured.py similarity index 94% rename from mathesar/rpc/configured_roles.py rename to mathesar/rpc/roles/configured.py index 6c587b4905..8c92444127 100644 --- a/mathesar/rpc/configured_roles.py +++ b/mathesar/rpc/roles/configured.py @@ -29,7 +29,7 @@ def from_model(cls, model): ) -@rpc_method(name="configured_roles.list") +@rpc_method(name="roles.configured.list") @http_basic_auth_login_required @handle_rpc_exceptions def list_(*, server_id: int, **kwargs) -> list[ConfiguredRoleInfo]: @@ -47,7 +47,7 @@ def list_(*, server_id: int, **kwargs) -> list[ConfiguredRoleInfo]: return [ConfiguredRoleInfo.from_model(db_model) for db_model in configured_role_qs] -@rpc_method(name='configured_roles.add') +@rpc_method(name='roles.configured.add') @http_basic_auth_superuser_required @handle_rpc_exceptions def add( @@ -77,7 +77,7 @@ def add( return ConfiguredRoleInfo.from_model(configured_role) -@rpc_method(name='configured_roles.delete') +@rpc_method(name='roles.configured.delete') @http_basic_auth_superuser_required @handle_rpc_exceptions def delete(*, configured_role_id: int, **kwargs): @@ -91,7 +91,7 @@ def delete(*, configured_role_id: int, **kwargs): configured_role.delete() -@rpc_method(name='configured_roles.set_password') +@rpc_method(name='roles.configured.set_password') @http_basic_auth_superuser_required @handle_rpc_exceptions def set_password( diff --git a/mathesar/rpc/schemas/__init__.py b/mathesar/rpc/schemas/__init__.py new file mode 100644 index 0000000000..44c8b96840 --- /dev/null +++ b/mathesar/rpc/schemas/__init__.py @@ -0,0 +1 @@ +from .base import * # noqa diff --git a/mathesar/rpc/schemas.py b/mathesar/rpc/schemas/base.py similarity index 100% rename from mathesar/rpc/schemas.py rename to mathesar/rpc/schemas/base.py diff --git a/mathesar/rpc/schema_privileges.py b/mathesar/rpc/schemas/privileges.py similarity index 96% rename from mathesar/rpc/schema_privileges.py rename to mathesar/rpc/schemas/privileges.py index 3257e514e0..d96893d4d8 100644 --- a/mathesar/rpc/schema_privileges.py +++ b/mathesar/rpc/schemas/privileges.py @@ -28,7 +28,7 @@ def from_dict(cls, d): ) -@rpc_method(name="schema_privileges.list_direct") +@rpc_method(name="schemas.privileges.list_direct") @http_basic_auth_login_required @handle_rpc_exceptions def list_direct( @@ -50,7 +50,7 @@ def list_direct( return [SchemaPrivileges.from_dict(i) for i in raw_priv] -@rpc_method(name="schema_privileges.replace_for_roles") +@rpc_method(name="schemas.privileges.replace_for_roles") @http_basic_auth_login_required @handle_rpc_exceptions def replace_for_roles( diff --git a/mathesar/rpc/servers/__init__.py b/mathesar/rpc/servers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mathesar/rpc/servers.py b/mathesar/rpc/servers/configured.py similarity index 95% rename from mathesar/rpc/servers.py rename to mathesar/rpc/servers/configured.py index 71602e9a39..a85845a15e 100644 --- a/mathesar/rpc/servers.py +++ b/mathesar/rpc/servers/configured.py @@ -29,7 +29,7 @@ def from_model(cls, model): ) -@rpc_method(name="servers.list") +@rpc_method(name="servers.configured.list") @http_basic_auth_login_required @handle_rpc_exceptions def list_() -> list[ServerInfo]: diff --git a/mathesar/rpc/table_privileges.py b/mathesar/rpc/tables/privileges.py similarity index 96% rename from mathesar/rpc/table_privileges.py rename to mathesar/rpc/tables/privileges.py index 801ac0a5e9..64ae9ec7dd 100644 --- a/mathesar/rpc/table_privileges.py +++ b/mathesar/rpc/tables/privileges.py @@ -27,7 +27,7 @@ def from_dict(cls, d): ) -@rpc_method(name="table_privileges.list_direct") +@rpc_method(name="tables.privileges.list_direct") @http_basic_auth_login_required @handle_rpc_exceptions def list_direct( @@ -47,7 +47,7 @@ def list_direct( return [TablePrivileges.from_dict(i) for i in raw_priv] -@rpc_method(name="table_privileges.replace_for_roles") +@rpc_method(name="tables.privileges.replace_for_roles") @http_basic_auth_login_required @handle_rpc_exceptions def replace_for_roles( diff --git a/mathesar/views.py b/mathesar/views.py index 0db30e56b1..1f4080aeb9 100644 --- a/mathesar/views.py +++ b/mathesar/views.py @@ -7,9 +7,9 @@ from rest_framework.decorators import api_view from rest_framework.response import Response -from mathesar.rpc.databases import list_ as databases_list +from mathesar.rpc.databases.configured import list_ as databases_list from mathesar.rpc.schemas import list_ as schemas_list -from mathesar.rpc.servers import list_ as get_servers_list +from mathesar.rpc.servers.configured import list_ as get_servers_list from mathesar.api.serializers.databases import TypeSerializer from mathesar.api.serializers.tables import TableSerializer from mathesar.api.serializers.queries import QuerySerializer From 7b47adc93a2c4c7c2e985d229fcb7712ca7214a5 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 15:04:55 +0800 Subject: [PATCH 0774/1141] fix endpoints and roles tests --- mathesar/tests/rpc/test_endpoints.py | 223 ++++++++++++++------------- mathesar/tests/rpc/test_roles.py | 12 +- 2 files changed, 125 insertions(+), 110 deletions(-) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index 979277f76a..cede022e5c 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -10,21 +10,16 @@ from mathesar.rpc import collaborators from mathesar.rpc import columns -from mathesar.rpc import configured_roles from mathesar.rpc import connections from mathesar.rpc import constraints from mathesar.rpc import data_modeling -from mathesar.rpc import database_privileges -from mathesar.rpc import database_setup from mathesar.rpc import databases from mathesar.rpc import explorations from mathesar.rpc import records from mathesar.rpc import roles from mathesar.rpc import schemas -from mathesar.rpc import schema_privileges from mathesar.rpc import servers from mathesar.rpc import tables -from mathesar.rpc import table_privileges from mathesar.rpc import types METHODS = [ @@ -48,6 +43,12 @@ "collaborators.set_role", [user_is_superuser] ), + + ( + columns.add, + "columns.add", + [user_is_authenticated] + ), ( columns.delete, "columns.delete", @@ -59,20 +60,16 @@ [user_is_authenticated] ), ( - columns.patch, - "columns.patch", - [user_is_authenticated] - ), - ( - columns.add, - "columns.add", + columns.list_with_metadata, + "columns.list_with_metadata", [user_is_authenticated] ), ( - columns.list_with_metadata, - "columns.list_with_metadata", + columns.patch, + "columns.patch", [user_is_authenticated] ), + ( columns.metadata.list_, "columns.metadata.list", @@ -83,26 +80,7 @@ "columns.metadata.set", [user_is_authenticated] ), - ( - configured_roles.add, - "configured_roles.add", - [user_is_superuser] - ), - ( - configured_roles.list_, - "configured_roles.list", - [user_is_authenticated] - ), - ( - configured_roles.delete, - "configured_roles.delete", - [user_is_superuser] - ), - ( - configured_roles.set_password, - "configured_roles.set_password", - [user_is_superuser] - ), + ( connections.add_from_known_connection, "connections.add_from_known_connection", @@ -118,6 +96,7 @@ "connections.grant_access_to_user", [user_is_superuser] ), + ( constraints.list_, "constraints.list", @@ -133,6 +112,7 @@ "constraints.delete", [user_is_authenticated] ), + ( data_modeling.add_foreign_key_column, "data_modeling.add_foreign_key_column", @@ -148,101 +128,108 @@ "data_modeling.suggest_types", [user_is_authenticated] ), + ( - database_privileges.list_direct, - "database_privileges.list_direct", + databases.get, + "databases.get", [user_is_authenticated] ), + ( - database_privileges.get_self, - "database_privileges.get_self", + databases.configured.list_, + "databases.configured.list", [user_is_authenticated] ), + ( - database_privileges.replace_for_roles, - "database_privileges.replace_for_roles", + databases.privileges.list_direct, + "databases.privileges.list_direct", [user_is_authenticated] ), ( - database_setup.create_new, - "database_setup.create_new", - [user_is_superuser] + databases.privileges.replace_for_roles, + "databases.privileges.replace_for_roles", + [user_is_authenticated] ), + ( - database_setup.connect_existing, - "database_setup.connect_existing", + databases.setup.create_new, + "databases.setup.create_new", [user_is_superuser] ), ( - databases.list_, - "databases.list", - [user_is_authenticated] + databases.setup.connect_existing, + "databases.setup.connect_existing", + [user_is_superuser] ), + ( - records.list_, - "records.list", + explorations.add, + "explorations.add", [user_is_authenticated] ), ( - records.get, - "records.get", + explorations.delete, + "explorations.delete", [user_is_authenticated] ), ( - records.add, - "records.add", + explorations.get, + "explorations.get", [user_is_authenticated] ), ( - records.patch, - "records.patch", + explorations.list_, + "explorations.list", [user_is_authenticated] ), ( - records.delete, - "records.delete", + explorations.replace, + "explorations.replace", [user_is_authenticated] ), ( - records.search, - "records.search", + explorations.run, + "explorations.run", [user_is_authenticated] ), ( - explorations.list_, - "explorations.list", + explorations.run_saved, + "explorations.run_saved", [user_is_authenticated] ), + ( - explorations.get, - "explorations.get", + records.add, + "records.add", [user_is_authenticated] ), ( - explorations.add, - "explorations.add", + records.delete, + "records.delete", [user_is_authenticated] ), ( - explorations.delete, - "explorations.delete", + records.get, + "records.get", [user_is_authenticated] ), ( - explorations.replace, - "explorations.replace", + records.list_, + "records.list", [user_is_authenticated] ), ( - explorations.run, - "explorations.run", + records.patch, + "records.patch", [user_is_authenticated] ), ( - explorations.run_saved, - "explorations.run_saved", + records.search, + "records.search", [user_is_authenticated] ), + ( roles.list_, "roles.list", @@ -258,14 +245,31 @@ "roles.get_current_role", [user_is_authenticated] ), + + ( + roles.configured.add, + "roles.configured.add", + [user_is_superuser] + ), ( - schemas.add, - "schemas.add", + roles.configured.delete, + "roles.configured.delete", + [user_is_superuser] + ), + ( + roles.configured.list_, + "roles.configured.list", [user_is_authenticated] ), ( - schemas.list_, - "schemas.list", + roles.configured.set_password, + "roles.configured.set_password", + [user_is_superuser] + ), + + ( + schemas.add, + "schemas.add", [user_is_authenticated] ), ( @@ -273,40 +277,49 @@ "schemas.delete", [user_is_authenticated] ), + ( + schemas.list_, + "schemas.list", + [user_is_authenticated] + ), ( schemas.patch, "schemas.patch", [user_is_authenticated] ), + ( - schema_privileges.list_direct, - "schema_privileges.list_direct", + schemas.privileges.list_direct, + "schemas.privileges.list_direct", [user_is_authenticated] ), ( - schema_privileges.replace_for_roles, - "schema_privileges.replace_for_roles", + schemas.privileges.replace_for_roles, + "schemas.privileges.replace_for_roles", [user_is_authenticated] ), + ( - servers.list_, - "servers.list", + servers.configured.list_, + "servers.configured.list", [user_is_authenticated] ), + ( types.list_, "types.list", [user_is_authenticated] ), + ( - tables.list_, - "tables.list", + tables.add, + "tables.add", [user_is_authenticated] ), ( - tables.list_with_metadata, - "tables.list_with_metadata", + tables.delete, + "tables.delete", [user_is_authenticated] ), ( @@ -315,45 +328,47 @@ [user_is_authenticated] ), ( - tables.add, - "tables.add", + tables.get_import_preview, + "tables.get_import_preview", [user_is_authenticated] ), ( - tables.delete, - "tables.delete", + tables.import_, + "tables.import", [user_is_authenticated] ), ( - tables.patch, - "tables.patch", + tables.list_, + "tables.list", [user_is_authenticated] ), ( - tables.import_, - "tables.import", + tables.list_joinable, + "tables.list_joinable", [user_is_authenticated] ), ( - tables.get_import_preview, - "tables.get_import_preview", + tables.list_with_metadata, + "tables.list_with_metadata", [user_is_authenticated] ), ( - tables.list_joinable, - "tables.list_joinable", + tables.patch, + "tables.patch", [user_is_authenticated] ), + ( - table_privileges.list_direct, - "table_privileges.list_direct", + tables.privileges.list_direct, + "tables.privileges.list_direct", [user_is_authenticated] ), ( - table_privileges.replace_for_roles, - "table_privileges.replace_for_roles", + tables.privileges.replace_for_roles, + "tables.privileges.replace_for_roles", [user_is_authenticated] ), + ( tables.metadata.list_, "tables.metadata.list", diff --git a/mathesar/tests/rpc/test_roles.py b/mathesar/tests/rpc/test_roles.py index b5af4f5d06..7d9cbe539f 100644 --- a/mathesar/tests/rpc/test_roles.py +++ b/mathesar/tests/rpc/test_roles.py @@ -65,8 +65,8 @@ def mock_list_roles(conn): }, ] - monkeypatch.setattr(roles, 'connect', mock_connect) - monkeypatch.setattr(roles, 'list_roles', mock_list_roles) + monkeypatch.setattr(roles.base, 'connect', mock_connect) + monkeypatch.setattr(roles.base, 'list_roles', mock_list_roles) roles.list_(database_id=_database_id, request=request) @@ -105,8 +105,8 @@ def mock_create_role(rolename, password, login, conn): 'description': None } - monkeypatch.setattr(roles, 'connect', mock_connect) - monkeypatch.setattr(roles, 'create_role', mock_create_role) + monkeypatch.setattr(roles.base, 'connect', mock_connect) + monkeypatch.setattr(roles.base, 'create_role', mock_create_role) roles.add(rolename=_username, database_id=_database_id, password=_password, login=True, request=request) @@ -154,6 +154,6 @@ def mock_get_current_role(conn): } ] } - monkeypatch.setattr(roles, 'connect', mock_connect) - monkeypatch.setattr(roles, 'get_current_role_from_db', mock_get_current_role) + monkeypatch.setattr(roles.base, 'connect', mock_connect) + monkeypatch.setattr(roles.base, 'get_current_role_from_db', mock_get_current_role) roles.get_current_role(database_id=_database_id, request=request) From e8003dc03a525e5ba562af22c5452ed18c147729 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 15:11:17 +0800 Subject: [PATCH 0775/1141] fix namespaced db module tests --- mathesar/tests/rpc/test_database_privileges.py | 8 ++++---- mathesar/tests/rpc/test_database_setup.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/mathesar/tests/rpc/test_database_privileges.py b/mathesar/tests/rpc/test_database_privileges.py index 688a254990..3e904f5513 100644 --- a/mathesar/tests/rpc/test_database_privileges.py +++ b/mathesar/tests/rpc/test_database_privileges.py @@ -7,7 +7,7 @@ """ from contextlib import contextmanager -from mathesar.rpc import database_privileges +from mathesar.rpc.databases import privileges from mathesar.models.users import User @@ -37,9 +37,9 @@ def mock_replace_privileges( raise AssertionError('incorrect parameters passed') return _privileges + [{"role_oid": 67890, "direct": ["CONNECT", "TEMPORARY"]}] - monkeypatch.setattr(database_privileges, 'connect', mock_connect) + monkeypatch.setattr(privileges, 'connect', mock_connect) monkeypatch.setattr( - database_privileges, + privileges, 'replace_database_privileges_for_roles', mock_replace_privileges ) @@ -47,7 +47,7 @@ def mock_replace_privileges( {"role_oid": 12345, "direct": ["CONNECT"]}, {"role_oid": 67890, "direct": ["CONNECT", "TEMPORARY"]} ] - actual_response = database_privileges.replace_for_roles( + actual_response = privileges.replace_for_roles( privileges=_privileges, database_id=_database_id, request=request ) assert actual_response == expect_response diff --git a/mathesar/tests/rpc/test_database_setup.py b/mathesar/tests/rpc/test_database_setup.py index be7c6ac15d..c1a20f1a24 100644 --- a/mathesar/tests/rpc/test_database_setup.py +++ b/mathesar/tests/rpc/test_database_setup.py @@ -7,7 +7,10 @@ """ from mathesar.models.users import User from mathesar.models.base import Database, ConfiguredRole, Server, UserDatabaseRoleMap -from mathesar.rpc import database_setup, servers, configured_roles, databases +from mathesar.rpc.databases import configured as configured_databases +from mathesar.rpc.databases import setup as database_setup +from mathesar.rpc.roles import configured as configured_roles +from mathesar.rpc.servers import configured as configured_servers def test_create_new(monkeypatch, rf): @@ -36,8 +39,8 @@ def mock_set_up_new_for_user(database, user, sample_data=[]): mock_set_up_new_for_user, ) expect_response = database_setup.DatabaseConnectionResult( - server=servers.ServerInfo.from_model(server_model), - database=databases.DatabaseInfo.from_model(db_model), + server=configured_servers.ServerInfo.from_model(server_model), + database=configured_databases.DatabaseInfo.from_model(db_model), configured_role=configured_roles.ConfiguredRoleInfo.from_model(role_model) ) @@ -83,8 +86,8 @@ def mock_set_up_preexisting_database_for_user( mock_set_up_preexisting_database_for_user, ) expect_response = database_setup.DatabaseConnectionResult( - server=servers.ServerInfo.from_model(server_model), - database=databases.DatabaseInfo.from_model(db_model), + server=configured_servers.ServerInfo.from_model(server_model), + database=configured_databases.DatabaseInfo.from_model(db_model), configured_role=configured_roles.ConfiguredRoleInfo.from_model(role_model) ) From 2feae908f88a8066feac261224f6faa574511b8d Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 15:15:12 +0800 Subject: [PATCH 0776/1141] fix schema and table submodule tests --- mathesar/tests/rpc/test_schema_privileges.py | 2 +- mathesar/tests/rpc/test_table_privileges.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mathesar/tests/rpc/test_schema_privileges.py b/mathesar/tests/rpc/test_schema_privileges.py index 23b2ff9f48..8a934afc00 100644 --- a/mathesar/tests/rpc/test_schema_privileges.py +++ b/mathesar/tests/rpc/test_schema_privileges.py @@ -7,7 +7,7 @@ """ from contextlib import contextmanager -from mathesar.rpc import schema_privileges +from mathesar.rpc.schemas import privileges as schema_privileges from mathesar.models.users import User diff --git a/mathesar/tests/rpc/test_table_privileges.py b/mathesar/tests/rpc/test_table_privileges.py index 29f784902a..bbd190b83b 100644 --- a/mathesar/tests/rpc/test_table_privileges.py +++ b/mathesar/tests/rpc/test_table_privileges.py @@ -7,7 +7,7 @@ """ from contextlib import contextmanager -from mathesar.rpc import table_privileges +from mathesar.rpc.tables import privileges as table_privileges from mathesar.models.users import User From 1ac4b60157cd311c1b4052f706d4a4ccac438667 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 15:32:50 +0800 Subject: [PATCH 0777/1141] update docs to match new namespacing --- docs/docs/api/rpc.md | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 63ccc9150d..7938546c3c 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -48,17 +48,6 @@ To use an RPC function: - set_role - CollaboratorInfo -## ConfiguredRoles - -::: configured_roles - options: - members: - - list_ - - add - - delete - - set_password - - ConfiguredRoleInfo - ## Connections ::: connections @@ -72,6 +61,14 @@ To use an RPC function: ## Databases ::: databases + options: + members: + - get + - CurrentDBPrivileges + +## Configured Databases + +::: databases.configured options: members: - list_ @@ -79,18 +76,16 @@ To use an RPC function: ## Database Privileges -::: database_privileges +::: databases.privileges options: members: - list_direct - - get_owner_oid_and_curr_role_db_priv - replace_for_roles - DBPrivileges - - CurrentDBPrivileges ## Database Setup -::: database_setup +::: databases.setup options: members: - create_new @@ -111,7 +106,7 @@ To use an RPC function: ## Schema Privileges -::: schema_privileges +::: schemas.privileges options: members: - list_direct @@ -138,7 +133,7 @@ To use an RPC function: ## Table Privileges -::: table_privileges +::: tables.privileges options: members: - list_direct @@ -255,6 +250,17 @@ To use an RPC function: - RoleInfo - RoleMember +## Configured Roles + +::: roles.configured + options: + members: + - list_ + - add + - delete + - set_password + - ConfiguredRoleInfo + ## Servers ::: servers From 9aa29e7d4988ae6ee7f17541d774eec08f9bded6 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 15:45:02 +0800 Subject: [PATCH 0778/1141] change DatabaseInfo to ConfiguredDatabaseInfo --- docs/docs/api/rpc.md | 2 +- mathesar/rpc/databases/configured.py | 6 +++--- mathesar/rpc/databases/setup.py | 6 +++--- mathesar/tests/rpc/test_database_setup.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 7938546c3c..4a0af96f10 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -72,7 +72,7 @@ To use an RPC function: options: members: - list_ - - DatabaseInfo + - ConfiguredDatabaseInfo ## Database Privileges diff --git a/mathesar/rpc/databases/configured.py b/mathesar/rpc/databases/configured.py index d37e2cd732..f22ca281e3 100644 --- a/mathesar/rpc/databases/configured.py +++ b/mathesar/rpc/databases/configured.py @@ -7,7 +7,7 @@ from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -class DatabaseInfo(TypedDict): +class ConfiguredDatabaseInfo(TypedDict): """ Information about a database. @@ -32,7 +32,7 @@ def from_model(cls, model): @rpc_method(name="databases.configured.list") @http_basic_auth_login_required @handle_rpc_exceptions -def list_(*, server_id: int = None, **kwargs) -> list[DatabaseInfo]: +def list_(*, server_id: int = None, **kwargs) -> list[ConfiguredDatabaseInfo]: """ List information about databases for a server. Exposed as `list`. @@ -49,4 +49,4 @@ def list_(*, server_id: int = None, **kwargs) -> list[DatabaseInfo]: else: database_qs = Database.objects.all() - return [DatabaseInfo.from_model(db_model) for db_model in database_qs] + return [ConfiguredDatabaseInfo.from_model(db_model) for db_model in database_qs] diff --git a/mathesar/rpc/databases/setup.py b/mathesar/rpc/databases/setup.py index 92f61dfa66..52b9a296ab 100644 --- a/mathesar/rpc/databases/setup.py +++ b/mathesar/rpc/databases/setup.py @@ -9,7 +9,7 @@ from mathesar.utils import permissions from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.servers.configured import ServerInfo -from mathesar.rpc.databases.configured import DatabaseInfo +from mathesar.rpc.databases.configured import ConfiguredDatabaseInfo from mathesar.rpc.roles.configured import ConfiguredRoleInfo @@ -26,14 +26,14 @@ class DatabaseConnectionResult(TypedDict): configured_role: Information on the ConfiguredRole model instance. """ server: ServerInfo - database: DatabaseInfo + database: ConfiguredDatabaseInfo configured_role: ConfiguredRoleInfo @classmethod def from_model(cls, model): return cls( server=ServerInfo.from_model(model.server), - database=DatabaseInfo.from_model(model.database), + database=ConfiguredDatabaseInfo.from_model(model.database), configured_role=ConfiguredRoleInfo.from_model(model.configured_role), ) diff --git a/mathesar/tests/rpc/test_database_setup.py b/mathesar/tests/rpc/test_database_setup.py index c1a20f1a24..68a6c9c7b5 100644 --- a/mathesar/tests/rpc/test_database_setup.py +++ b/mathesar/tests/rpc/test_database_setup.py @@ -40,7 +40,7 @@ def mock_set_up_new_for_user(database, user, sample_data=[]): ) expect_response = database_setup.DatabaseConnectionResult( server=configured_servers.ServerInfo.from_model(server_model), - database=configured_databases.DatabaseInfo.from_model(db_model), + database=configured_databases.ConfiguredDatabaseInfo.from_model(db_model), configured_role=configured_roles.ConfiguredRoleInfo.from_model(role_model) ) @@ -87,7 +87,7 @@ def mock_set_up_preexisting_database_for_user( ) expect_response = database_setup.DatabaseConnectionResult( server=configured_servers.ServerInfo.from_model(server_model), - database=configured_databases.DatabaseInfo.from_model(db_model), + database=configured_databases.ConfiguredDatabaseInfo.from_model(db_model), configured_role=configured_roles.ConfiguredRoleInfo.from_model(role_model) ) From 46aa8c5504b97c9bcf678219b5cec366230e9888 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 15:51:11 +0800 Subject: [PATCH 0779/1141] change ServerInfo to ConfiguredServerInfo --- docs/docs/api/rpc.md | 2 +- mathesar/rpc/databases/setup.py | 6 +++--- mathesar/rpc/servers/configured.py | 6 +++--- mathesar/tests/rpc/test_database_setup.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 4a0af96f10..f585a0fb99 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -267,7 +267,7 @@ To use an RPC function: options: members: - list_ - - ServerInfo + - ConfiguredServerInfo ## Data Modeling diff --git a/mathesar/rpc/databases/setup.py b/mathesar/rpc/databases/setup.py index 52b9a296ab..d423d3438e 100644 --- a/mathesar/rpc/databases/setup.py +++ b/mathesar/rpc/databases/setup.py @@ -8,7 +8,7 @@ from mathesar.utils import permissions from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -from mathesar.rpc.servers.configured import ServerInfo +from mathesar.rpc.servers.configured import ConfiguredServerInfo from mathesar.rpc.databases.configured import ConfiguredDatabaseInfo from mathesar.rpc.roles.configured import ConfiguredRoleInfo @@ -25,14 +25,14 @@ class DatabaseConnectionResult(TypedDict): database: Information on the Database model instance. configured_role: Information on the ConfiguredRole model instance. """ - server: ServerInfo + server: ConfiguredServerInfo database: ConfiguredDatabaseInfo configured_role: ConfiguredRoleInfo @classmethod def from_model(cls, model): return cls( - server=ServerInfo.from_model(model.server), + server=ConfiguredServerInfo.from_model(model.server), database=ConfiguredDatabaseInfo.from_model(model.database), configured_role=ConfiguredRoleInfo.from_model(model.configured_role), ) diff --git a/mathesar/rpc/servers/configured.py b/mathesar/rpc/servers/configured.py index a85845a15e..0d488966d6 100644 --- a/mathesar/rpc/servers/configured.py +++ b/mathesar/rpc/servers/configured.py @@ -7,7 +7,7 @@ from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -class ServerInfo(TypedDict): +class ConfiguredServerInfo(TypedDict): """ Information about a database server. @@ -32,7 +32,7 @@ def from_model(cls, model): @rpc_method(name="servers.configured.list") @http_basic_auth_login_required @handle_rpc_exceptions -def list_() -> list[ServerInfo]: +def list_() -> list[ConfiguredServerInfo]: """ List information about servers. Exposed as `list`. @@ -41,4 +41,4 @@ def list_() -> list[ServerInfo]: """ server_qs = Server.objects.all() - return [ServerInfo.from_model(db_model) for db_model in server_qs] + return [ConfiguredServerInfo.from_model(db_model) for db_model in server_qs] diff --git a/mathesar/tests/rpc/test_database_setup.py b/mathesar/tests/rpc/test_database_setup.py index 68a6c9c7b5..83263fe237 100644 --- a/mathesar/tests/rpc/test_database_setup.py +++ b/mathesar/tests/rpc/test_database_setup.py @@ -39,7 +39,7 @@ def mock_set_up_new_for_user(database, user, sample_data=[]): mock_set_up_new_for_user, ) expect_response = database_setup.DatabaseConnectionResult( - server=configured_servers.ServerInfo.from_model(server_model), + server=configured_servers.ConfiguredServerInfo.from_model(server_model), database=configured_databases.ConfiguredDatabaseInfo.from_model(db_model), configured_role=configured_roles.ConfiguredRoleInfo.from_model(role_model) ) @@ -86,7 +86,7 @@ def mock_set_up_preexisting_database_for_user( mock_set_up_preexisting_database_for_user, ) expect_response = database_setup.DatabaseConnectionResult( - server=configured_servers.ServerInfo.from_model(server_model), + server=configured_servers.ConfiguredServerInfo.from_model(server_model), database=configured_databases.ConfiguredDatabaseInfo.from_model(db_model), configured_role=configured_roles.ConfiguredRoleInfo.from_model(role_model) ) From f821adc44562dbb6eba3a1ae55afac9403ae37b0 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 16:20:35 +0800 Subject: [PATCH 0780/1141] remove unneeded param from list_db_priv --- db/roles/operations/select.py | 4 ++-- db/sql/00_msar.sql | 7 ++++--- mathesar/rpc/databases/privileges.py | 3 +-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py index 49e05e5eeb..6e5ab208e2 100644 --- a/db/roles/operations/select.py +++ b/db/roles/operations/select.py @@ -9,8 +9,8 @@ def get_current_role_from_db(conn): return exec_msar_func(conn, 'get_current_role').fetchone()[0] -def list_db_priv(db_name, conn): - return exec_msar_func(conn, 'list_db_priv', db_name).fetchone()[0] +def list_db_priv(conn): + return exec_msar_func(conn, 'list_db_priv').fetchone()[0] def list_schema_privileges(schema_oid, conn): diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 5f7d33f653..52c67e6011 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -1118,7 +1118,7 @@ AND role_data.name != current_role; $$ LANGUAGE SQL STABLE; -CREATE OR REPLACE FUNCTION msar.list_db_priv(db_name text) RETURNS jsonb AS $$/* +CREATE OR REPLACE FUNCTION msar.list_db_priv() RETURNS jsonb AS $$/* Given a database name, returns a json array of objects with database privileges for non-inherited roles. Each returned JSON object in the array has the form: @@ -1137,7 +1137,8 @@ WITH priv_cte AS ( pg_catalog.pg_roles AS pgr, pg_catalog.pg_database AS pgd, aclexplode(COALESCE(pgd.datacl, acldefault('d', pgd.datdba))) AS acl - WHERE pgd.datname = db_name AND pgr.oid = acl.grantee AND pgr.rolname NOT LIKE 'pg_%' + WHERE pgd.datname = pg_catalog.current_database() + AND pgr.oid = acl.grantee AND pgr.rolname NOT LIKE 'pg_%' GROUP BY pgr.oid, pgd.oid ) SELECT COALESCE(jsonb_agg(priv_cte.p), '[]'::jsonb) FROM priv_cte; @@ -1292,7 +1293,7 @@ EXECUTE string_agg( E';\n' ) || ';' FROM jsonb_to_recordset(priv_spec) AS x(role_oid regrole, direct jsonb); -RETURN msar.list_db_priv(current_database()); +RETURN msar.list_db_priv(); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; diff --git a/mathesar/rpc/databases/privileges.py b/mathesar/rpc/databases/privileges.py index 265284e345..ae5993bf7e 100644 --- a/mathesar/rpc/databases/privileges.py +++ b/mathesar/rpc/databases/privileges.py @@ -44,8 +44,7 @@ def list_direct(*, database_id: int, **kwargs) -> list[DBPrivileges]: """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: - db_name = Database.objects.get(id=database_id).name - raw_db_priv = list_db_priv(db_name, conn) + raw_db_priv = list_db_priv(conn) return [DBPrivileges.from_dict(i) for i in raw_db_priv] From ec6274843d199e89150d018a0487473d6128063d Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 16:36:03 +0800 Subject: [PATCH 0781/1141] modify database getter to conform with other functions --- db/databases/__init__.py | 0 db/databases/operations/__init__.py | 0 db/databases/operations/select.py | 5 +++++ db/roles/operations/select.py | 9 --------- mathesar/rpc/databases/base.py | 22 ++++++++++++++-------- 5 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 db/databases/__init__.py create mode 100644 db/databases/operations/__init__.py create mode 100644 db/databases/operations/select.py diff --git a/db/databases/__init__.py b/db/databases/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/db/databases/operations/__init__.py b/db/databases/operations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/db/databases/operations/select.py b/db/databases/operations/select.py new file mode 100644 index 0000000000..d95b1e8f75 --- /dev/null +++ b/db/databases/operations/select.py @@ -0,0 +1,5 @@ +from db.connection import exec_msar_func + + +def get_database(conn): + return exec_msar_func(conn, 'get_current_database_info').fetchone()[0] diff --git a/db/roles/operations/select.py b/db/roles/operations/select.py index 6e5ab208e2..daffc31550 100644 --- a/db/roles/operations/select.py +++ b/db/roles/operations/select.py @@ -19,12 +19,3 @@ def list_schema_privileges(schema_oid, conn): def list_table_privileges(table_oid, conn): return exec_msar_func(conn, 'list_table_privileges', table_oid).fetchone()[0] - - -def get_curr_role_db_priv(conn): - db_info = exec_msar_func(conn, 'get_current_database_info').fetchone()[0] - return { - "owner_oid": db_info["owner_oid"], - "current_role_priv": db_info["current_role_priv"], - "current_role_owns": db_info["current_role_owns"], - } diff --git a/mathesar/rpc/databases/base.py b/mathesar/rpc/databases/base.py index 42c8415559..da2968a447 100644 --- a/mathesar/rpc/databases/base.py +++ b/mathesar/rpc/databases/base.py @@ -4,19 +4,23 @@ from modernrpc.auth.basic import http_basic_auth_login_required from mathesar.rpc.utils import connect -from db.roles.operations.select import get_curr_role_db_priv +from db.databases.operations.select import get_database from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions -class CurrentDBPrivileges(TypedDict): +class DatabaseInfo(TypedDict): """ - Information about database privileges for current user. + Information about a database current user privileges on it. Attributes: + oid: The `oid` of the database on the server. + name: The name of the database on the server. owner_oid: The `oid` of the owner of the database. current_role_priv: A list of privileges available to the user. current_role_owns: Whether the user is an owner of the database. """ + oid: int + name: str owner_oid: int current_role_priv: list[Literal['CONNECT', 'CREATE', 'TEMPORARY']] current_role_owns: bool @@ -24,6 +28,8 @@ class CurrentDBPrivileges(TypedDict): @classmethod def from_dict(cls, d): return cls( + oid=d["oid"], + name=d["name"], owner_oid=d["owner_oid"], current_role_priv=d["current_role_priv"], current_role_owns=d["current_role_owns"] @@ -33,17 +39,17 @@ def from_dict(cls, d): @rpc_method(name="databases.get") @http_basic_auth_login_required @handle_rpc_exceptions -def get(*, database_id: int, **kwargs) -> CurrentDBPrivileges: +def get(*, database_id: int, **kwargs) -> DatabaseInfo: """ - Get database privileges for the current user. + Get information about a database. Args: database_id: The Django id of the database. Returns: - A dict describing current user's database privilege. + Information about the database, and the current user privileges. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: - curr_role_db_priv = get_curr_role_db_priv(conn) - return CurrentDBPrivileges.from_dict(curr_role_db_priv) + db_info = get_database(conn) + return DatabaseInfo.from_dict(db_info) From d0f1340bf52e8b87091dbc2415e26746429c31cc Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 16:45:14 +0800 Subject: [PATCH 0782/1141] update docs with new class name --- docs/docs/api/rpc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index f585a0fb99..2a9567210a 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -64,7 +64,7 @@ To use an RPC function: options: members: - get - - CurrentDBPrivileges + - DatabaseInfo ## Configured Databases From f35c62b1df683ebf91f9d06884e8759ab8368361 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 29 Aug 2024 16:47:37 +0800 Subject: [PATCH 0783/1141] satisfy the linter --- mathesar/rpc/databases/privileges.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mathesar/rpc/databases/privileges.py b/mathesar/rpc/databases/privileges.py index ae5993bf7e..98d1aada2f 100644 --- a/mathesar/rpc/databases/privileges.py +++ b/mathesar/rpc/databases/privileges.py @@ -6,7 +6,6 @@ from db.roles.operations.select import list_db_priv from db.roles.operations.update import replace_database_privileges_for_roles from mathesar.rpc.utils import connect -from mathesar.models.base import Database from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions From 01f7b4e229630061111078bf7f87a1577d79d3c8 Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 29 Aug 2024 18:20:44 +0530 Subject: [PATCH 0784/1141] Move configured_roles to roles.configured --- mathesar_ui/src/api/rpc/collaborators.ts | 2 +- mathesar_ui/src/api/rpc/configured_roles.ts | 43 --------------------- mathesar_ui/src/api/rpc/database_setup.ts | 2 +- mathesar_ui/src/api/rpc/index.ts | 2 - mathesar_ui/src/api/rpc/roles.ts | 40 +++++++++++++++++++ mathesar_ui/src/models/ConfiguredRole.ts | 6 +-- mathesar_ui/src/models/Database.ts | 2 +- mathesar_ui/src/models/Role.ts | 2 +- 8 files changed, 47 insertions(+), 52 deletions(-) delete mode 100644 mathesar_ui/src/api/rpc/configured_roles.ts diff --git a/mathesar_ui/src/api/rpc/collaborators.ts b/mathesar_ui/src/api/rpc/collaborators.ts index 5c89d2829f..d9b07786ba 100644 --- a/mathesar_ui/src/api/rpc/collaborators.ts +++ b/mathesar_ui/src/api/rpc/collaborators.ts @@ -1,7 +1,7 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; -import type { RawConfiguredRole } from './configured_roles'; import type { RawDatabase } from './databases'; +import type { RawConfiguredRole } from './roles'; export interface RawCollaborator { id: number; diff --git a/mathesar_ui/src/api/rpc/configured_roles.ts b/mathesar_ui/src/api/rpc/configured_roles.ts deleted file mode 100644 index bb30bae9ec..0000000000 --- a/mathesar_ui/src/api/rpc/configured_roles.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; - -import type { RawServer } from './servers'; - -export interface RawConfiguredRole { - id: number; - server_id: RawServer['id']; - name: string; -} - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const configured_roles = { - list: rpcMethodTypeContainer< - { - server_id: RawConfiguredRole['server_id']; - }, - Array - >(), - - add: rpcMethodTypeContainer< - { - server_id: RawConfiguredRole['server_id']; - name: RawConfiguredRole['name']; - password: string; - }, - RawConfiguredRole - >(), - - delete: rpcMethodTypeContainer< - { - configured_role_id: RawConfiguredRole['id']; - }, - void - >(), - - set_password: rpcMethodTypeContainer< - { - configured_role_id: RawConfiguredRole['id']; - password: string; - }, - void - >(), -}; diff --git a/mathesar_ui/src/api/rpc/database_setup.ts b/mathesar_ui/src/api/rpc/database_setup.ts index 81c5a0f93b..97524c477a 100644 --- a/mathesar_ui/src/api/rpc/database_setup.ts +++ b/mathesar_ui/src/api/rpc/database_setup.ts @@ -1,7 +1,7 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; -import type { RawConfiguredRole } from './configured_roles'; import type { RawDatabase } from './databases'; +import type { RawConfiguredRole } from './roles'; import type { RawServer } from './servers'; export const sampleDataOptions = [ diff --git a/mathesar_ui/src/api/rpc/index.ts b/mathesar_ui/src/api/rpc/index.ts index a48b958e06..628207f507 100644 --- a/mathesar_ui/src/api/rpc/index.ts +++ b/mathesar_ui/src/api/rpc/index.ts @@ -4,7 +4,6 @@ import { buildRpcApi } from '@mathesar/packages/json-rpc-client-builder'; import { collaborators } from './collaborators'; import { columns } from './columns'; -import { configured_roles } from './configured_roles'; import { constraints } from './constraints'; import { database_setup } from './database_setup'; import { databases } from './databases'; @@ -20,7 +19,6 @@ export const api = buildRpcApi({ getHeaders: () => ({ 'X-CSRFToken': Cookies.get('csrftoken') }), methodTree: { collaborators, - configured_roles, database_setup, databases, records, diff --git a/mathesar_ui/src/api/rpc/roles.ts b/mathesar_ui/src/api/rpc/roles.ts index 82bbb2366c..4db5252a03 100644 --- a/mathesar_ui/src/api/rpc/roles.ts +++ b/mathesar_ui/src/api/rpc/roles.ts @@ -1,6 +1,7 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; import type { RawDatabase } from './databases'; +import type { RawServer } from './servers'; export interface RawRoleMember { oid: number; @@ -19,6 +20,12 @@ export interface RawRole { members?: RawRoleMember[]; } +export interface RawConfiguredRole { + id: number; + server_id: RawServer['id']; + name: string; +} + export const roles = { list: rpcMethodTypeContainer< { @@ -53,4 +60,37 @@ export const roles = { }, void >(), + + configured: { + list: rpcMethodTypeContainer< + { + server_id: RawConfiguredRole['server_id']; + }, + Array + >(), + + add: rpcMethodTypeContainer< + { + server_id: RawConfiguredRole['server_id']; + name: RawConfiguredRole['name']; + password: string; + }, + RawConfiguredRole + >(), + + delete: rpcMethodTypeContainer< + { + configured_role_id: RawConfiguredRole['id']; + }, + void + >(), + + set_password: rpcMethodTypeContainer< + { + configured_role_id: RawConfiguredRole['id']; + password: string; + }, + void + >(), + }, }; diff --git a/mathesar_ui/src/models/ConfiguredRole.ts b/mathesar_ui/src/models/ConfiguredRole.ts index 3ebfaf5a46..b581dde208 100644 --- a/mathesar_ui/src/models/ConfiguredRole.ts +++ b/mathesar_ui/src/models/ConfiguredRole.ts @@ -1,5 +1,5 @@ import { api } from '@mathesar/api/rpc'; -import type { RawConfiguredRole } from '@mathesar/api/rpc/configured_roles'; +import type { RawConfiguredRole } from '@mathesar/api/rpc/roles'; import type { Database } from './Database'; @@ -20,7 +20,7 @@ export class ConfiguredRole { } setPassword(password: string) { - return api.configured_roles + return api.roles.configured .set_password({ configured_role_id: this.id, password, @@ -29,6 +29,6 @@ export class ConfiguredRole { } delete() { - return api.configured_roles.delete({ configured_role_id: this.id }).run(); + return api.roles.configured.delete({ configured_role_id: this.id }).run(); } } diff --git a/mathesar_ui/src/models/Database.ts b/mathesar_ui/src/models/Database.ts index be35d9b2e9..ccc0dbe1e1 100644 --- a/mathesar_ui/src/models/Database.ts +++ b/mathesar_ui/src/models/Database.ts @@ -26,7 +26,7 @@ export class Database { } constructConfiguredRolesStore() { - return new AsyncRpcApiStore(api.configured_roles.list, { + return new AsyncRpcApiStore(api.roles.configured.list, { postProcess: (rawConfiguredRoles) => new SortedImmutableMap( (v) => [...v].sort(([, a], [, b]) => a.name.localeCompare(b.name)), diff --git a/mathesar_ui/src/models/Role.ts b/mathesar_ui/src/models/Role.ts index 97ebbe432f..80397c58a3 100644 --- a/mathesar_ui/src/models/Role.ts +++ b/mathesar_ui/src/models/Role.ts @@ -54,7 +54,7 @@ export class Role { } configure(password: string): CancellablePromise { - const promise = api.configured_roles + const promise = api.roles.configured .add({ server_id: this.database.server.id, name: this.name, From ebfce9ae135b515cc79dbcf508fe810dbf9bfada Mon Sep 17 00:00:00 2001 From: pavish Date: Thu, 29 Aug 2024 18:26:51 +0530 Subject: [PATCH 0785/1141] Update namespaces for servers, databases.configured, and databases.setup --- mathesar_ui/src/api/rpc/database_setup.ts | 41 ---------------- mathesar_ui/src/api/rpc/databases.ts | 48 ++++++++++++++++--- mathesar_ui/src/api/rpc/index.ts | 2 - mathesar_ui/src/api/rpc/servers.ts | 4 +- mathesar_ui/src/stores/databases.ts | 8 ++-- .../InstallationSchemaSelector.svelte | 2 +- .../create-database/createDatabaseUtils.ts | 2 +- 7 files changed, 51 insertions(+), 56 deletions(-) delete mode 100644 mathesar_ui/src/api/rpc/database_setup.ts diff --git a/mathesar_ui/src/api/rpc/database_setup.ts b/mathesar_ui/src/api/rpc/database_setup.ts deleted file mode 100644 index 97524c477a..0000000000 --- a/mathesar_ui/src/api/rpc/database_setup.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; - -import type { RawDatabase } from './databases'; -import type { RawConfiguredRole } from './roles'; -import type { RawServer } from './servers'; - -export const sampleDataOptions = [ - 'library_management', - 'movie_collection', -] as const; - -export type SampleDataSchemaIdentifier = (typeof sampleDataOptions)[number]; - -export interface DatabaseConnectionResult { - server: RawServer; - database: RawDatabase; - configured_role: RawConfiguredRole; -} - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const database_setup = { - create_new: rpcMethodTypeContainer< - { - database: RawDatabase['name']; - sample_data?: SampleDataSchemaIdentifier[]; - }, - DatabaseConnectionResult - >(), - - connect_existing: rpcMethodTypeContainer< - { - host: RawServer['host']; - port: RawServer['port']; - database: RawDatabase['name']; - role: RawConfiguredRole['name']; - password: string; - sample_data?: SampleDataSchemaIdentifier[]; - }, - DatabaseConnectionResult - >(), -}; diff --git a/mathesar_ui/src/api/rpc/databases.ts b/mathesar_ui/src/api/rpc/databases.ts index 10e9a11628..488e335d93 100644 --- a/mathesar_ui/src/api/rpc/databases.ts +++ b/mathesar_ui/src/api/rpc/databases.ts @@ -1,5 +1,6 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; +import type { RawConfiguredRole } from './roles'; import type { RawServer } from './servers'; export interface RawDatabase { @@ -8,11 +9,46 @@ export interface RawDatabase { server_id: RawServer['id']; } +export const sampleDataOptions = [ + 'library_management', + 'movie_collection', +] as const; + +export type SampleDataSchemaIdentifier = (typeof sampleDataOptions)[number]; + +export interface DatabaseConnectionResult { + server: RawServer; + database: RawDatabase; + configured_role: RawConfiguredRole; +} + export const databases = { - list: rpcMethodTypeContainer< - { - server_id?: RawDatabase['server_id']; - }, - Array - >(), + configured: { + list: rpcMethodTypeContainer< + { + server_id?: RawDatabase['server_id']; + }, + Array + >(), + }, + setup: { + create_new: rpcMethodTypeContainer< + { + database: RawDatabase['name']; + sample_data?: SampleDataSchemaIdentifier[]; + }, + DatabaseConnectionResult + >(), + connect_existing: rpcMethodTypeContainer< + { + host: RawServer['host']; + port: RawServer['port']; + database: RawDatabase['name']; + role: RawConfiguredRole['name']; + password: string; + sample_data?: SampleDataSchemaIdentifier[]; + }, + DatabaseConnectionResult + >(), + }, }; diff --git a/mathesar_ui/src/api/rpc/index.ts b/mathesar_ui/src/api/rpc/index.ts index 628207f507..f1dd6a8533 100644 --- a/mathesar_ui/src/api/rpc/index.ts +++ b/mathesar_ui/src/api/rpc/index.ts @@ -5,7 +5,6 @@ import { buildRpcApi } from '@mathesar/packages/json-rpc-client-builder'; import { collaborators } from './collaborators'; import { columns } from './columns'; import { constraints } from './constraints'; -import { database_setup } from './database_setup'; import { databases } from './databases'; import { records } from './records'; import { roles } from './roles'; @@ -19,7 +18,6 @@ export const api = buildRpcApi({ getHeaders: () => ({ 'X-CSRFToken': Cookies.get('csrftoken') }), methodTree: { collaborators, - database_setup, databases, records, roles, diff --git a/mathesar_ui/src/api/rpc/servers.ts b/mathesar_ui/src/api/rpc/servers.ts index f92a29bebb..1a1253def4 100644 --- a/mathesar_ui/src/api/rpc/servers.ts +++ b/mathesar_ui/src/api/rpc/servers.ts @@ -7,5 +7,7 @@ export interface RawServer { } export const servers = { - list: rpcMethodTypeContainer>(), + configured: { + list: rpcMethodTypeContainer>(), + }, }; diff --git a/mathesar_ui/src/stores/databases.ts b/mathesar_ui/src/stores/databases.ts index 574262215a..bcd58a8434 100644 --- a/mathesar_ui/src/stores/databases.ts +++ b/mathesar_ui/src/stores/databases.ts @@ -74,9 +74,9 @@ class DatabasesStore { } async connectExistingDatabase( - props: Parameters[0], + props: Parameters[0], ) { - const { database, server } = await api.database_setup + const { database, server } = await api.databases.setup .connect_existing(props) .run(); const connectedDatabase = new Database({ @@ -88,9 +88,9 @@ class DatabasesStore { } async createNewDatabase( - props: Parameters[0], + props: Parameters[0], ) { - const { database, server } = await api.database_setup + const { database, server } = await api.databases.setup .create_new(props) .run(); const connectedDatabase = new Database({ diff --git a/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte b/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte index 276a3cf05c..618a66d1a0 100644 --- a/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte +++ b/mathesar_ui/src/systems/databases/create-database/InstallationSchemaSelector.svelte @@ -4,7 +4,7 @@ import { type SampleDataSchemaIdentifier, sampleDataOptions, - } from '@mathesar/api/rpc/database_setup'; + } from '@mathesar/api/rpc/databases'; import type { RequiredField } from '@mathesar/components/form/field'; import { CheckboxGroup } from '@mathesar-component-library'; diff --git a/mathesar_ui/src/systems/databases/create-database/createDatabaseUtils.ts b/mathesar_ui/src/systems/databases/create-database/createDatabaseUtils.ts index 8e38255a37..94be6c5774 100644 --- a/mathesar_ui/src/systems/databases/create-database/createDatabaseUtils.ts +++ b/mathesar_ui/src/systems/databases/create-database/createDatabaseUtils.ts @@ -1,7 +1,7 @@ import { type SampleDataSchemaIdentifier, sampleDataOptions, -} from '@mathesar/api/rpc/database_setup'; +} from '@mathesar/api/rpc/databases'; export type InstallationSchema = SampleDataSchemaIdentifier | 'internal'; From 83413ce6c282607615030127631026f5487e675a Mon Sep 17 00:00:00 2001 From: Sean Colsen Date: Thu, 29 Aug 2024 14:46:51 -0400 Subject: [PATCH 0786/1141] Hard-code abstract types response in client --- mathesar_ui/src/AppTypes.ts | 7 - .../AbstractTypeSelector.svelte | 4 +- .../cell-fabric/data-types/utils.ts | 6 +- .../pages/exploration/ExplorationPage.svelte | 4 +- .../preview/ImportPreviewContent.svelte | 4 +- .../src/pages/record/RecordPage.svelte | 4 +- .../src/pages/record/TableWidget.svelte | 3 +- mathesar_ui/src/pages/table/TablePage.svelte | 3 +- .../src/routes/DataExplorerRoute.svelte | 4 +- .../abstract-types/abstractTypeCategories.ts | 10 -- .../stores/abstract-types/abstractTypesMap.ts | 89 ++++++++++++ .../src/stores/abstract-types/index.ts | 2 +- .../src/stores/abstract-types/store.ts | 127 ------------------ .../src/stores/abstract-types/types.ts | 6 - .../RecordSelectorWindow.svelte | 4 +- .../constraints/NewFkConstraint.svelte | 4 +- .../new-column-cell/ColumnTypeSelector.svelte | 12 +- .../FkRecordSummaryConfig.svelte | 3 +- mathesar_ui/src/utils/columnUtils.ts | 9 +- mathesar_ui/src/utils/preloadData.ts | 2 - 20 files changed, 114 insertions(+), 193 deletions(-) create mode 100644 mathesar_ui/src/stores/abstract-types/abstractTypesMap.ts delete mode 100644 mathesar_ui/src/stores/abstract-types/store.ts diff --git a/mathesar_ui/src/AppTypes.ts b/mathesar_ui/src/AppTypes.ts index 5a12d255e0..5fdce9a6d8 100644 --- a/mathesar_ui/src/AppTypes.ts +++ b/mathesar_ui/src/AppTypes.ts @@ -15,10 +15,3 @@ export interface FilterConfiguration { }; }[]; } - -export interface AbstractTypeResponse { - name: string; - identifier: string; - db_types: DbType[]; - filters?: FilterConfiguration; -} diff --git a/mathesar_ui/src/components/abstract-type-control/AbstractTypeSelector.svelte b/mathesar_ui/src/components/abstract-type-control/AbstractTypeSelector.svelte index 1bd153be53..a71c0932d8 100644 --- a/mathesar_ui/src/components/abstract-type-control/AbstractTypeSelector.svelte +++ b/mathesar_ui/src/components/abstract-type-control/AbstractTypeSelector.svelte @@ -4,7 +4,7 @@ import NameWithIcon from '@mathesar/components/NameWithIcon.svelte'; import { - currentDbAbstractTypes, + abstractTypesMap, getAllowedAbstractTypesForDbTypeAndItsTargetTypes, } from '@mathesar/stores/abstract-types'; import type { AbstractType } from '@mathesar/stores/abstract-types/types'; @@ -35,7 +35,7 @@ // column.valid_target_types ?? [], [], - $currentDbAbstractTypes.data, + abstractTypesMap, ).filter((item) => !['jsonlist', 'map'].includes(item.identifier)); function selectAbstractType( diff --git a/mathesar_ui/src/components/cell-fabric/data-types/utils.ts b/mathesar_ui/src/components/cell-fabric/data-types/utils.ts index 0266834589..944edc35cc 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/utils.ts +++ b/mathesar_ui/src/components/cell-fabric/data-types/utils.ts @@ -1,8 +1,6 @@ -import { get } from 'svelte/store'; - import type { DbType } from '@mathesar/AppTypes'; import { - currentDbAbstractTypes, + abstractTypesMap, getAbstractTypeForDbType, } from '@mathesar/stores/abstract-types'; import type { CellInfo } from '@mathesar/stores/abstract-types/types'; @@ -10,7 +8,7 @@ import type { CellInfo } from '@mathesar/stores/abstract-types/types'; export function getCellInfo(dbType: DbType): CellInfo | undefined { const abstractTypeOfColumn = getAbstractTypeForDbType( dbType, - get(currentDbAbstractTypes)?.data, + abstractTypesMap, ); return abstractTypeOfColumn?.cellInfo; } diff --git a/mathesar_ui/src/pages/exploration/ExplorationPage.svelte b/mathesar_ui/src/pages/exploration/ExplorationPage.svelte index fd103a1cee..501472fdd4 100644 --- a/mathesar_ui/src/pages/exploration/ExplorationPage.svelte +++ b/mathesar_ui/src/pages/exploration/ExplorationPage.svelte @@ -7,7 +7,7 @@ import LayoutWithHeader from '@mathesar/layouts/LayoutWithHeader.svelte'; import type { Database } from '@mathesar/models/Database'; import { getSchemaPageUrl } from '@mathesar/routes/urls'; - import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { abstractTypesMap } from '@mathesar/stores/abstract-types'; import type { AbstractTypesMap } from '@mathesar/stores/abstract-types/types'; import { ExplorationResult, @@ -42,7 +42,7 @@ let context: 'shared-consumer-page' | 'page' = 'page'; $: context = shareConsumer ? 'shared-consumer-page' : 'page'; - $: createQueryRunner(query, $currentDbAbstractTypes.data); + $: createQueryRunner(query, abstractTypesMap); function gotoSchemaPage() { router.goto(getSchemaPageUrl(database.id, schema.oid)); diff --git a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte index cab3b75235..4b3ea04dd5 100644 --- a/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte +++ b/mathesar_ui/src/pages/import/preview/ImportPreviewContent.svelte @@ -24,7 +24,7 @@ getSchemaPageUrl, getTablePageUrl, } from '@mathesar/routes/urls'; - import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { abstractTypesMap } from '@mathesar/stores/abstract-types'; import AsyncStore from '@mathesar/stores/AsyncStore'; import { currentDatabase } from '@mathesar/stores/databases'; import { currentTables } from '@mathesar/stores/tables'; @@ -103,7 +103,7 @@ $: records = $previewRequest.resolvedValue?.records ?? getSkeletonRecords(); $: formInputsAreDisabled = !$previewRequest.isOk; $: canProceed = $previewRequest.isOk && $form.canSubmit; - $: processedColumns = processColumns(columns, $currentDbAbstractTypes.data); + $: processedColumns = processColumns(columns, abstractTypesMap); async function init() { const columnsResponse = await columnsFetch.run({ diff --git a/mathesar_ui/src/pages/record/RecordPage.svelte b/mathesar_ui/src/pages/record/RecordPage.svelte index 073447100a..8770fc0244 100644 --- a/mathesar_ui/src/pages/record/RecordPage.svelte +++ b/mathesar_ui/src/pages/record/RecordPage.svelte @@ -2,7 +2,7 @@ import type { Table } from '@mathesar/api/rpc/tables'; import LayoutWithHeader from '@mathesar/layouts/LayoutWithHeader.svelte'; import { makeSimplePageTitle } from '@mathesar/pages/pageTitleUtils'; - import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { abstractTypesMap } from '@mathesar/stores/abstract-types'; import { currentDatabase } from '@mathesar/stores/databases'; import { TableStructure } from '@mathesar/stores/table-data'; import { displayRecordSummaryAsPlainText } from '@mathesar/stores/table-data/record-summaries/recordSummaryUtils'; @@ -18,7 +18,7 @@ $: tableStructure = new TableStructure({ database: $currentDatabase, table, - abstractTypesMap: $currentDbAbstractTypes.data, + abstractTypesMap, }); $: tableStructureIsLoading = tableStructure.isLoading; $: recordStoreFetchRequest = record.fetchRequest; diff --git a/mathesar_ui/src/pages/record/TableWidget.svelte b/mathesar_ui/src/pages/record/TableWidget.svelte index 9388c362ab..664732dcbd 100644 --- a/mathesar_ui/src/pages/record/TableWidget.svelte +++ b/mathesar_ui/src/pages/record/TableWidget.svelte @@ -2,7 +2,7 @@ import type { Column } from '@mathesar/api/rpc/columns'; import type { Table } from '@mathesar/api/rpc/tables'; import TableName from '@mathesar/components/TableName.svelte'; - import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { abstractTypesMap } from '@mathesar/stores/abstract-types'; import { currentDatabase } from '@mathesar/stores/databases'; import { Meta, @@ -26,7 +26,6 @@ export let table: Table; export let fkColumn: Pick; - $: abstractTypesMap = $currentDbAbstractTypes.data; $: tabularData = new TabularData({ database: $currentDatabase, table, diff --git a/mathesar_ui/src/pages/table/TablePage.svelte b/mathesar_ui/src/pages/table/TablePage.svelte index 865735b000..a5502a0b6f 100644 --- a/mathesar_ui/src/pages/table/TablePage.svelte +++ b/mathesar_ui/src/pages/table/TablePage.svelte @@ -6,7 +6,7 @@ import { focusActiveCell } from '@mathesar/components/sheet/utils'; import LayoutWithHeader from '@mathesar/layouts/LayoutWithHeader.svelte'; import { makeSimplePageTitle } from '@mathesar/pages/pageTitleUtils'; - import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { abstractTypesMap } from '@mathesar/stores/abstract-types'; import { currentDatabase } from '@mathesar/stores/databases'; import { Meta, @@ -33,7 +33,6 @@ let sheetElement: HTMLElement; - $: abstractTypesMap = $currentDbAbstractTypes.data; $: ({ query } = $router); $: meta = Meta.fromSerialization(query[metaSerializationQueryKey] ?? ''); $: tabularData = new TabularData({ diff --git a/mathesar_ui/src/routes/DataExplorerRoute.svelte b/mathesar_ui/src/routes/DataExplorerRoute.svelte index d1bc8bcc33..bdbcf4c53c 100644 --- a/mathesar_ui/src/routes/DataExplorerRoute.svelte +++ b/mathesar_ui/src/routes/DataExplorerRoute.svelte @@ -15,7 +15,7 @@ getDataExplorerPageUrl, getExplorationEditorPageUrl, } from '@mathesar/routes/urls'; - import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { abstractTypesMap } from '@mathesar/stores/abstract-types'; import type { UnsavedQueryInstance } from '@mathesar/stores/queries'; import { getQuery } from '@mathesar/stores/queries'; import { @@ -38,7 +38,7 @@ queryManager?.destroy(); queryManager = new QueryManager({ query: new QueryModel(queryInstance), - abstractTypeMap: $currentDbAbstractTypes.data, + abstractTypeMap: abstractTypesMap, onSave: async (instance) => { try { const url = getExplorationEditorPageUrl( diff --git a/mathesar_ui/src/stores/abstract-types/abstractTypeCategories.ts b/mathesar_ui/src/stores/abstract-types/abstractTypeCategories.ts index 04f492de3b..68817bf3c3 100644 --- a/mathesar_ui/src/stores/abstract-types/abstractTypeCategories.ts +++ b/mathesar_ui/src/stores/abstract-types/abstractTypeCategories.ts @@ -63,16 +63,6 @@ export function constructAbstractTypeMapFromResponse( > & { factory: AbstractTypeConfigurationFactory })[] = []; abstractTypesResponse.forEach((entry) => { - if (entry.identifier === 'other') { - /** - * Ignore "Other" type sent in response. - * This is a failsafe to ensure that the frontend does not - * break when the "Other" type does not contain db_types which - * are either the type or valid_target_type for any column. - */ - return; - } - const partialAbstractType = { identifier: entry.identifier, name: entry.name, diff --git a/mathesar_ui/src/stores/abstract-types/abstractTypesMap.ts b/mathesar_ui/src/stores/abstract-types/abstractTypesMap.ts new file mode 100644 index 0000000000..a38f74810d --- /dev/null +++ b/mathesar_ui/src/stores/abstract-types/abstractTypesMap.ts @@ -0,0 +1,89 @@ +import { constructAbstractTypeMapFromResponse } from './abstractTypeCategories'; +import type { AbstractTypeResponse } from './types'; + +/** + * This is called "Response" because we originally designed the types + * architecture to be client-server oriented. But later we decided to hard-code + * this data in the front end. + */ +const typesResponse: AbstractTypeResponse[] = [ + { + identifier: 'boolean', + name: 'Boolean', + db_types: ['boolean'], + }, + { + identifier: 'date', + name: 'Date', + db_types: ['date'], + }, + { + identifier: 'time', + name: 'Time', + db_types: ['time with time zone', 'time without time zone'], + }, + { + identifier: 'datetime', + name: 'Date & Time', + db_types: ['timestamp with time zone', 'timestamp without time zone'], + }, + { + identifier: 'duration', + name: 'Duration', + db_types: ['interval'], + }, + { + identifier: 'email', + name: 'Email', + db_types: ['mathesar_types.email'], + }, + { + identifier: 'money', + name: 'Money', + db_types: [ + 'mathesar_types.multicurrency_money', + 'mathesar_types.mathesar_money', + 'money', + ], + }, + { + identifier: 'number', + name: 'Number', + db_types: [ + 'double precision', + 'real', + 'smallint', + 'bigint', + 'integer', + 'numeric', + ], + }, + { + identifier: 'text', + name: 'Text', + db_types: ['text', 'character', '"char"', 'name', 'character varying'], + }, + { + identifier: 'uri', + name: 'URI', + db_types: ['mathesar_types.uri'], + }, + { + identifier: 'jsonlist', + name: 'JSON List', + db_types: ['mathesar_types.mathesar_json_array'], + }, + { + identifier: 'map', + name: 'Map', + db_types: ['mathesar_types.mathesar_json_object'], + }, + { + identifier: 'array', + name: 'Array', + db_types: ['_array'], + }, +]; + +export const abstractTypesMap = + constructAbstractTypeMapFromResponse(typesResponse); diff --git a/mathesar_ui/src/stores/abstract-types/index.ts b/mathesar_ui/src/stores/abstract-types/index.ts index 36478a84f1..4cc26e1b2b 100644 --- a/mathesar_ui/src/stores/abstract-types/index.ts +++ b/mathesar_ui/src/stores/abstract-types/index.ts @@ -5,7 +5,7 @@ export { getDefaultDbTypeOfAbstractType, } from './abstractTypeCategories'; export { getAbstractTypeForDbType } from './utils'; -export { currentDbAbstractTypes, refetchTypesForDb } from './store'; +export { abstractTypesMap } from './abstractTypesMap'; export { filterDefinitionMap, getEqualityFiltersForAbstractType, diff --git a/mathesar_ui/src/stores/abstract-types/store.ts b/mathesar_ui/src/stores/abstract-types/store.ts deleted file mode 100644 index 3a58807aa8..0000000000 --- a/mathesar_ui/src/stores/abstract-types/store.ts +++ /dev/null @@ -1,127 +0,0 @@ -import type { Readable, Unsubscriber, Writable } from 'svelte/store'; -import { derived, get, writable } from 'svelte/store'; - -import { States, getAPI } from '@mathesar/api/rest/utils/requestUtils'; -import type { Database } from '@mathesar/models/Database'; -import { databasesStore } from '@mathesar/stores/databases'; -import { preloadCommonData } from '@mathesar/utils/preloadData'; -import type { CancellablePromise } from '@mathesar-component-library'; - -import { constructAbstractTypeMapFromResponse } from './abstractTypeCategories'; -import type { - AbstractTypeResponse, - AbstractTypesMap, - AbstractTypesSubstance, -} from './types'; - -const commonData = preloadCommonData(); -const { currentDatabase } = databasesStore; - -const databasesToAbstractTypesStoreMap: Map< - Database['id'], - Writable -> = new Map(); -const abstractTypesRequestMap: Map< - Database['id'], - CancellablePromise -> = new Map(); - -export async function refetchTypesForDb( - databaseId: Database['id'], -): Promise { - const store = databasesToAbstractTypesStoreMap.get(databaseId); - if (!store) { - console.error(`DB Types store for db: ${databaseId} not found.`); - return undefined; - } - - try { - store.update((currentData) => ({ - ...currentData, - state: States.Loading, - })); - - abstractTypesRequestMap.get(databaseId)?.cancel(); - - const typesRequest = getAPI( - `/api/ui/v0/connections/${databaseId}/types/`, - ); - abstractTypesRequestMap.set(databaseId, typesRequest); - const response = await typesRequest; - - const abstractTypesMap = constructAbstractTypeMapFromResponse(response); - - store.update((currentData) => ({ - ...currentData, - state: States.Done, - data: abstractTypesMap, - })); - - return abstractTypesMap; - } catch (err) { - store.update((currentData) => ({ - ...currentData, - state: States.Error, - error: err instanceof Error ? err.message : 'Error in fetching schemas', - })); - return undefined; - } -} - -/** - * TODO: Find a better way to preload data instead of using this variable. - * Each store needs to be able to use the common data and preloading is - * specific to each of them and not common. - */ -let preload = true; - -function getTypesForDatabase( - database: Database, -): Writable { - let store = databasesToAbstractTypesStoreMap.get(database.id); - if (!store) { - store = writable({ - state: States.Loading, - data: new Map(), - }); - databasesToAbstractTypesStoreMap.set(database.id, store); - - if (preload && commonData.current_database === database.id) { - store.update((currentData) => ({ - ...currentData, - state: States.Done, - // @ts-ignore: https://github.com/centerofci/mathesar/issues/1055 - data: constructAbstractTypeMapFromResponse(commonData.abstract_types), - })); - } else { - void refetchTypesForDb(database.id); - } - preload = false; - } else if (get(store).error) { - void refetchTypesForDb(database.id); - } - return store; -} - -export const currentDbAbstractTypes: Readable = derived( - currentDatabase, - ($currentDatabase, set) => { - let unsubscribe: Unsubscriber; - - if (!$currentDatabase) { - set({ - state: States.Done, - data: new Map(), - }); - } else { - const store = getTypesForDatabase($currentDatabase); - unsubscribe = store.subscribe((typesData) => { - set(typesData); - }); - } - - return () => { - unsubscribe?.(); - }; - }, -); diff --git a/mathesar_ui/src/stores/abstract-types/types.ts b/mathesar_ui/src/stores/abstract-types/types.ts index 0d96333b22..3c00223dff 100644 --- a/mathesar_ui/src/stores/abstract-types/types.ts +++ b/mathesar_ui/src/stores/abstract-types/types.ts @@ -92,12 +92,6 @@ export type AbstractTypeConfigurationFactory = ( map: AbstractTypesMap, ) => AbstractTypeConfiguration; -export interface AbstractTypesSubstance { - state: States; - data: AbstractTypesMap; - error?: string; -} - /** * These filter ids represent the filter functions used for the _old_ filtering * system (circa 2023). The UI is still designed around these filter functions diff --git a/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte b/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte index cd98a1e086..ca84d00e6c 100644 --- a/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte +++ b/mathesar_ui/src/systems/record-selector/RecordSelectorWindow.svelte @@ -3,7 +3,7 @@ import { RichText } from '@mathesar/components/rich-text'; import TableName from '@mathesar/components/TableName.svelte'; - import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { abstractTypesMap } from '@mathesar/stores/abstract-types'; import { currentDatabase } from '@mathesar/stores/databases'; import { Meta, TabularData } from '@mathesar/stores/table-data'; import { currentTablesMap } from '@mathesar/stores/tables'; @@ -36,7 +36,7 @@ $tableId && table ? new TabularData({ database: $currentDatabase, - abstractTypesMap: $currentDbAbstractTypes.data, + abstractTypesMap, meta: new Meta({ pagination: new Pagination({ size: 10 }) }), hasEnhancedPrimaryKeyCell: false, table, diff --git a/mathesar_ui/src/systems/table-view/constraints/NewFkConstraint.svelte b/mathesar_ui/src/systems/table-view/constraints/NewFkConstraint.svelte index 9f478ce067..3e00bd9762 100644 --- a/mathesar_ui/src/systems/table-view/constraints/NewFkConstraint.svelte +++ b/mathesar_ui/src/systems/table-view/constraints/NewFkConstraint.svelte @@ -15,7 +15,7 @@ import SelectProcessedColumn from '@mathesar/components/SelectProcessedColumn.svelte'; import SelectTable from '@mathesar/components/SelectTable.svelte'; import TableName from '@mathesar/components/TableName.svelte'; - import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { abstractTypesMap } from '@mathesar/stores/abstract-types'; import { currentDatabase } from '@mathesar/stores/databases'; import { type ProcessedColumn, @@ -81,7 +81,7 @@ ? new TableStructure({ database: $currentDatabase, table: $targetTable, - abstractTypesMap: $currentDbAbstractTypes.data, + abstractTypesMap, }) : undefined; $: targetTableStructureIsLoading = ensureReadable( diff --git a/mathesar_ui/src/systems/table-view/header/new-column-cell/ColumnTypeSelector.svelte b/mathesar_ui/src/systems/table-view/header/new-column-cell/ColumnTypeSelector.svelte index 5fb14722fe..c99c0cc021 100644 --- a/mathesar_ui/src/systems/table-view/header/new-column-cell/ColumnTypeSelector.svelte +++ b/mathesar_ui/src/systems/table-view/header/new-column-cell/ColumnTypeSelector.svelte @@ -1,7 +1,7 @@ import type { Table } from '@mathesar/api/rpc/tables'; - import { currentDbAbstractTypes } from '@mathesar/stores/abstract-types'; + import { abstractTypesMap } from '@mathesar/stores/abstract-types'; import { currentDatabase } from '@mathesar/stores/databases'; import { Meta, TabularData } from '@mathesar/stores/table-data'; import RecordSummaryConfig from '@mathesar/systems/table-view/table-inspector/record-summary/RecordSummaryConfig.svelte'; @@ -8,7 +8,6 @@ export let table: Table; - $: abstractTypesMap = $currentDbAbstractTypes.data; $: tabularData = new TabularData({ database: $currentDatabase, table, diff --git a/mathesar_ui/src/utils/columnUtils.ts b/mathesar_ui/src/utils/columnUtils.ts index 00fd0eb8df..f51843ee21 100644 --- a/mathesar_ui/src/utils/columnUtils.ts +++ b/mathesar_ui/src/utils/columnUtils.ts @@ -1,11 +1,9 @@ -import { get } from 'svelte/store'; - import type { Table } from '@mathesar/api/rpc/tables'; import type { DisplayColumn } from '@mathesar/components/column/types'; import { type ValidationFn, uniqueWith } from '@mathesar/components/form'; import { iconConstraint, iconTableLink } from '@mathesar/icons'; import { - currentDbAbstractTypes, + abstractTypesMap, getAbstractTypeForDbType, } from '@mathesar/stores/abstract-types'; import type { ProcessedColumn } from '@mathesar/stores/table-data'; @@ -25,10 +23,7 @@ export function getColumnIconProps( return iconTableLink; } - return getAbstractTypeForDbType( - _column.type, - get(currentDbAbstractTypes)?.data, - ).getIcon({ + return getAbstractTypeForDbType(_column.type, abstractTypesMap).getIcon({ dbType: _column.type, typeOptions: _column.type_options, }); diff --git a/mathesar_ui/src/utils/preloadData.ts b/mathesar_ui/src/utils/preloadData.ts index 167c0d8bd0..e9373c091a 100644 --- a/mathesar_ui/src/utils/preloadData.ts +++ b/mathesar_ui/src/utils/preloadData.ts @@ -4,7 +4,6 @@ import type { RawDatabase } from '@mathesar/api/rpc/databases'; import type { Schema } from '@mathesar/api/rpc/schemas'; import type { RawServer } from '@mathesar/api/rpc/servers'; import type { Table } from '@mathesar/api/rpc/tables'; -import type { AbstractTypeResponse } from '@mathesar/AppTypes'; export interface CommonData { databases: RawDatabase[]; @@ -21,7 +20,6 @@ export interface CommonData { user: string; }; current_schema: number | null; - abstract_types: AbstractTypeResponse[]; user: User; current_release_tag_name: string; supported_languages: Record; From f64f23b6226f11f07a62b83ed7ee7b6332642a5f Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 30 Aug 2024 17:08:29 +0530 Subject: [PATCH 0787/1141] change response structure for record summary --- db/sql/00_msar.sql | 6 +++--- db/sql/test_00_msar.sql | 36 ++++++++++++++++++------------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 2fb08a9fe8..c3f953750f 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -4126,7 +4126,7 @@ SELECT ', ' || string_agg( format( $c$summary_cte_%1$s AS ( SELECT - msar.format_data(%2$I) AS key, + msar.format_data(%2$I) AS fkey, %3$s AS summary FROM %4$I.%5$I )$c$, @@ -4153,7 +4153,7 @@ WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_cte(tab_id)) SELECT string_agg( format( $j$ - LEFT JOIN summary_cte_%1$s ON %2$I.%1$I = summary_cte_%1$s.key$j$, + LEFT JOIN summary_cte_%1$s ON %2$I.%1$I = summary_cte_%1$s.fkey$j$, conkey, cte_name ), ' ' @@ -4174,7 +4174,7 @@ SELECT 'jsonb_build_object(' || string_agg( format( $j$ %1$L, jsonb_agg( - DISTINCT jsonb_build_object('key', summary_cte_%1$s.key, 'summary', summary_cte_%1$s.summary) + DISTINCT jsonb_build_object(summary_cte_%1$s.fkey, summary_cte_%1$s.summary) ) $j$, conkey diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index aa891db25c..53b65c84bf 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4265,13 +4265,13 @@ BEGIN "grouping": null, "preview_data": { "2": [ - {"key": 1.234, "summary": "Alice Alison"}, - {"key": 2.345, "summary": "Bob Bobinson"} + {"1.234": "Alice Alison"}, + {"2.345": "Bob Bobinson"} ], "3": [ - {"key": 1, "summary": "Carol Carlson"}, - {"key": 2, "summary": "Dave Davidson"}, - {"key": 3, "summary": "Eve Evilson"} + {"1": "Carol Carlson"}, + {"2": "Dave Davidson"}, + {"3": "Eve Evilson"} ] } }$j$ || jsonb_build_object( @@ -4302,12 +4302,12 @@ BEGIN "grouping": null, "preview_data": { "2": [ - {"key": 1.234, "summary": "Alice Alison"}, - {"key": 2.345, "summary": "Bob Bobinson"} + {"1.234": "Alice Alison"}, + {"2.345": "Bob Bobinson"} ], "3": [ - {"key": 1, "summary": "Carol Carlson"}, - {"key": 2, "summary": "Dave Davidson"} + {"1": "Carol Carlson"}, + {"2": "Dave Davidson"} ] } }$j$ || jsonb_build_object( @@ -4341,11 +4341,11 @@ BEGIN }, "preview_data": { "2": [ - {"key": 1.234, "summary": "Alice Alison"} + {"1.234": "Alice Alison"} ], "3": [ - {"key": 1, "summary": "Carol Carlson"}, - {"key": 2, "summary": "Dave Davidson"} + {"1": "Carol Carlson"}, + {"2": "Dave Davidson"} ] } }$j$ || jsonb_build_object( @@ -4374,8 +4374,8 @@ BEGIN {"1": 7, "2": 2.345, "3": 1, "4": "Larry Laurelson", "5": 70, "6": "llaurelson@example.edu"} ], "preview_data": { - "2": [{"key": 2.345, "summary": "Bob Bobinson"}], - "3": [{"key": 1, "summary": "Carol Carlson"}] + "2": [{"2.345": "Bob Bobinson"}], + "3": [{"1": "Carol Carlson"}] } }$a$ ); @@ -4397,8 +4397,8 @@ BEGIN {"1": 2, "2": 2.345, "3": 2, "4": "Gabby Gabberson", "5": 85, "6": "ggabberson@example.edu"} ], "preview_data": { - "2": [{"key": 2.345, "summary": "Bob Bobinson"}], - "3": [{"key": 2, "summary": "Dave Davidson"}] + "2": [{"2.345": "Bob Bobinson"}], + "3": [{"2": "Dave Davidson"}] } }$a$ ); @@ -4416,8 +4416,8 @@ BEGIN 2 ) -> 'preview_data', $a${ - "2": [{"key": 1.234, "summary": "Alice Alison"}, {"key": 2.345, "summary": "Bob Bobinson"}], - "3": [{"key": 3, "summary": "Eve Evilson"}] + "2": [{"1.234": "Alice Alison"}, {"2.345": "Bob Bobinson"}], + "3": [{"3": "Eve Evilson"}] }$a$ ); END; From b214094a7b45185196e72b7ce044340eddf0aec8 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 30 Aug 2024 23:43:49 +0530 Subject: [PATCH 0788/1141] concat multiple json objects --- db/sql/00_msar.sql | 4 +-- db/sql/test_00_msar.sql | 60 ++++++++++++++++++++--------------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index c3f953750f..4c00e785d8 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -4173,8 +4173,8 @@ WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_cte(tab_id)) SELECT 'jsonb_build_object(' || string_agg( format( $j$ - %1$L, jsonb_agg( - DISTINCT jsonb_build_object(summary_cte_%1$s.fkey, summary_cte_%1$s.summary) + %1$L, jsonb_object_agg( + summary_cte_%1$s.fkey, summary_cte_%1$s.summary ) $j$, conkey diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 53b65c84bf..864bf1df08 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4264,15 +4264,15 @@ BEGIN ], "grouping": null, "preview_data": { - "2": [ - {"1.234": "Alice Alison"}, - {"2.345": "Bob Bobinson"} - ], - "3": [ - {"1": "Carol Carlson"}, - {"2": "Dave Davidson"}, - {"3": "Eve Evilson"} - ] + "2": { + "1.234": "Alice Alison", + "2.345": "Bob Bobinson" + }, + "3": { + "1": "Carol Carlson", + "2": "Dave Davidson", + "3": "Eve Evilson" + } } }$j$ || jsonb_build_object( 'query', concat( @@ -4301,14 +4301,14 @@ BEGIN ], "grouping": null, "preview_data": { - "2": [ - {"1.234": "Alice Alison"}, - {"2.345": "Bob Bobinson"} - ], - "3": [ - {"1": "Carol Carlson"}, - {"2": "Dave Davidson"} - ] + "2": { + "1.234": "Alice Alison", + "2.345": "Bob Bobinson" + }, + "3": { + "1": "Carol Carlson", + "2": "Dave Davidson" + } } }$j$ || jsonb_build_object( 'query', concat( @@ -4340,13 +4340,13 @@ BEGIN "groups": [{"id": 1, "count": 3, "results_eq": {"2": 1.234}, "result_indices": [0, 1]}] }, "preview_data": { - "2": [ - {"1.234": "Alice Alison"} - ], - "3": [ - {"1": "Carol Carlson"}, - {"2": "Dave Davidson"} - ] + "2": { + "1.234": "Alice Alison" + }, + "3": { + "1": "Carol Carlson", + "2": "Dave Davidson" + } } }$j$ || jsonb_build_object( 'query', concat( @@ -4374,8 +4374,8 @@ BEGIN {"1": 7, "2": 2.345, "3": 1, "4": "Larry Laurelson", "5": 70, "6": "llaurelson@example.edu"} ], "preview_data": { - "2": [{"2.345": "Bob Bobinson"}], - "3": [{"1": "Carol Carlson"}] + "2": {"2.345": "Bob Bobinson"}, + "3": {"1": "Carol Carlson"} } }$a$ ); @@ -4397,8 +4397,8 @@ BEGIN {"1": 2, "2": 2.345, "3": 2, "4": "Gabby Gabberson", "5": 85, "6": "ggabberson@example.edu"} ], "preview_data": { - "2": [{"2.345": "Bob Bobinson"}], - "3": [{"2": "Dave Davidson"}] + "2": {"2.345": "Bob Bobinson"}, + "3": {"2": "Dave Davidson"} } }$a$ ); @@ -4416,8 +4416,8 @@ BEGIN 2 ) -> 'preview_data', $a${ - "2": [{"1.234": "Alice Alison"}, {"2.345": "Bob Bobinson"}], - "3": [{"3": "Eve Evilson"}] + "2": {"1.234": "Alice Alison", "2.345": "Bob Bobinson"}, + "3": {"3": "Eve Evilson"} }$a$ ); END; From ddf5c33a67cdef3df4d072707b16d5827d30ab43 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Sat, 31 Aug 2024 00:19:53 +0530 Subject: [PATCH 0789/1141] change python mock tests to conform to new record summary response --- docs/docs/api/rpc.md | 1 - mathesar/rpc/records.py | 22 ++++++---------------- mathesar/tests/rpc/test_records.py | 20 ++++++++++---------- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 2a9567210a..fb6ec1742a 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -217,7 +217,6 @@ To use an RPC function: - Filter - FilterAttnum - FilterLiteral - - PreviewEntry - Grouping - Group - GroupingResponse diff --git a/mathesar/rpc/records.py b/mathesar/rpc/records.py index 2239b0aab3..4c62ea48d6 100644 --- a/mathesar/rpc/records.py +++ b/mathesar/rpc/records.py @@ -127,18 +127,6 @@ class GroupingResponse(TypedDict): groups: list[Group] -class PreviewEntry(TypedDict): - """ - Preview entry object. Maps a foreign key to a text summary. - - Attributes: - key: The foreign key value. - summary: The summary of the referenced record. - """ - key: Any - summary: str - - class RecordList(TypedDict): """ Records from a table, along with some meta data @@ -152,12 +140,13 @@ class RecordList(TypedDict): count: The total number of records in the table. results: An array of record objects. grouping: Information for displaying grouped records. - preview_data: Information for previewing foreign key values. + preview_data: Information for previewing foreign key values, + provides a map of foreign key to a text summary. """ count: int results: list[dict] grouping: GroupingResponse - preview_data: dict[str, list[PreviewEntry]] + preview_data: dict[str, dict[Any, str]] @classmethod def from_dict(cls, d): @@ -181,10 +170,11 @@ class RecordAdded(TypedDict): Attributes: results: An array of a single record objects (the one added). - preview_data: Information for previewing foreign key values. + preview_data: Information for previewing foreign key values, + provides a map of foreign key to a text summary. """ results: list[dict] - preview_data: dict[str, list[PreviewEntry]] + preview_data: dict[str, dict[Any, str]] @classmethod def from_dict(cls, d): diff --git a/mathesar/tests/rpc/test_records.py b/mathesar/tests/rpc/test_records.py index 7a9099a4df..1b90c7366e 100644 --- a/mathesar/tests/rpc/test_records.py +++ b/mathesar/tests/rpc/test_records.py @@ -50,7 +50,7 @@ def mock_list_records( {"id": 3, "count": 8, "results_eq": {"1": "lsfj", "2": 3422}} ] }, - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}} } monkeypatch.setattr(records, 'connect', mock_connect) @@ -64,7 +64,7 @@ def mock_list_records( {"id": 3, "count": 8, "results_eq": {"1": "lsfj", "2": 3422}} ] }, - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}}, "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', } actual_records_list = records.list_( @@ -104,7 +104,7 @@ def mock_get_record( "results": [{"1": "abcde", "2": 12345}, {"1": "fghij", "2": 67890}], "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', "grouping": None, - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}}, } monkeypatch.setattr(records, 'connect', mock_connect) @@ -113,7 +113,7 @@ def mock_get_record( "count": 1, "results": [{"1": "abcde", "2": 12345}, {"1": "fghij", "2": 67890}], "grouping": None, - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}}, "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', } actual_record = records.get( @@ -150,14 +150,14 @@ def mock_add_record( raise AssertionError('incorrect parameters passed') return { "results": [_record_def], - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}}, } monkeypatch.setattr(records, 'connect', mock_connect) monkeypatch.setattr(records.record_insert, 'add_record_to_table', mock_add_record) expect_record = { "results": [record_def], - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}}, } actual_record = records.add( record_def=record_def, table_oid=table_oid, database_id=database_id, request=request @@ -195,14 +195,14 @@ def mock_patch_record( raise AssertionError('incorrect parameters passed') return { "results": [_record_def | {"3": "another"}], - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}}, } monkeypatch.setattr(records, 'connect', mock_connect) monkeypatch.setattr(records.record_update, 'patch_record_in_table', mock_patch_record) expect_record = { "results": [record_def | {"3": "another"}], - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}}, } actual_record = records.patch( record_def=record_def, @@ -280,7 +280,7 @@ def mock_search_records( return { "count": 50123, "results": [{"1": "abcde", "2": 12345}, {"1": "fghij", "2": 67890}], - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}}, "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', } @@ -290,7 +290,7 @@ def mock_search_records( "count": 50123, "results": [{"1": "abcde", "2": 12345}, {"1": "fghij", "2": 67890}], "grouping": None, - "preview_data": {"2": [{"key": 12345, "summary": "blkjdfslkj"}]}, + "preview_data": {"2": {"12345": "blkjdfslkj"}}, "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', } actual_records_list = records.search( From 977f1965eedbfc8829cb46276a4fb3a16cc4d235 Mon Sep 17 00:00:00 2001 From: Ghislaine Guerin Date: Mon, 2 Sep 2024 09:53:53 +0200 Subject: [PATCH 0790/1141] Update Create Link Modal --- .../labeled-input/LabeledInput.scss | 2 +- .../components/message-boxes/InfoBox.svelte | 3 +- mathesar_ui/src/i18n/languages/en/dict.json | 4 +- .../link-table/LinkTableForm.svelte | 79 +++++++++++++++---- .../table-view/link-table/NewColumn.svelte | 9 ++- 5 files changed, 76 insertions(+), 21 deletions(-) diff --git a/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss b/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss index ced2535658..93d42c92ab 100644 --- a/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss +++ b/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss @@ -1,5 +1,5 @@ .labeled-input { - --spacing-y-default: 0.4em; + --spacing-y-default: 0.25em; --spacing-x-default: 0.4em; .label { diff --git a/mathesar_ui/src/components/message-boxes/InfoBox.svelte b/mathesar_ui/src/components/message-boxes/InfoBox.svelte index 2fcffa1303..c5d5500d3b 100644 --- a/mathesar_ui/src/components/message-boxes/InfoBox.svelte +++ b/mathesar_ui/src/components/message-boxes/InfoBox.svelte @@ -6,10 +6,11 @@ import MessageBox from './MessageBox.svelte'; type $$Props = ComponentProps; + export let fullWidth = true;
- +
diff --git a/mathesar_ui/src/pages/database/permissions/PrivilegesSection.svelte b/mathesar_ui/src/pages/database/permissions/PrivilegesSection.svelte new file mode 100644 index 0000000000..1996ff45fa --- /dev/null +++ b/mathesar_ui/src/pages/database/permissions/PrivilegesSection.svelte @@ -0,0 +1,126 @@ + + +
+ {#if isLoading} + + {:else if isSuccess && $roles.resolvedValue} +
+
{$_('owner')}
+
+
+
+
{$_('granted_access')}
+
+ {#each [...dbPrivilegesMap.values()] as dbPrivilegeForRole (dbPrivilegeForRole)} +
+ +
+ {/each} +
+
+ {:else} + + {/if} +
+ + + + diff --git a/mathesar_ui/src/pages/database/permissions/TransferOwnershipSection.svelte b/mathesar_ui/src/pages/database/permissions/TransferOwnershipSection.svelte new file mode 100644 index 0000000000..8bf8a565a0 --- /dev/null +++ b/mathesar_ui/src/pages/database/permissions/TransferOwnershipSection.svelte @@ -0,0 +1,29 @@ + + + diff --git a/mathesar_ui/src/stores/AsyncRpcApiStore.ts b/mathesar_ui/src/stores/AsyncRpcApiStore.ts index 741e6ac803..6375e0b7c8 100644 --- a/mathesar_ui/src/stores/AsyncRpcApiStore.ts +++ b/mathesar_ui/src/stores/AsyncRpcApiStore.ts @@ -7,7 +7,7 @@ import { batchSend, } from '@mathesar/packages/json-rpc-client-builder'; -import AsyncStore, { AsyncStoreValue } from './AsyncStore'; +import AsyncStore, { type AsyncStoreValue } from './AsyncStore'; type BatchRunner = { send: RpcRequest; @@ -16,7 +16,7 @@ type BatchRunner = { getValue: () => AsyncStoreValue; }; -export default class AsyncRpcApiStore extends AsyncStore< +export default class AsyncRpcApiStore extends AsyncStore< Props, U > { diff --git a/mathesar_ui/src/utils/typeUtils.ts b/mathesar_ui/src/utils/typeUtils.ts index 5bf3df7aae..88e459d4c9 100644 --- a/mathesar_ui/src/utils/typeUtils.ts +++ b/mathesar_ui/src/utils/typeUtils.ts @@ -1,6 +1,6 @@ import type { Readable, Writable } from 'svelte/store'; -type ChangeWritableToReadable = T extends Writable +export type ChangeWritableToReadable = T extends Writable ? Readable : T; From dea235b43851bb6db4c6803a36bf4b27166a49e1 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 2 Sep 2024 19:12:41 +0530 Subject: [PATCH 0793/1141] fix return type for fkey from Any->str --- mathesar/rpc/records.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mathesar/rpc/records.py b/mathesar/rpc/records.py index 4c62ea48d6..95ea27b861 100644 --- a/mathesar/rpc/records.py +++ b/mathesar/rpc/records.py @@ -146,7 +146,7 @@ class RecordList(TypedDict): count: int results: list[dict] grouping: GroupingResponse - preview_data: dict[str, dict[Any, str]] + preview_data: dict[str, dict[str, str]] @classmethod def from_dict(cls, d): @@ -174,7 +174,7 @@ class RecordAdded(TypedDict): provides a map of foreign key to a text summary. """ results: list[dict] - preview_data: dict[str, dict[Any, str]] + preview_data: dict[str, dict[str, str]] @classmethod def from_dict(cls, d): From c6c2057428e8f8952b82d9993621468b35cb7149 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Mon, 2 Sep 2024 19:24:21 +0530 Subject: [PATCH 0794/1141] remove playwright dependency --- requirements-dev.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1c69cf20dc..0716dcc75b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,5 @@ pytest-django==4.2.0 pytest-env==0.6.2 pytest-cov==3.0.0 pytest-xdist[psutil]==2.5.0 -playwright==1.18.1 -pytest-playwright==0.2.3 mkdocs-material==8.5.11 mkdocs-redirects==1.2.0 From 138d62bf0a077a9d4424d282b66e57e4e6eeaf05 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Mon, 2 Sep 2024 23:15:34 +0800 Subject: [PATCH 0795/1141] use fkey map, hide column choosing inside function --- db/sql/00_msar.sql | 69 ++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index dffbf8dd8c..a75265cca4 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -762,6 +762,30 @@ SELECT EXISTS ( $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION msar.get_fkey_map_table(tab_id oid) + RETURNS TABLE (target_oid oid, conkey smallint, confkey smallint) +AS $$/* +Generate a table mapping foreign key values from refererrer to referant tables. + +Given an input table (identified by OID), we return a table with each row representing a foreign key +constraint on that table. We return only single-column foreign keys, and only one per foreign key +column. + +Args: + tab_id: The OID of the table containing the foreign key columns to map. +*/ +SELECT DISTINCT ON (conkey) pgc.confrelid AS target_oid, x.conkey AS conkey, y.confkey AS confkey +FROM pg_constraint pgc, LATERAL unnest(conkey) x(conkey), LATERAL unnest(confkey) y(confkey) +WHERE + pgc.conrelid = tab_id + AND pgc.contype='f' + AND cardinality(pgc.confkey) = 1 + AND has_column_privilege(tab_id, x.conkey, 'SELECT') + AND has_column_privilege(pgc.confrelid, y.confkey, 'SELECT') +ORDER BY conkey, target_oid, confkey; +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.list_column_privileges_for_current_role(tab_id regclass, attnum smallint) RETURNS jsonb AS $$/* Return a JSONB array of all privileges current_user holds on the passed table. @@ -3859,20 +3883,23 @@ $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.move_columns_between_tables( +msar.move_columns_to_referenced_table( source_tab_id regclass, target_tab_id regclass, - move_col_ids smallint[], - source_join_col_id smallint, - target_join_col_id smallint + move_col_ids smallint[] ) RETURNS void AS $$ DECLARE + source_join_col_id smallint; + target_join_col_id smallint; preexisting_col_expr CONSTANT text := msar.build_all_columns_expr(target_tab_id); move_col_expr CONSTANT text := msar.build_columns_expr(source_tab_id, move_col_ids); move_col_defs CONSTANT jsonb := msar.get_extracted_col_def_jsonb(source_tab_id, move_col_ids); move_con_defs CONSTANT jsonb := msar.get_extracted_con_def_jsonb(source_tab_id, move_col_ids); added_col_ids smallint[]; BEGIN + SELECT conkey, confkey INTO source_join_col_id, target_join_col_id + FROM msar.get_fkey_map_table(source_tab_id) + WHERE target_oid = target_tab_id; IF move_col_ids @> ARRAY[source_join_col_id] THEN RAISE EXCEPTION 'The joining column cannot be moved.'; END IF; @@ -3888,7 +3915,7 @@ BEGIN UPDATE %6$I.%7$I SET (%9$s) = ( SELECT %9$s FROM row_numbered_cte - WHERE row_numbered_cte.%8$s=%6$I.%7$I.%8$I + WHERE row_numbered_cte.%8$I=%6$I.%7$I.%8$I AND __msar_row_number = 1 ) RETURNING * @@ -3919,10 +3946,10 @@ BEGIN ), msar.build_source_update_move_cols_equal_expr(source_tab_id, move_col_ids, 'insert_cte') ); + PERFORM msar.drop_columns(source_tab_id, variadic move_col_ids); END; $$ LANGUAGE plpgsql; --- mathesar=# with join_cte AS (SELECT DISTINCT "Books"."Favorite Number" AS L1, "Authors".* FROM "Books" JOIN "Authors" ON "Books"."Author"="Authors".id) SELECT *, row_number() OVER (PARTITION BY id ORDER BY l1) FROM join_cte; ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- @@ -4326,30 +4353,6 @@ LIMIT 1; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION msar.get_fkey_map_cte(tab_id oid) - RETURNS TABLE (target_oid oid, conkey smallint, confkey smallint) -AS $$/* -Generate a table mapping foreign key values from refererrer to referant tables. - -Given an input table (identified by OID), we return a table with each row representing a foreign key -constraint on that table. We return only single-column foreign keys, and only one per foreign key -column. - -Args: - tab_id: The OID of the table containing the foreign key columns to map. -*/ -SELECT DISTINCT ON (conkey) pgc.confrelid AS target_oid, x.conkey AS conkey, y.confkey AS confkey -FROM pg_constraint pgc, LATERAL unnest(conkey) x(conkey), LATERAL unnest(confkey) y(confkey) -WHERE - pgc.conrelid = tab_id - AND pgc.contype='f' - AND cardinality(pgc.confkey) = 1 - AND has_column_privilege(tab_id, x.conkey, 'SELECT') - AND has_column_privilege(pgc.confrelid, y.confkey, 'SELECT') -ORDER BY conkey, target_oid, confkey; -$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; - - CREATE OR REPLACE FUNCTION msar.build_summary_expr(tab_id oid) RETURNS TEXT AS $$/* Given a table, return an SQL expression that will build a summary for each row of the table. @@ -4371,7 +4374,7 @@ This summary amounts to just the first string-like column value for that linked Args: tab_id: The table for whose fkey values' linked records we'll get summaries. */ -WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_cte(tab_id)) +WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_table(tab_id)) SELECT ', ' || string_agg( format( $c$summary_cte_%1$s AS ( @@ -4399,7 +4402,7 @@ Args: tab_oid: The table defining the columns of the main CTE. cte_name: The name of the main CTE we'll join the summary CTEs to. */ -WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_cte(tab_id)) +WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_table(tab_id)) SELECT string_agg( format( $j$ @@ -4419,7 +4422,7 @@ Build a JSON object with the results of summarizing linked records. Args: tab_oid: The OID of the table for which we're getting linked record summaries. */ -WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_cte(tab_id)) +WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_table(tab_id)) SELECT 'jsonb_build_object(' || string_agg( format( $j$ From e1d76b795d40e75352dc0b6f3f7bb12a5dacd683 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 3 Sep 2024 00:59:56 +0530 Subject: [PATCH 0796/1141] wire up column extraction to python --- db/tables/operations/split.py | 19 ++++++++++++++++++- mathesar/rpc/data_modeling.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/db/tables/operations/split.py b/db/tables/operations/split.py index c9260eacc0..6ae2b274cb 100644 --- a/db/tables/operations/split.py +++ b/db/tables/operations/split.py @@ -1,4 +1,4 @@ -from db.connection import execute_msar_func_with_engine +from db.connection import execute_msar_func_with_engine, exec_msar_func def extract_columns_from_table( @@ -14,3 +14,20 @@ def extract_columns_from_table( ) extracted_table_oid, new_fkey_attnum = curr.fetchone()[0] return extracted_table_oid, old_table_oid, new_fkey_attnum + + +def split_table( + conn, + old_table_oid, + extracted_column_attnums, + extracted_table_name, + relationship_fk_column_name +): + exec_msar_func( + conn, + 'extract_columns_from_table', + old_table_oid, + extracted_column_attnums, + extracted_table_name, + relationship_fk_column_name + ) diff --git a/mathesar/rpc/data_modeling.py b/mathesar/rpc/data_modeling.py index 30c385b4ff..76f8b3bcfa 100644 --- a/mathesar/rpc/data_modeling.py +++ b/mathesar/rpc/data_modeling.py @@ -7,7 +7,7 @@ from modernrpc.auth.basic import http_basic_auth_login_required from db.links.operations import create as links_create -from db.tables.operations import infer_types +from db.tables.operations import infer_types, split from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions from mathesar.rpc.utils import connect @@ -104,3 +104,35 @@ def suggest_types(*, table_oid: int, database_id: int, **kwargs) -> dict: user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: return infer_types.infer_table_column_data_types(conn, table_oid) + + +@rpc_method(name="data_modeling.split_table") +@http_basic_auth_login_required +@handle_rpc_exceptions +def split_table( + *, + table_oid: int, + column_attnums: list, + extracted_table_name: str, + relationship_fk_column_name: str, + database_id: int, + **kwargs +) -> None: + """ + Extract columns from a table to create a new table, linked by a foreign key. + + Args: + table_oid: The OID of the table whose columns we'll extract. + column_attnums: A list of the attnums of the columns to extract. + extracted_table_name: The name of the new table to be made from the extracted columns. + relationship_fk_column_name: The name to give the new foreign key column in the remainder table (optional) + """ + user = kwargs.get(REQUEST_KEY).user + with connect(database_id, user) as conn: + split.split_table( + conn, + table_oid, + column_attnums, + extracted_table_name, + relationship_fk_column_name + ) From aa9f9a502557bda6468588d4edb8ec3b84267c81 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 3 Sep 2024 01:02:16 +0530 Subject: [PATCH 0797/1141] add docs and tests for column extraction --- docs/docs/api/rpc.md | 1 + mathesar/tests/rpc/test_data_modeling.py | 48 ++++++++++++++++++++++++ mathesar/tests/rpc/test_endpoints.py | 5 +++ 3 files changed, 54 insertions(+) diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index fb6ec1742a..0c63655df4 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -276,6 +276,7 @@ To use an RPC function: - add_foreign_key_column - add_mapping_table - suggest_types + - split_table - MappingColumn ## Responses diff --git a/mathesar/tests/rpc/test_data_modeling.py b/mathesar/tests/rpc/test_data_modeling.py index 963acf055b..a0099f83ef 100644 --- a/mathesar/tests/rpc/test_data_modeling.py +++ b/mathesar/tests/rpc/test_data_modeling.py @@ -132,3 +132,51 @@ def mock_suggest_types(conn, table_oid): database_id=_database_id, request=request, ) + + +def test_data_modeling_split_table(rf, monkeypatch): + _username = 'alice' + _password = 'pass1234' + _table_oid = 12345 + _database_id = 2 + _column_attnums = [2, 3, 4] + _extracted_table_name = 'extracted_table' + _relationship_fk_column_name = 'fk_col' + request = rf.post('/api/rpc/v0/', data={}) + request.user = User(username=_username, password=_password) + + @contextmanager + def mock_connect(database_id, user): + if database_id == _database_id and user.username == _username: + try: + yield True + finally: + pass + else: + raise AssertionError('incorrect parameters passed') + + def mock_split_table( + conn, + table_oid, + column_attnums, + extracted_table_name, + relationship_fk_column_name + ): + if ( + table_oid != _table_oid + and column_attnums != _column_attnums + and extracted_table_name != _extracted_table_name + and relationship_fk_column_name != _relationship_fk_column_name + ): + raise AssertionError('incorrect parameters passed') + + monkeypatch.setattr(data_modeling, 'connect', mock_connect) + monkeypatch.setattr(data_modeling.split, 'split_table', mock_split_table) + data_modeling.split_table( + table_oid=_table_oid, + column_attnums=_column_attnums, + extracted_table_name=_extracted_table_name, + relationship_fk_column_name=_relationship_fk_column_name, + database_id=_database_id, + request=request + ) diff --git a/mathesar/tests/rpc/test_endpoints.py b/mathesar/tests/rpc/test_endpoints.py index cede022e5c..7dc0e2cfbb 100644 --- a/mathesar/tests/rpc/test_endpoints.py +++ b/mathesar/tests/rpc/test_endpoints.py @@ -128,6 +128,11 @@ "data_modeling.suggest_types", [user_is_authenticated] ), + ( + data_modeling.split_table, + "data_modeling.split_table", + [user_is_authenticated] + ), ( databases.get, From 80d2b87d5d9b42d966b1bbbcb079aa71ee0d36f0 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Tue, 3 Sep 2024 09:07:49 +0800 Subject: [PATCH 0798/1141] add constraint copying into column mover --- db/sql/00_msar.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index a75265cca4..d4f1f47e15 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3946,6 +3946,7 @@ BEGIN ), msar.build_source_update_move_cols_equal_expr(source_tab_id, move_col_ids, 'insert_cte') ); + PERFORM msar.add_constraints(target_tab_id, move_con_defs); PERFORM msar.drop_columns(source_tab_id, variadic move_col_ids); END; $$ LANGUAGE plpgsql; From 2a065e342140ac66632085a05090f4484962fdf2 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 3 Sep 2024 15:41:59 +0530 Subject: [PATCH 0799/1141] make fk_column_name optional --- db/tables/operations/split.py | 2 +- mathesar/rpc/data_modeling.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/tables/operations/split.py b/db/tables/operations/split.py index 6ae2b274cb..aa2869073b 100644 --- a/db/tables/operations/split.py +++ b/db/tables/operations/split.py @@ -21,7 +21,7 @@ def split_table( old_table_oid, extracted_column_attnums, extracted_table_name, - relationship_fk_column_name + relationship_fk_column_name=None ): exec_msar_func( conn, diff --git a/mathesar/rpc/data_modeling.py b/mathesar/rpc/data_modeling.py index 76f8b3bcfa..06843e3f27 100644 --- a/mathesar/rpc/data_modeling.py +++ b/mathesar/rpc/data_modeling.py @@ -114,8 +114,8 @@ def split_table( table_oid: int, column_attnums: list, extracted_table_name: str, - relationship_fk_column_name: str, database_id: int, + relationship_fk_column_name: str = None, **kwargs ) -> None: """ From a2c8360f8b0e0bb56853555fa8224aaba360d915 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 4 Sep 2024 15:09:37 +0800 Subject: [PATCH 0800/1141] add some todos for edge cases --- db/sql/00_msar.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index d4f1f47e15..52465972e4 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3845,6 +3845,7 @@ CREATE OR REPLACE FUNCTION msar.build_source_update_move_cols_equal_expr( ) RETURNS text AS $$ SELECT string_agg( format( + -- TODO should be IS NOT DISTINCT FROM '%1$I.%2$I.%3$I = %4$I.%3$I', msar.get_relation_schema_name(source_tab_id), msar.get_relation_name(source_tab_id), @@ -3947,6 +3948,8 @@ BEGIN msar.build_source_update_move_cols_equal_expr(source_tab_id, move_col_ids, 'insert_cte') ); PERFORM msar.add_constraints(target_tab_id, move_con_defs); + -- TODO make sure no multi-col fkeys reference the moved columns + -- TODO just throw error if _any_ multicol constraint references the moved columns. PERFORM msar.drop_columns(source_tab_id, variadic move_col_ids); END; $$ LANGUAGE plpgsql; From 546fa0e4407cf5736ead77886348ba4b5b1b8e2d Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 4 Sep 2024 18:10:28 +0800 Subject: [PATCH 0801/1141] modify pkey finder to return false when no pkey --- db/sql/00_msar.sql | 9 +++++---- db/sql/test_00_msar.sql | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 9c69e9ffbe..472d1945d0 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -408,10 +408,11 @@ Args: rel_id: The OID of the relation. col_id: The attnum of the column in the relation. */ -BEGIN - RETURN ARRAY[col_id::smallint] <@ conkey FROM pg_constraint WHERE conrelid=rel_id and contype='p'; -END; -$$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; +SELECT EXISTS ( + SELECT 1 FROM pg_constraint WHERE + ARRAY[col_id::smallint] <@ conkey AND conrelid=rel_id AND contype='p' +); +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index c16c1c33f2..ade5855591 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -2273,6 +2273,32 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION __setup_is_pkey_col_tests() RETURNS SETOF TEXT AS $$ +BEGIN + CREATE TABLE simple_pkey (col1 text, col2 text PRIMARY KEY, col3 integer); + CREATE TABLE multi_pkey (col1 text, col2 text, col3 integer); + ALTER TABLE multi_pkey ADD PRIMARY KEY (col1, col2); + CREATE TABLE no_pkey (col1 text, col2 text, col3 integer); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_is_pkey_col() RETURNS SETOF TEXT AS $$ +BEGIN + PERFORM __setup_is_pkey_col_tests(); + RETURN NEXT is(msar.is_pkey_col('simple_pkey'::regclass::oid, 1), false); + RETURN NEXT is(msar.is_pkey_col('simple_pkey'::regclass::oid, 2), true); + RETURN NEXT is(msar.is_pkey_col('simple_pkey'::regclass::oid, 3), false); + RETURN NEXT is(msar.is_pkey_col('multi_pkey'::regclass::oid, 1), true); + RETURN NEXT is(msar.is_pkey_col('multi_pkey'::regclass::oid, 2), true); + RETURN NEXT is(msar.is_pkey_col('multi_pkey'::regclass::oid, 3), false); + RETURN NEXT is(msar.is_pkey_col('no_pkey'::regclass::oid, 1), false); + RETURN NEXT is(msar.is_pkey_col('no_pkey'::regclass::oid, 2), false); + RETURN NEXT is(msar.is_pkey_col('no_pkey'::regclass::oid, 3), false); +END; +$$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION test_create_role() RETURNS SETOF TEXT AS $$ BEGIN PERFORM msar.create_role('testuser', 'mypass1234', true); From 16f746eeba00a3d883697489b9b5201a2d022a2c Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 4 Sep 2024 18:27:43 +0800 Subject: [PATCH 0802/1141] fix small bug preventing copying unique constraints --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 3664a6295c..399d77f563 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2695,7 +2695,7 @@ SELECT jsonb_agg( 'columns', ARRAY[attname], 'deferrable', condeferrable, 'fkey_relation_id', confrelid::bigint, - 'fkey_columns', confkey, + 'fkey_columns', coalesce(confkey, ARRAY[]::smallint[]), 'fkey_update_action', confupdtype, 'fkey_delete_action', confdeltype, 'fkey_match_type', confmatchtype From addd30c8a4daa66189b741f63d39dcbd38542c42 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 4 Sep 2024 21:31:26 +0800 Subject: [PATCH 0803/1141] add initial tests for column moving --- db/sql/test_00_msar.sql | 151 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index c16c1c33f2..e86e6c012e 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4941,3 +4941,154 @@ SET ROLE test_intern2; RETURN NEXT is(msar.list_database_privileges_for_current_role(dat_id), '["CONNECT", "TEMPORARY"]'); END; $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION __setup_move_columns() RETURNS SETOF TEXT AS $$ +BEGIN +-- Authors ----------------------------------------------------------------------------------------- +CREATE TABLE "Authors" ( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + "First Name" text, + "Last Name" text, + "Website" text +); +INSERT INTO "Authors" OVERRIDING SYSTEM VALUE VALUES + (1, 'Edwin A.', 'Abbott', NULL), + (2, 'M.A.S.', 'Abdel Haleem', NULL), + (3, 'Joe', 'Abercrombie', 'https://joeabercrombie.com/'), + (4, 'Daniel', 'Abraham', 'https://www.danielabraham.com/'), + (5, NULL, 'Abu''l-Fazl', NULL); +PERFORM setval(pg_get_serial_sequence('"Authors"', 'id'), (SELECT max(id) FROM "Authors")); +-- colors ------------------------------------------------------------------------------------------ +CREATE TABLE colors (id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, name text); +INSERT INTO colors (name) VALUES ('red'), ('blue'); +-- fav_combos -------------------------------------------------------------------------------------- +CREATE TABLE fav_combos (number integer, color integer); +ALTER TABLE fav_combos ADD UNIQUE (number, color); +INSERT INTO fav_combos VALUES (5, 1), (5, 2), (10, 1), (10, 2); +-- Books ------------------------------------------------------------------------------------------- +CREATE TABLE "Books" ( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + "Title" text, + "Publication Year" date, + "ISBN" text, + "Dewey Decimal" text, + "Author" integer REFERENCES "Authors"(id), + "Publisher" integer, + "Favorite Number" integer, + "Favorite Color" integer REFERENCES colors(id) +); +ALTER TABLE "Books" DROP COLUMN "Publication Year"; +INSERT INTO "Books" OVERRIDING SYSTEM VALUE VALUES + (1059, 'The History of Akbar, Volume 7', '06-742-4416-8', NULL, 5, 116, 5, 1), + (960, 'The Dragon''s Path', '978-68173-11-59-3', '813.6', 4, 167, 5, 1), + (419, 'Half a King', '0007-55-020-0', '823.92', 3, 113, 5, 1), + (1047, 'The Heroes', '0-3-1604498-9', '823.92', 3, 167, 5, 1), + (103, 'Best Served Cold', '031604-49-5-4', '823.92', 3, 167, 5, 1), + (1302, 'The Widow''s House', '0-31-620398-X', '813.6', 4, 167, 5, NULL), + (99, 'Before They Are Hanged', '1-5910-2641-5', '823.92', 3, 195, 5, 2), + (530, 'Last Argument of Kings', '1591-02-690-3', '823.92', 3, 195, NULL, 1), + (104, 'Best Served Cold', '978-9552-8856-8-1', '823.92', 3, 167, 5, 1), + (1185, 'The Qur''an', '0-19-957071-X', '297.122521', 2, 171, 5, 1), + (1053, 'The History of Akbar, Volume 1', '0-674-42775-0', '954.02', 5, 116, 5, 1), + (959, 'The Dragon''s Path', '978-0-316080-68-2', '813.6', 4, 167, 5, 1), + (1056, 'The History of Akbar, Volume 4', '0-67497-503-0', NULL, 5, 116, 5, 1), + (69, 'A Shadow in Summer', '07-6-531340-5', '813.6', 4, 243, 5, 2), + (907, 'The Blade Itself', '978-1984-1-1636-1', '823.92', 3, 195, 5, 1), + (1086, 'The King''s Blood', '978-03-1608-077-4', '813.6', 4, 167, 5, 1), + (1060, 'The History of Akbar, Volume 8', '0-674-24417-6', NULL, 5, 116, 5, 1), + (70, 'A Shadow in Summer', '978-9-5-7802049-0', '813.6', 4, 243, 5, 2), + (1278, 'The Tyrant''s Law', '0-316-08070-5', '813.6', 4, 167, 5, 1), + (1054, 'The History of Akbar, Volume 2', '0-67-450494-1', NULL, 5, 116, 10, 1), + (1057, 'The History of Akbar, Volume 5', '0-6-7498395-5', NULL, 5, 116, 5, 1), + (351, 'Flatland: A Romance of Many Dimensions', '0-486-27263-X', '530.11', 1, 71, 5, 1), + (729, 'Red Country', '03161-87-20-8', '823.92', 3, 167, 5, 1), + (906, 'The Blade Itself', '1-591-02594-X', '823.92', 3, 195, 5, 1), + (1058, 'The History of Akbar, Volume 6', '067-4-98613-X', NULL, 5, 116, 10, 1), + (1055, 'The History of Akbar, Volume 3', '0-6-7465982-1', NULL, 5, 116, 5, 1); +PERFORM setval(pg_get_serial_sequence('"Books"', 'id'), (SELECT max(id) FROM "Books")); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION __setup_move_columns_nodata() RETURNS SETOF TEXT AS $$ +BEGIN +-- Authors ----------------------------------------------------------------------------------------- +CREATE TABLE "Authors" ( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + "First Name" text, + "Last Name" text, + "Website" text +); +-- colors ------------------------------------------------------------------------------------------ +CREATE TABLE colors (id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, name text); +-- fav_combos -------------------------------------------------------------------------------------- +CREATE TABLE fav_combos (number integer, color integer); +ALTER TABLE fav_combos ADD UNIQUE (number, color); +-- Books ------------------------------------------------------------------------------------------- +CREATE TABLE "Books" ( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + "Title" text, + "Publication Year" date, + "ISBN" text, + "Dewey Decimal" text, + "Author" integer REFERENCES "Authors"(id), + "Publisher" integer, + "Favorite Number" integer UNIQUE, + "Favorite Color" integer REFERENCES colors(id) +); +ALTER TABLE "Books" DROP COLUMN "Publication Year"; +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_move_columns_to_referenced_table_nodata() RETURNS SETOF TEXT AS $$ +BEGIN + PERFORM __setup_move_columns_nodata(); + PERFORM msar.move_columns_to_referenced_table( + '"Books"'::regclass, '"Authors"'::regclass, ARRAY[8, 9]::smallint[] + ); + RETURN NEXT columns_are( + 'Authors', + ARRAY['id', 'First Name', 'Last Name', 'Website', 'Favorite Number', 'Favorite Color'] + ); + RETURN NEXT columns_are( + 'Books', + ARRAY['id', 'Title', 'ISBN', 'Dewey Decimal', 'Author', 'Publisher'] + ); + RETURN NEXT col_is_unique('Authors', 'Favorite Number'); + RETURN NEXT fk_ok('Authors', 'Favorite Color', 'colors', 'id'); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_move_columns_to_referenced_table() RETURNS SETOF TEXT AS $$ +BEGIN + PERFORM __setup_move_columns(); + PERFORM msar.move_columns_to_referenced_table( + '"Books"'::regclass, '"Authors"'::regclass, ARRAY[8, 9]::smallint[] + ); + RETURN NEXT columns_are( + 'Authors', + ARRAY['id', 'First Name', 'Last Name', 'Website', 'Favorite Number', 'Favorite Color'] + ); + RETURN NEXT columns_are( + 'Books', + ARRAY['id', 'Title', 'ISBN', 'Dewey Decimal', 'Author', 'Publisher'] + ); + RETURN NEXT fk_ok('Authors', 'Favorite Color', 'colors', 'id'); + RETURN NEXT results_eq( + $h$SELECT * FROM "Authors" ORDER BY id;$h$, + $w$VALUES + (1, 'Edwin A.', 'Abbott', NULL, 5, 1), + (2, 'M.A.S.', 'Abdel Haleem', NULL, 5, 1), + (3, 'Joe', 'Abercrombie', 'https://joeabercrombie.com/', 5, 1), + (4, 'Daniel', 'Abraham', 'https://www.danielabraham.com/', 5, 1), + (5, NULL, 'Abu''l-Fazl', NULL, 5, 1), + (6, 'Joe', 'Abercrombie', 'https://joeabercrombie.com/', 5, 2), + (7, 'Joe', 'Abercrombie', 'https://joeabercrombie.com/',NULL,1), + (8, 'Daniel', 'Abraham', 'https://www.danielabraham.com/', 5, 2); + $w$ + ); +END; +$$ LANGUAGE plpgsql; From 9bbf2a4a9cfbe41a9f30456a3e291e2df3005080 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 4 Sep 2024 21:49:29 +0800 Subject: [PATCH 0804/1141] upgrade python-decouple in an attempt to get pipeline passing --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e290467dd7..bb80b496a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ drf-nested-routers==0.93.3 psycopg==3.1.18 psycopg-binary==3.1.18 psycopg2-binary==2.9.7 -python-decouple==3.4 +python-decouple==3.8 requests==2.32.0 SQLAlchemy==1.4.26 responses==0.22.0 From 2a7786acb7ea7d03e37cfcd4f31ee7bfb17ad835 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 4 Sep 2024 21:59:43 +0800 Subject: [PATCH 0805/1141] upgrade another package to satisfy pipeline --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bb80b496a2..22a99ee461 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ Django==4.2.11 dj-database-url==0.5.0 django-filter==23.5 django-modern-rpc==1.0.3 -django-property-filter==1.1.0 +django-property-filter==1.2.0 django-request-cache==1.3.0 djangorestframework==3.14.0 django-fernet-encrypted-fields==0.1.3 From 19503945dd0bc9083bf392a09d7f67bf70b693af Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Wed, 4 Sep 2024 22:06:21 +0800 Subject: [PATCH 0806/1141] upgrade another package to satisfy pipeline :hankey: --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 22a99ee461..6f3d5caeb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ alembic==1.6.5 bidict==0.21.4 frozendict==2.1.3 charset-normalizer==2.0.7 -clevercsv==0.6.8 +clevercsv==0.8.2 Django==4.2.11 dj-database-url==0.5.0 django-filter==23.5 From 2031651e0dcf9e4eaf1236b6f1538a2fa57c9dc2 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 5 Sep 2024 15:12:20 +0800 Subject: [PATCH 0807/1141] add record self summary for lister --- db/sql/00_msar.sql | 45 ++++++++++++++++++++++++++++++++++---- db/sql/test_00_msar.sql | 48 ++++++++++++++++++++++++++--------------- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 472d1945d0..36569f2c13 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -415,6 +415,18 @@ SELECT EXISTS ( $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION +msar.get_pkey_attnum(rel_id regclass) RETURNS smallint AS $$/* +Get the attnum of the single-column primary key for a relation if it has one. If not, return null. + +Args: + rel_id: The OID of the relation. +*/ +SELECT conkey[1] FROM pg_constraint +WHERE cardinality(conkey) = 1 AND conrelid=rel_id AND contype='p'; +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.is_default_possibly_dynamic(tab_id oid, col_id integer) RETURNS boolean AS $$/* Determine whether the default value for the given column is an expression or constant. @@ -4260,6 +4272,21 @@ FROM fkey_map_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION msar.build_self_summary_expr(tab_id oid) RETURNS TEXT AS $$/* +*/ +SELECT CONCAT(msar.build_summary_expr(tab_id), ' AS __mathesar_summary'); +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.build_self_summary_json_expr(tab_id oid, cte_name text) RETURNS TEXT AS $$/* +*/ +SELECT format('jsonb_object_agg(%1$I.', cte_name) + || quote_ident(msar.get_pkey_attnum(tab_id)::text) + || ', __mathesar_summary)' +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.list_records_from_table( tab_id oid, @@ -4267,7 +4294,8 @@ msar.list_records_from_table( offset_ integer, order_ jsonb, filter_ jsonb, - group_ jsonb + group_ jsonb, + return_record_summaries boolean DEFAULT false ) RETURNS jsonb AS $$/* Get records from a table. Only columns to which the user has access are returned. @@ -4278,6 +4306,7 @@ Args: order_: An array of ordering definition objects. filter_: An array of filter definition objects. group_: An array of group definition objects. + return_record_summaries : Whether to return a summary for each record listed. The order definition objects should have the form {"attnum": , "direction": } @@ -4290,7 +4319,7 @@ BEGIN WITH count_cte AS ( SELECT count(1) AS count FROM %2$I.%3$I %7$s ), enriched_results_cte AS ( - SELECT %1$s, %8$s FROM %2$I.%3$I %7$s %6$s LIMIT %4$L OFFSET %5$L + SELECT %1$s, %8$s, %15$s FROM %2$I.%3$I %7$s %6$s LIMIT %4$L OFFSET %5$L ), results_ranked_cte AS ( SELECT *, row_number() OVER (%6$s) - 1 AS __mathesar_result_idx FROM enriched_results_cte ), groups_cte AS ( @@ -4300,7 +4329,8 @@ BEGIN 'results', %9$s, 'count', coalesce(max(count_cte.count), 0), 'grouping', %10$s, - 'preview_data', %14$s, + 'linked_record_summaries', %14$s, + 'record_summaries', %16$s, 'query', $iq$SELECT %1$s FROM %2$I.%3$I %7$s %6$s LIMIT %4$L OFFSET %5$L$iq$ ) FROM enriched_results_cte @@ -4320,7 +4350,14 @@ BEGIN COALESCE(msar.build_groups_cte_expr(tab_id, 'results_ranked_cte', group_), 'NULL AS id'), msar.build_summary_cte_expr_for_table(tab_id), msar.build_summary_join_expr_for_table(tab_id, 'enriched_results_cte'), - COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL') + COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL'), + msar.build_self_summary_expr(tab_id), + COALESCE( + CASE WHEN return_record_summaries THEN + msar.build_self_summary_json_expr(tab_id, 'enriched_results_cte') + END, + 'NULL' + ) ) INTO records; RETURN records; END; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index ade5855591..c654dcae5d 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -3087,7 +3087,8 @@ BEGIN {"1": 3, "2": 2, "3": "abcde", "4": {"k": 3242348}, "5": true} ], "grouping": null, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data(col1) AS "2",' @@ -3113,7 +3114,8 @@ BEGIN {"1": 1, "2": 5, "3": "sdflkj", "4": "s", "5": {"a": "val"}} ], "grouping": null, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data(col1) AS "2",' @@ -3139,7 +3141,8 @@ BEGIN {"1": 1, "2": 5, "3": "sdflkj", "4": "s", "5": {"a": "val"}} ], "grouping": null, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data(col1) AS "2",', @@ -3171,7 +3174,8 @@ BEGIN {"2": 34, "3": "sdflfflsk", "4": null, "5": [1, 2, 3, 4]} ], "grouping": null, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(col1) AS "2", msar.format_data(col2) AS "3",', @@ -3197,7 +3201,8 @@ BEGIN {"2": 2, "3": "abcde", "4": {"k": 3242348}, "5": true} ], "grouping": null, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(col1) AS "2", msar.format_data(col2) AS "3",', @@ -3275,7 +3280,8 @@ BEGIN } ] }, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data("First Name") AS "2",' @@ -3312,7 +3318,8 @@ BEGIN } ] }, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data("First Name") AS "2",' @@ -3345,7 +3352,8 @@ BEGIN {"id": 2, "count": 2, "results_eq": {"4": "2020-04 AD"}, "result_indices": [1, 2]} ] }, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data("First Name") AS "2",' @@ -3379,7 +3387,8 @@ BEGIN {"id": 1, "count": 8, "results_eq": {"4": "2020 AD"}, "result_indices": [0, 1, 2, 3, 4]} ] }, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data("First Name") AS "2",' @@ -3891,7 +3900,8 @@ BEGIN {"1": 2, "2": 34, "3": "sdflfflsk", "4": null, "5": [1, 2, 3, 4]} ], "grouping": null, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data(col1) AS "2",', @@ -3907,7 +3917,8 @@ BEGIN "count": 0, "results": [], "grouping": null, - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data(col1) AS "2",', @@ -4296,7 +4307,7 @@ BEGIN {"1": 6, "2": 1.234, "3": 3, "4": "Kelly Kellison", "5": 80, "6": "kkellison@example.edu"} ], "grouping": null, - "preview_data": { + "linked_record_summaries": { "2": { "1.234": "Alice Alison", "2.345": "Bob Bobinson" @@ -4306,7 +4317,8 @@ BEGIN "2": "Dave Davidson", "3": "Eve Evilson" } - } + }, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data("Counselor") AS "2",', @@ -4333,7 +4345,7 @@ BEGIN {"1": 4, "2": 2.345, "3": 1, "4": "Ida Idalia", "5": 90, "6": "iidalia@example.edu"} ], "grouping": null, - "preview_data": { + "linked_record_summaries": { "2": { "1.234": "Alice Alison", "2.345": "Bob Bobinson" @@ -4342,7 +4354,8 @@ BEGIN "1": "Carol Carlson", "2": "Dave Davidson" } - } + }, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data("Counselor") AS "2",', @@ -4372,7 +4385,7 @@ BEGIN "preproc": null, "groups": [{"id": 1, "count": 3, "results_eq": {"2": 1.234}, "result_indices": [0, 1]}] }, - "preview_data": { + "linked_record_summaries": { "2": { "1.234": "Alice Alison" }, @@ -4380,7 +4393,8 @@ BEGIN "1": "Carol Carlson", "2": "Dave Davidson" } - } + }, + "record_summaries": null }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data("Counselor") AS "2",', From e3218e4eb179fe6be079f208d12b33e2b902ab77 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 5 Sep 2024 16:49:01 +0800 Subject: [PATCH 0808/1141] simplify record self summaries --- db/sql/00_msar.sql | 86 ++++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 36569f2c13..e577686efb 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -4210,21 +4210,35 @@ Args: tab_id: The table for whose fkey values' linked records we'll get summaries. */ WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_cte(tab_id)) -SELECT ', ' || string_agg( - format( - $c$summary_cte_%1$s AS ( - SELECT - msar.format_data(%2$I) AS fkey, - %3$s AS summary - FROM %4$I.%5$I - )$c$, - conkey, - msar.get_column_name(target_oid, confkey), - msar.build_summary_expr(target_oid), - msar.get_relation_schema_name(target_oid), - msar.get_relation_name(target_oid) - ), ', ' -) +SELECT ', ' + || NULLIF( + concat_ws(', ', + 'summary_cte_self AS (SELECT msar.format_data(' + || quote_ident(msar.get_column_name(tab_id, msar.get_pkey_attnum(tab_id))) + || format( + ') AS key, %1$s AS summary FROM %2$I.%3$I)', + msar.build_summary_expr(tab_id), + msar.get_relation_schema_name(tab_id), + msar.get_relation_name(tab_id) + ), + string_agg( + format( + $c$summary_cte_%1$s AS ( + SELECT + msar.format_data(%2$I) AS fkey, + %3$s AS summary + FROM %4$I.%5$I + )$c$, + conkey, + msar.get_column_name(target_oid, confkey), + msar.build_summary_expr(target_oid), + msar.get_relation_schema_name(target_oid), + msar.get_relation_name(target_oid) + ), ', ' + ) + ), + '' + ) FROM fkey_map_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; @@ -4238,13 +4252,18 @@ Args: cte_name: The name of the main CTE we'll join the summary CTEs to. */ WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_cte(tab_id)) -SELECT string_agg( - format( - $j$ - LEFT JOIN summary_cte_%1$s ON %2$I.%1$I = summary_cte_%1$s.fkey$j$, - conkey, - cte_name - ), ' ' +SELECT concat( + format(E'\nLEFT JOIN summary_cte_self ON %1$I.', cte_name) + || quote_ident(msar.get_pkey_attnum(tab_id)::text) + || ' = summary_cte_self.key' , + string_agg( + format( + $j$ + LEFT JOIN summary_cte_%1$s ON %2$I.%1$I = summary_cte_%1$s.fkey$j$, + conkey, + cte_name + ), ' ' + ) ) FROM fkey_map_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; @@ -4272,18 +4291,12 @@ FROM fkey_map_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION msar.build_self_summary_expr(tab_id oid) RETURNS TEXT AS $$/* -*/ -SELECT CONCAT(msar.build_summary_expr(tab_id), ' AS __mathesar_summary'); -$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; - - CREATE OR REPLACE FUNCTION -msar.build_self_summary_json_expr(tab_id oid, cte_name text) RETURNS TEXT AS $$/* +msar.build_self_summary_json_expr(tab_id oid) RETURNS TEXT AS $$/* */ -SELECT format('jsonb_object_agg(%1$I.', cte_name) - || quote_ident(msar.get_pkey_attnum(tab_id)::text) - || ', __mathesar_summary)' +SELECT CASE WHEN quote_ident(msar.get_pkey_attnum(tab_id)::text) IS NOT NULL THEN + 'jsonb_object_agg(summary_cte_self.key, summary_cte_self.summary)' +END; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; @@ -4319,7 +4332,7 @@ BEGIN WITH count_cte AS ( SELECT count(1) AS count FROM %2$I.%3$I %7$s ), enriched_results_cte AS ( - SELECT %1$s, %8$s, %15$s FROM %2$I.%3$I %7$s %6$s LIMIT %4$L OFFSET %5$L + SELECT %1$s, %8$s FROM %2$I.%3$I %7$s %6$s LIMIT %4$L OFFSET %5$L ), results_ranked_cte AS ( SELECT *, row_number() OVER (%6$s) - 1 AS __mathesar_result_idx FROM enriched_results_cte ), groups_cte AS ( @@ -4330,7 +4343,7 @@ BEGIN 'count', coalesce(max(count_cte.count), 0), 'grouping', %10$s, 'linked_record_summaries', %14$s, - 'record_summaries', %16$s, + 'record_summaries', %15$s, 'query', $iq$SELECT %1$s FROM %2$I.%3$I %7$s %6$s LIMIT %4$L OFFSET %5$L$iq$ ) FROM enriched_results_cte @@ -4351,11 +4364,8 @@ BEGIN msar.build_summary_cte_expr_for_table(tab_id), msar.build_summary_join_expr_for_table(tab_id, 'enriched_results_cte'), COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL'), - msar.build_self_summary_expr(tab_id), COALESCE( - CASE WHEN return_record_summaries THEN - msar.build_self_summary_json_expr(tab_id, 'enriched_results_cte') - END, + CASE WHEN return_record_summaries THEN msar.build_self_summary_json_expr(tab_id) END, 'NULL' ) ) INTO records; From 616303aa0190b22575a4e9ae173a6e985bde6a03 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 5 Sep 2024 18:05:16 +0800 Subject: [PATCH 0809/1141] set up some todos for while I'm gone --- db/sql/00_msar.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 399d77f563..70a3c5abd1 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3899,6 +3899,9 @@ DECLARE move_con_defs CONSTANT jsonb := msar.get_extracted_con_def_jsonb(source_tab_id, move_col_ids); added_col_ids smallint[]; BEGIN + -- TODO test to make sure no multi-col fkeys reference the moved columns + -- TODO just throw error if _any_ multicol constraint references the moved columns. + -- TODO check behavior if one of the moving columns is referenced by another table (should raise) SELECT conkey, confkey INTO source_join_col_id, target_join_col_id FROM msar.get_fkey_map_table(source_tab_id) WHERE target_oid = target_tab_id; @@ -3949,8 +3952,6 @@ BEGIN msar.build_source_update_move_cols_equal_expr(source_tab_id, move_col_ids, 'insert_cte') ); PERFORM msar.add_constraints(target_tab_id, move_con_defs); - -- TODO make sure no multi-col fkeys reference the moved columns - -- TODO just throw error if _any_ multicol constraint references the moved columns. PERFORM msar.drop_columns(source_tab_id, variadic move_col_ids); END; $$ LANGUAGE plpgsql; From 6046b5b8f5013b61f2b047867cd4d3a355ef87a0 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 5 Sep 2024 18:23:12 +0800 Subject: [PATCH 0810/1141] add summaries to record search and get --- db/sql/00_msar.sql | 37 +++++++++++++++++++++++++++---------- db/sql/test_00_msar.sql | 2 +- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index e577686efb..e3d9cf10b7 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -416,14 +416,20 @@ $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.get_pkey_attnum(rel_id regclass) RETURNS smallint AS $$/* +msar.get_selectable_pkey_attnum(rel_id regclass) RETURNS smallint AS $$/* Get the attnum of the single-column primary key for a relation if it has one. If not, return null. +The attnum will only be returned if the current user has SELECT on that column. + Args: rel_id: The OID of the relation. */ SELECT conkey[1] FROM pg_constraint -WHERE cardinality(conkey) = 1 AND conrelid=rel_id AND contype='p'; +WHERE + conrelid = rel_id + AND cardinality(conkey) = 1 + AND contype='p' + AND has_column_privilege(rel_id, conkey[1], 'SELECT'); $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; @@ -4214,7 +4220,7 @@ SELECT ', ' || NULLIF( concat_ws(', ', 'summary_cte_self AS (SELECT msar.format_data(' - || quote_ident(msar.get_column_name(tab_id, msar.get_pkey_attnum(tab_id))) + || quote_ident(msar.get_column_name(tab_id, msar.get_selectable_pkey_attnum(tab_id))) || format( ') AS key, %1$s AS summary FROM %2$I.%3$I)', msar.build_summary_expr(tab_id), @@ -4254,7 +4260,7 @@ Args: WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_cte(tab_id)) SELECT concat( format(E'\nLEFT JOIN summary_cte_self ON %1$I.', cte_name) - || quote_ident(msar.get_pkey_attnum(tab_id)::text) + || quote_ident(msar.get_selectable_pkey_attnum(tab_id)::text) || ' = summary_cte_self.key' , string_agg( format( @@ -4294,7 +4300,7 @@ $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION msar.build_self_summary_json_expr(tab_id oid) RETURNS TEXT AS $$/* */ -SELECT CASE WHEN quote_ident(msar.get_pkey_attnum(tab_id)::text) IS NOT NULL THEN +SELECT CASE WHEN quote_ident(msar.get_selectable_pkey_attnum(tab_id)::text) IS NOT NULL THEN 'jsonb_object_agg(summary_cte_self.key, summary_cte_self.summary)' END; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; @@ -4407,7 +4413,8 @@ CREATE OR REPLACE FUNCTION msar.search_records_from_table( tab_id oid, search_ jsonb, - limit_ integer + limit_ integer, + return_record_summaries boolean DEFAULT false ) RETURNS jsonb AS $$/* Get records from a table, filtering and sorting according to a search specification. @@ -4434,7 +4441,8 @@ BEGIN SELECT jsonb_build_object( 'results', coalesce(jsonb_agg(row_to_json(results_cte.*)), jsonb_build_array()), 'count', coalesce(max(count_cte.count), 0), - 'preview_data', %9$s, + 'linked_record_summaries', %9$s, + 'record_summaries', %10$s, 'query', $iq$SELECT %1$s FROM %2$I.%3$I %4$s ORDER BY %6$s LIMIT %5$L$iq$ ) FROM results_cte %8$s @@ -4451,7 +4459,11 @@ BEGIN ), msar.build_summary_cte_expr_for_table(tab_id), msar.build_summary_join_expr_for_table(tab_id, 'results_cte'), - COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL') + COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL'), + COALESCE( + CASE WHEN return_record_summaries THEN msar.build_self_summary_json_expr(tab_id) END, + 'NULL' + ) ) INTO records; RETURN records; END; @@ -4459,7 +4471,11 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION -msar.get_record_from_table(tab_id oid, rec_id anyelement) RETURNS jsonb AS $$/* +msar.get_record_from_table( + tab_id oid, + rec_id anyelement, + return_record_summaries boolean DEFAULT false +) RETURNS jsonb AS $$/* Get single record from a table. Only columns to which the user has access are returned. Args: @@ -4476,7 +4492,8 @@ SELECT msar.list_records_from_table( jsonb_build_object('type', 'literal', 'value', rec_id) ) ), - null + null, + return_record_summaries ) $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index c654dcae5d..96dc7b4de1 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4461,7 +4461,7 @@ BEGIN '"Students"'::regclass::oid, '[{"attnum": 4, "literal": "k"}]', 2 - ) -> 'preview_data', + ) -> 'linked_record_summaries', $a${ "2": {"1.234": "Alice Alison", "2.345": "Bob Bobinson"}, "3": {"3": "Eve Evilson"} From 58b8b96b82e5230e5b27861773fc932f330b8f90 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 5 Sep 2024 18:30:06 +0800 Subject: [PATCH 0811/1141] add record self summaries to record adder --- db/sql/00_msar.sql | 15 ++++++++++++--- db/sql/test_00_msar.sql | 20 +++++++++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index e3d9cf10b7..3d84166ce1 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -4551,7 +4551,11 @@ $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.add_record_to_table(tab_id oid, rec_def jsonb) RETURNS jsonb AS $$/* +msar.add_record_to_table( + tab_id oid, + rec_def jsonb, + return_record_summaries boolean DEFAULT false +) RETURNS jsonb AS $$/* Add a record to a table. Args: @@ -4571,7 +4575,8 @@ BEGIN WITH insert_cte AS (%1$s RETURNING %2$s)%4$s SELECT jsonb_build_object( 'results', %3$s, - 'preview_data', %6$s + 'linked_record_summaries', %6$s, + 'record_summaries', %7$s ) FROM insert_cte %5$s $i$, @@ -4580,7 +4585,11 @@ BEGIN msar.build_results_jsonb_expr(tab_id, 'insert_cte', null), msar.build_summary_cte_expr_for_table(tab_id), msar.build_summary_join_expr_for_table(tab_id, 'insert_cte'), - COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL') + COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL'), + COALESCE( + CASE WHEN return_record_summaries THEN msar.build_self_summary_json_expr(tab_id) END, + 'NULL' + ) ) INTO rec_created; RETURN rec_created; END; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 96dc7b4de1..27e38a1479 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4102,7 +4102,8 @@ BEGIN ), $a${ "results": [{"1": 4, "2": 234, "3": "ab234", "4": {"key": "val"}, "5": {"key2": "val2"}}], - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$a$ ); END; @@ -4122,7 +4123,8 @@ BEGIN ), $a${ "results": [{"1": 4, "2": 234, "3": "ab234", "4": {"key": "val"}, "5": {"key2": "val2"}}], - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$a$ ); END; @@ -4142,7 +4144,8 @@ BEGIN ), $a${ "results": [{"1": 4, "2": 200, "3": "ab234", "4": {"key": "val"}, "5": {"key2": "val2"}}], - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$a$ ); END; @@ -4162,7 +4165,8 @@ BEGIN ), $a${ "results": [{"1": 4, "2": null, "3": "ab234", "4": {"key": "val"}, "5": {"key2": "val2"}}], - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$a$ ); END; @@ -4182,7 +4186,8 @@ BEGIN ), $a${ "results": [{"1": 4, "2": null, "3": "ab234", "4": 3, "5": "234"}], - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$a$ ); END; @@ -4420,10 +4425,11 @@ BEGIN "results": [ {"1": 7, "2": 2.345, "3": 1, "4": "Larry Laurelson", "5": 70, "6": "llaurelson@example.edu"} ], - "preview_data": { + "linked_record_summaries": { "2": {"2.345": "Bob Bobinson"}, "3": {"1": "Carol Carlson"} - } + }, + "record_summaries": null }$a$ ); END; From dfa07b87f3f1212518e9960081082ce94c708e0c Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Thu, 5 Sep 2024 18:34:34 +0800 Subject: [PATCH 0812/1141] add record self summaries to record patch --- db/sql/00_msar.sql | 16 +++++++++++++--- db/sql/test_00_msar.sql | 11 +++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 3d84166ce1..0f47594e89 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -4611,7 +4611,12 @@ $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; CREATE OR REPLACE FUNCTION -msar.patch_record_in_table(tab_id oid, rec_id anyelement, rec_def jsonb) RETURNS jsonb AS $$/* +msar.patch_record_in_table( + tab_id oid, + rec_id anyelement, + rec_def jsonb, + return_record_summaries boolean DEFAULT false +) RETURNS jsonb AS $$/* Modify (update/patch) a record in a table. Args: @@ -4632,7 +4637,8 @@ BEGIN WITH update_cte AS (%1$s %2$s RETURNING %3$s)%5$s SELECT jsonb_build_object( 'results', %4$s, - 'preview_data', %7$s + 'linked_record_summaries', %7$s, + 'record_summaries', %8$s ) FROM update_cte %6$s $i$, @@ -4649,7 +4655,11 @@ BEGIN msar.build_results_jsonb_expr(tab_id, 'update_cte', null), msar.build_summary_cte_expr_for_table(tab_id), msar.build_summary_join_expr_for_table(tab_id, 'update_cte'), - COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL') + COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL'), + COALESCE( + CASE WHEN return_record_summaries THEN msar.build_self_summary_json_expr(tab_id) END, + 'NULL' + ) ) INTO rec_modified; RETURN rec_modified; END; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index 27e38a1479..d535d21beb 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4204,7 +4204,8 @@ BEGIN msar.patch_record_in_table( rel_id, 2, '{"2": 10}'), $p${ "results": [{"1": 2, "2": 10, "3": "sdflfflsk", "4": null, "5": [1, 2, 3, 4]}], - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$p$ ); END; @@ -4221,7 +4222,8 @@ BEGIN msar.patch_record_in_table( rel_id, 2, '{"2": 10, "4": {"a": "json"}}'), $p${ "results": [{"1": 2, "2": 10, "3": "sdflfflsk", "4": {"a": "json"}, "5": [1, 2, 3, 4]}], - "preview_data": null + "linked_record_summaries": null, + "record_summaries": null }$p$ ); END; @@ -4449,10 +4451,11 @@ BEGIN "results": [ {"1": 2, "2": 2.345, "3": 2, "4": "Gabby Gabberson", "5": 85, "6": "ggabberson@example.edu"} ], - "preview_data": { + "linked_record_summaries": { "2": {"2.345": "Bob Bobinson"}, "3": {"2": "Dave Davidson"} - } + }, + "record_summaries": null }$a$ ); END; From a12e90d61052e27ae54af3a1edc03deedb530d7c Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 6 Sep 2024 00:05:08 +0800 Subject: [PATCH 0813/1141] simplify response getting for insert/update --- db/sql/00_msar.sql | 61 ++++++++++++++++------------------------- db/sql/test_00_msar.sql | 17 +++++++++--- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 0f47594e89..510ab0538f 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -4568,30 +4568,21 @@ insert. */ DECLARE + rec_created_id text; rec_created jsonb; BEGIN EXECUTE format( - $i$ - WITH insert_cte AS (%1$s RETURNING %2$s)%4$s - SELECT jsonb_build_object( - 'results', %3$s, - 'linked_record_summaries', %6$s, - 'record_summaries', %7$s - ) - FROM insert_cte %5$s - $i$, + 'WITH insert_cte AS (%1$s RETURNING %2$s) SELECT msar.format_data(%3$I)::text FROM insert_cte', msar.build_single_insert_expr(tab_id, rec_def), msar.build_selectable_column_expr(tab_id), - msar.build_results_jsonb_expr(tab_id, 'insert_cte', null), - msar.build_summary_cte_expr_for_table(tab_id), - msar.build_summary_join_expr_for_table(tab_id, 'insert_cte'), - COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL'), - COALESCE( - CASE WHEN return_record_summaries THEN msar.build_self_summary_json_expr(tab_id) END, - 'NULL' - ) - ) INTO rec_created; - RETURN rec_created; + msar.get_pk_column(tab_id) + ) INTO rec_created_id; + rec_created := msar.get_record_from_table(tab_id, rec_created_id, return_record_summaries); + RETURN jsonb_build_object( + 'results', rec_created -> 'results', + 'record_summaries', rec_created -> 'record_summaries', + 'linked_record_summaries', rec_created -> 'linked_record_summaries' + ); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; @@ -4630,18 +4621,14 @@ The `rec_def` object's form is defined by the record being updated. It should h corresponding to the attnums of desired columns and values corresponding to values we should set. */ DECLARE + rec_modified_id integer; rec_modified jsonb; BEGIN EXECUTE format( - $i$ - WITH update_cte AS (%1$s %2$s RETURNING %3$s)%5$s - SELECT jsonb_build_object( - 'results', %4$s, - 'linked_record_summaries', %7$s, - 'record_summaries', %8$s - ) - FROM update_cte %6$s - $i$, + $p$ + WITH update_cte AS (%1$s %2$s RETURNING %3$s) + SELECT msar.format_data(%4$I)::text FROM update_cte + $p$, msar.build_update_expr(tab_id, rec_def), msar.build_where_clause( tab_id, jsonb_build_object( @@ -4652,15 +4639,13 @@ BEGIN ) ), msar.build_selectable_column_expr(tab_id), - msar.build_results_jsonb_expr(tab_id, 'update_cte', null), - msar.build_summary_cte_expr_for_table(tab_id), - msar.build_summary_join_expr_for_table(tab_id, 'update_cte'), - COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL'), - COALESCE( - CASE WHEN return_record_summaries THEN msar.build_self_summary_json_expr(tab_id) END, - 'NULL' - ) - ) INTO rec_modified; - RETURN rec_modified; + msar.get_pk_column(tab_id) + ) INTO rec_modified_id; + rec_modified := msar.get_record_from_table(tab_id, rec_modified_id, return_record_summaries); + RETURN jsonb_build_object( + 'results', rec_modified -> 'results', + 'record_summaries', rec_modified -> 'record_summaries', + 'linked_record_summaries', rec_modified -> 'linked_record_summaries' + ); END; $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index d535d21beb..8b0bf6cbca 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -4301,7 +4301,8 @@ BEGIN offset_ => null, order_ => null, filter_ => null, - group_ => null + group_ => null, + return_record_summaries => true ), $j${ "count": 6, @@ -4325,7 +4326,14 @@ BEGIN "3": "Eve Evilson" } }, - "record_summaries": null + "record_summaries": { + "1": "Fred Fredrickson", + "2": "Gabby Gabberson", + "3": "Hank Hankson", + "4": "Ida Idalia", + "5": "James Jameson", + "6": "Kelly Kellison" + } }$j$ || jsonb_build_object( 'query', concat( 'SELECT msar.format_data(id) AS "1", msar.format_data("Counselor") AS "2",', @@ -4421,7 +4429,8 @@ BEGIN RETURN NEXT is( msar.add_record_to_table( '"Students"'::regclass::oid, - '{"2": 2.345, "3": 1, "4": "Larry Laurelson", "5": 70, "6": "llaurelson@example.edu"}' + '{"2": 2.345, "3": 1, "4": "Larry Laurelson", "5": 70, "6": "llaurelson@example.edu"}', + true ), $a${ "results": [ @@ -4431,7 +4440,7 @@ BEGIN "2": {"2.345": "Bob Bobinson"}, "3": {"1": "Carol Carlson"} }, - "record_summaries": null + "record_summaries": {"7": "Larry Laurelson"} }$a$ ); END; From 14ae5e374a112e1b7914bfbf413a00c5d8e0c88c Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 6 Sep 2024 00:10:22 +0800 Subject: [PATCH 0814/1141] add record self-summary to python wrappers for select --- db/records/operations/select.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/db/records/operations/select.py b/db/records/operations/select.py index b93418626c..9ef568d79e 100644 --- a/db/records/operations/select.py +++ b/db/records/operations/select.py @@ -19,6 +19,7 @@ def list_records_from_table( order=None, filter=None, group=None, + return_record_summaries=False ): """ Get records from a table. @@ -46,6 +47,7 @@ def list_records_from_table( json.dumps(order) if order is not None else None, json.dumps(filter) if filter is not None else None, json.dumps(group) if group is not None else None, + return_record_summaries ).fetchone()[0] return result @@ -54,6 +56,7 @@ def get_record_from_table( conn, record_id, table_oid, + return_record_summaries=False ): """ Get single record from a table by its primary key @@ -69,6 +72,7 @@ def get_record_from_table( 'get_record_from_table', table_oid, record_id, + return_record_summaries, ).fetchone()[0] return result @@ -78,6 +82,7 @@ def search_records_from_table( table_oid, search=[], limit=10, + return_record_summaries=False, ): """ Get records from a table, according to a search specification @@ -94,7 +99,8 @@ def search_records_from_table( """ search = search or [] result = db_conn.exec_msar_func( - conn, 'search_records_from_table', table_oid, json.dumps(search), limit + conn, 'search_records_from_table', + table_oid, json.dumps(search), limit, return_record_summaries ).fetchone()[0] return result From 177bfd81ae7af87ccc53f037c0f811bb53b8dd77 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 6 Sep 2024 00:13:08 +0800 Subject: [PATCH 0815/1141] wire record summary getting to update/insert python --- db/records/operations/insert.py | 5 +++-- db/records/operations/update.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/db/records/operations/insert.py b/db/records/operations/insert.py index 26def03bb0..bec20df1e8 100644 --- a/db/records/operations/insert.py +++ b/db/records/operations/insert.py @@ -16,13 +16,14 @@ READ_SIZE = 20000 -def add_record_to_table(conn, record_def, table_oid): +def add_record_to_table(conn, record_def, table_oid, return_record_summaries=False): """Add a record to a table.""" result = db_conn.exec_msar_func( conn, 'add_record_to_table', table_oid, - json.dumps(record_def) + json.dumps(record_def), + return_record_summaries ).fetchone()[0] return result diff --git a/db/records/operations/update.py b/db/records/operations/update.py index 0d06bd2bb6..1535251966 100644 --- a/db/records/operations/update.py +++ b/db/records/operations/update.py @@ -7,14 +7,15 @@ from db.records.exceptions import InvalidDate, InvalidDateFormat -def patch_record_in_table(conn, record_def, record_id, table_oid): +def patch_record_in_table(conn, record_def, record_id, table_oid, return_record_summaries=False): """Update a record in a table.""" result = db_conn.exec_msar_func( conn, 'patch_record_in_table', table_oid, record_id, - json.dumps(record_def) + json.dumps(record_def), + return_record_summaries ).fetchone()[0] return result From 4e3217cb50734b1e0b138594254d5aba0e6458cb Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 5 Sep 2024 23:06:48 +0530 Subject: [PATCH 0816/1141] add sql changes for retuning oid with name for add_mathesar_table and prepare_table_for_import --- db/sql/00_msar.sql | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 472d1945d0..2edd7cb4d8 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2838,8 +2838,8 @@ DROP FUNCTION IF EXISTS msar.add_mathesar_table(oid, text, jsonb, jsonb, text); CREATE OR REPLACE FUNCTION msar.add_mathesar_table(sch_id oid, tab_name text, col_defs jsonb, con_defs jsonb, comment_ text) - RETURNS oid AS $$/* -Add a table, with a default id column, returning the OID of the created table. + RETURNS jsonb AS $$/* +Add a table, with a default id column, returning the OID & name of the created table. Args: sch_id: The OID of the schema where the table will be created. @@ -2886,7 +2886,10 @@ BEGIN PERFORM __msar.add_table(fq_table_name, column_defs, constraint_defs); created_table_id := fq_table_name::regclass::oid; PERFORM msar.comment_on_table(created_table_id, comment_); - RETURN created_table_id; + RETURN jsonb_build_object( + 'oid', created_table_id, + 'name', created_table_id::regclass::text + ); END; $$ LANGUAGE plpgsql; @@ -2932,7 +2935,7 @@ DECLARE copy_sql text; BEGIN -- Create string table - rel_id := msar.add_mathesar_table(sch_id, tab_name, col_defs, NULL, comment_); + rel_id := msar.add_mathesar_table(sch_id, tab_name, col_defs, NULL, comment_) ->> 'oid'; -- Get unquoted schema and table name for the created table SELECT nspname, relname INTO sch_name, rel_name FROM pg_catalog.pg_class AS pgc @@ -2956,7 +2959,8 @@ BEGIN copy_sql := format('COPY %I.%I (%s) FROM STDIN CSV %s', sch_name, rel_name, col_names_sql, options_sql); RETURN jsonb_build_object( 'copy_sql', copy_sql, - 'table_oid', rel_id + 'table_oid', rel_id, + 'table_name', rel_id::regclass::text ); END; $$ LANGUAGE plpgsql; @@ -3660,7 +3664,7 @@ The elements of the mapping_columns array must have the form DECLARE added_table_id oid; BEGIN - added_table_id := msar.add_mathesar_table(sch_id, tab_name, NULL, NULL, NULL); + added_table_id := msar.add_mathesar_table(sch_id, tab_name, NULL, NULL, NULL) ->> 'oid'; PERFORM msar.add_foreign_key_column(column_name, added_table_id, referent_table_oid) FROM jsonb_to_recordset(mapping_columns) AS x(column_name text, referent_table_oid oid); RETURN added_table_id; @@ -3708,7 +3712,7 @@ BEGIN extracted_col_defs, extracted_con_defs, format('Extracted from %s', __msar.get_qualified_relation_name(tab_id)) - ); + ) ->> 'oid'; -- Create a new fkey column and foreign key linking the original table to the extracted one. fkey_attnum := msar.add_foreign_key_column(fkey_name, tab_id, extracted_table_id); -- Insert the data from the original table's columns into the extracted columns, and add From fd12f00142cc6eaedc4a8df88b059e7f821c2040 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 5 Sep 2024 23:08:52 +0530 Subject: [PATCH 0817/1141] make response changes for db functions --- db/tables/operations/create.py | 3 ++- db/tables/operations/import_.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/db/tables/operations/create.py b/db/tables/operations/create.py index d2c486cbe5..c5840dee89 100644 --- a/db/tables/operations/create.py +++ b/db/tables/operations/create.py @@ -122,7 +122,8 @@ def prepare_table_for_import( ).fetchone()[0] return ( import_info['copy_sql'], - import_info['table_oid'] + import_info['table_oid'], + import_info['table_name'] ) diff --git a/db/tables/operations/import_.py b/db/tables/operations/import_.py index 7929bff2e3..5ece633728 100644 --- a/db/tables/operations/import_.py +++ b/db/tables/operations/import_.py @@ -25,7 +25,7 @@ def import_csv(data_file_id, table_name, schema_oid, conn, comment=None): with open(file_path, 'rb') as csv_file: csv_reader = get_sv_reader(csv_file, header, dialect) column_names = process_column_names(csv_reader.fieldnames) - copy_sql, table_oid = prepare_table_for_import( + copy_sql, table_oid, table_name = prepare_table_for_import( table_name, schema_oid, column_names, @@ -44,7 +44,7 @@ def import_csv(data_file_id, table_name, schema_oid, conn, comment=None): conversion_encoding, conn ) - return table_oid + return {"oid": table_oid, "name": table_name} def insert_csv_records( From 8da3167f33e391e40f8f37846581323c55e8fb63 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 5 Sep 2024 23:11:13 +0530 Subject: [PATCH 0818/1141] add documentation and wire up RPC endpoints --- db/sql/00_msar.sql | 5 +++-- docs/docs/api/rpc.md | 1 + mathesar/rpc/tables/base.py | 24 ++++++++++++++++++------ mathesar/tests/rpc/tables/test_t_base.py | 12 ++++++------ 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 2edd7cb4d8..c47ea237b7 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2907,12 +2907,13 @@ msar.prepare_table_for_import( comment_ text ) RETURNS jsonb AS $$/* Add a table, with a default id column, returning a JSON object containing -a properly formatted SQL statement to carry out `COPY FROM` and also contains table_oid of the created table. +a properly formatted SQL statement to carry out `COPY FROM`, table_oid & table_name of the created table. Each returned JSON object will have the form: { "copy_sql": , - "table_oid": + "table_oid": , + "table_name": } Args: diff --git a/docs/docs/api/rpc.md b/docs/docs/api/rpc.md index 0c63655df4..4610abae4a 100644 --- a/docs/docs/api/rpc.md +++ b/docs/docs/api/rpc.md @@ -127,6 +127,7 @@ To use an RPC function: - get_import_preview - list_joinable - TableInfo + - AddedTableInfo - SettableTableInfo - JoinableTableRecord - JoinableTableInfo diff --git a/mathesar/rpc/tables/base.py b/mathesar/rpc/tables/base.py index 8b971ca9a3..ffc05fc4c5 100644 --- a/mathesar/rpc/tables/base.py +++ b/mathesar/rpc/tables/base.py @@ -51,6 +51,18 @@ class TableInfo(TypedDict): current_role_owns: bool +class AddedTableInfo(TypedDict): + """ + Information about a newly created table. + + Attributes: + oid: The `oid` of the table in the schema. + name: The name of the table. + """ + oid: int + name: str + + class SettableTableInfo(TypedDict): """ Information about a table, restricted to settable fields. @@ -191,7 +203,7 @@ def add( constraint_data_list: list[CreatableConstraintInfo] = [], comment: str = None, **kwargs -) -> int: +) -> AddedTableInfo: """ Add a table with a default id column. @@ -204,7 +216,7 @@ def add( comment: The comment for the new table. Returns: - The `oid` of the created table. + The `oid` & `name` of the created table. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: @@ -264,24 +276,24 @@ def patch( def import_( *, data_file_id: int, - table_name: str, schema_oid: int, database_id: int, + table_name: str = None, comment: str = None, **kwargs -) -> int: +) -> AddedTableInfo: """ Import a CSV/TSV into a table. Args: data_file_id: The Django id of the DataFile containing desired CSV/TSV. - table_name: Name of the table to be imported. schema_oid: Identity of the schema in the user's database. database_id: The Django id of the database containing the table. + table_name: Name of the table to be imported. comment: The comment for the new table. Returns: - The `oid` of the created table. + The `oid` and `name` of the created table. """ user = kwargs.get(REQUEST_KEY).user with connect(database_id, user) as conn: diff --git a/mathesar/tests/rpc/tables/test_t_base.py b/mathesar/tests/rpc/tables/test_t_base.py index 714c1e4af3..2997c8b7e2 100644 --- a/mathesar/tests/rpc/tables/test_t_base.py +++ b/mathesar/tests/rpc/tables/test_t_base.py @@ -148,11 +148,11 @@ def mock_connect(_database_id, user): def mock_table_add(table_name, _schema_oid, conn, column_data_list, constraint_data_list, comment): if _schema_oid != schema_oid: raise AssertionError('incorrect parameters passed') - return 1964474 + return {"oid": 1964474, "name": "newtable"} monkeypatch.setattr(tables.base, 'connect', mock_connect) monkeypatch.setattr(tables.base, 'create_table_on_database', mock_table_add) - actual_table_oid = tables.add(table_name='newtable', schema_oid=2200, database_id=11, request=request) - assert actual_table_oid == 1964474 + actual_table_info = tables.add(table_name='newtable', schema_oid=2200, database_id=11, request=request) + assert actual_table_info == {"oid": 1964474, "name": "newtable"} def test_tables_patch(rf, monkeypatch): @@ -215,17 +215,17 @@ def mock_connect(_database_id, user): def mock_table_import(_data_file_id, table_name, _schema_oid, conn, comment): if _schema_oid != schema_oid and _data_file_id != data_file_id: raise AssertionError('incorrect parameters passed') - return 1964474 + return {"oid": 1964474, "name": "imported_table"} monkeypatch.setattr(tables.base, 'connect', mock_connect) monkeypatch.setattr(tables.base, 'import_csv', mock_table_import) - imported_table_oid = tables.import_( + imported_table_info = tables.import_( data_file_id=10, table_name='imported_table', schema_oid=2200, database_id=11, request=request ) - assert imported_table_oid == 1964474 + assert imported_table_info == {"oid": 1964474, "name": "imported_table"} def test_tables_preview(rf, monkeypatch): From e51e08e8b71c4d72ea5925a6914049c6433329c9 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 5 Sep 2024 23:30:35 +0530 Subject: [PATCH 0819/1141] fix tests and return non stringified oids --- db/sql/00_msar.sql | 4 ++-- db/tables/operations/create.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index c47ea237b7..429f7d5a03 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2887,7 +2887,7 @@ BEGIN created_table_id := fq_table_name::regclass::oid; PERFORM msar.comment_on_table(created_table_id, comment_); RETURN jsonb_build_object( - 'oid', created_table_id, + 'oid', created_table_id::bigint, 'name', created_table_id::regclass::text ); END; @@ -2960,7 +2960,7 @@ BEGIN copy_sql := format('COPY %I.%I (%s) FROM STDIN CSV %s', sch_name, rel_name, col_names_sql, options_sql); RETURN jsonb_build_object( 'copy_sql', copy_sql, - 'table_oid', rel_id, + 'table_oid', rel_id::bigint, 'table_name', rel_id::regclass::text ); END; diff --git a/db/tables/operations/create.py b/db/tables/operations/create.py index c5840dee89..699ac9aaad 100644 --- a/db/tables/operations/create.py +++ b/db/tables/operations/create.py @@ -30,7 +30,7 @@ def create_mathesar_table(engine, table_name, schema_oid, columns=[], constraint json.dumps(columns), json.dumps(constraints), comment - ).fetchone()[0] + ).fetchone()[0]["oid"] def create_table_on_database( @@ -52,7 +52,7 @@ def create_table_on_database( comment: The comment for the new table. (optional) Returns: - Returns the OID of the created table. + Returns the OID and name of the created table. """ return exec_msar_func( conn, From ecf5a425f39fa24a2e6425a70baf4a89779b4ce6 Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Thu, 5 Sep 2024 23:40:01 +0530 Subject: [PATCH 0820/1141] add optional tag for the table_name --- db/sql/00_msar.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 429f7d5a03..3a19c2e34a 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2918,7 +2918,7 @@ Each returned JSON object will have the form: Args: sch_id: The OID of the schema where the table will be created. - tab_name: The unquoted name for the new table. + tab_name (optional): The unquoted name for the new table. col_defs: The columns for the new table, in order. header: Whether or not the file contains a header line with the names of each column in the file. delimiter: The character that separates columns within each row (line) of the file. From ab4f7b6b846a5b27137bb18735d8a04c2bdb655d Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 6 Sep 2024 02:11:10 +0530 Subject: [PATCH 0821/1141] fix twice quoting of name in response --- db/sql/00_msar.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 3a19c2e34a..bb80ff2ae4 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -2888,8 +2888,8 @@ BEGIN PERFORM msar.comment_on_table(created_table_id, comment_); RETURN jsonb_build_object( 'oid', created_table_id::bigint, - 'name', created_table_id::regclass::text - ); + 'name', relname + ) FROM pg_catalog.pg_class WHERE oid = created_table_id; END; $$ LANGUAGE plpgsql; @@ -2961,8 +2961,8 @@ BEGIN RETURN jsonb_build_object( 'copy_sql', copy_sql, 'table_oid', rel_id::bigint, - 'table_name', rel_id::regclass::text - ); + 'table_name', relname + ) FROM pg_catalog.pg_class WHERE oid = rel_id; END; $$ LANGUAGE plpgsql; From 72bc62ba98ce10818c0b9f2fc7dfa54321067699 Mon Sep 17 00:00:00 2001 From: Brent Moran Date: Fri, 6 Sep 2024 11:05:23 +0800 Subject: [PATCH 0822/1141] wire up new record summaries param to RPC endpoint --- mathesar/rpc/records.py | 52 +++++++++++++++------ mathesar/tests/rpc/test_records.py | 73 ++++++++++++++++++++++-------- 2 files changed, 93 insertions(+), 32 deletions(-) diff --git a/mathesar/rpc/records.py b/mathesar/rpc/records.py index 95ea27b861..8558b67b6c 100644 --- a/mathesar/rpc/records.py +++ b/mathesar/rpc/records.py @@ -140,13 +140,16 @@ class RecordList(TypedDict): count: The total number of records in the table. results: An array of record objects. grouping: Information for displaying grouped records. - preview_data: Information for previewing foreign key values, - provides a map of foreign key to a text summary. + linked_record_smmaries: Information for previewing foreign key + values, provides a map of foreign key to a text summary. + record_summaries: Information for previewing returned records. """ count: int results: list[dict] grouping: GroupingResponse - preview_data: dict[str, dict[str, str]] + linked_record_summaries: dict[str, dict[str, str]] + record_summaries: dict[str, str] + query: str @classmethod def from_dict(cls, d): @@ -154,7 +157,8 @@ def from_dict(cls, d): count=d["count"], results=d["results"], grouping=d.get("grouping"), - preview_data=d.get("preview_data"), + linked_record_summaries=d.get("linked_record_summaries"), + record_summaries=d.get("record_summaries"), query=d["query"], ) @@ -170,17 +174,20 @@ class RecordAdded(TypedDict): Attributes: results: An array of a single record objects (the one added). - preview_data: Information for previewing foreign key values, - provides a map of foreign key to a text summary. + linked_record_summaries: Information for previewing foreign key + values, provides a map of foreign key to a text summary. + record_summaries: Information for previewing an added record. """ results: list[dict] - preview_data: dict[str, dict[str, str]] + linked_record_summaries: dict[str, dict[str, str]] + record_summaries: dict[str, str] @classmethod def from_dict(cls, d): return cls( results=d["results"], - preview_data=d.get("preview_data"), + linked_record_summaries=d.get("linked_record_summaries"), + record_summaries=d.get("record_summaries"), ) @@ -196,6 +203,7 @@ def list_( order: list[OrderBy] = None, filter: Filter = None, grouping: Grouping = None, + return_record_summaries: bool = False, **kwargs ) -> RecordList: """ @@ -206,10 +214,12 @@ def list_( database_id: The Django id of the database containing the table. limit: The maximum number of rows we'll return. offset: The number of rows to skip before returning records from - following rows. + following rows. order: An array of ordering definition objects. filter: An array of filter definition objects. grouping: An array of group definition objects. + return_record_summaries: Whether to return summaries of retrieved + records. Returns: The requested records, along with some metadata. @@ -224,6 +234,7 @@ def list_( order=order, filter=filter, group=grouping, + return_record_summaries=return_record_summaries, ) return RecordList.from_dict(record_info) @@ -236,6 +247,7 @@ def get( record_id: Any, table_oid: int, database_id: int, + return_record_summaries: bool = False, **kwargs ) -> RecordList: """ @@ -245,6 +257,8 @@ def get( record_id: The primary key value of the record to be gotten. table_oid: Identity of the table in the user's database. database_id: The Django id of the database containing the table. + return_record_summaries: Whether to return summaries of the + retrieved record. Returns: The requested record, along with some metadata. @@ -255,6 +269,7 @@ def get( conn, record_id, table_oid, + return_record_summaries=return_record_summaries ) return RecordList.from_dict(record_info) @@ -267,6 +282,7 @@ def add( record_def: dict, table_oid: int, database_id: int, + return_record_summaries: bool = False, **kwargs ) -> RecordAdded: """ @@ -283,6 +299,8 @@ def add( record_def: An object representing the record to be added. table_oid: Identity of the table in the user's database. database_id: The Django id of the database containing the table. + return_record_summaries: Whether to return summaries of the added + record. Returns: The created record, along with some metadata. @@ -293,6 +311,7 @@ def add( conn, record_def, table_oid, + return_record_summaries=return_record_summaries, ) return RecordAdded.from_dict(record_info) @@ -306,21 +325,25 @@ def patch( record_id: Any, table_oid: int, database_id: int, + return_record_summaries: bool = False, **kwargs ) -> RecordAdded: """ Modify a record in a table. - The form of the `record_def` is determined by the underlying table. Keys - should be attnums, and values should be the desired value for that column in - the modified record. Explicit `null` values will set null for that value - (with obvious exceptions where that would violate some constraint). + The form of the `record_def` is determined by the underlying table. + Keys should be attnums, and values should be the desired value for + that column in the modified record. Explicit `null` values will set + null for that value (with obvious exceptions where that would violate + some constraint). Args: record_def: An object representing the record to be added. record_id: The primary key value of the record to modify. table_oid: Identity of the table in the user's database. database_id: The Django id of the database containing the table. + return_record_summaries: Whether to return summaries of the + modified record. Returns: The modified record, along with some metadata. @@ -332,6 +355,7 @@ def patch( record_def, record_id, table_oid, + return_record_summaries=return_record_summaries, ) return RecordAdded.from_dict(record_info) @@ -376,6 +400,7 @@ def search( database_id: int, search_params: list[SearchParam] = [], limit: int = 10, + return_record_summaries: bool = False, **kwargs ) -> RecordList: """ @@ -405,5 +430,6 @@ def search( table_oid, search=search_params, limit=limit, + return_record_summaries=return_record_summaries, ) return RecordList.from_dict(record_info) diff --git a/mathesar/tests/rpc/test_records.py b/mathesar/tests/rpc/test_records.py index 1b90c7366e..a2dde6b6ba 100644 --- a/mathesar/tests/rpc/test_records.py +++ b/mathesar/tests/rpc/test_records.py @@ -37,8 +37,9 @@ def mock_list_records( order=None, filter=None, group=None, + return_record_summaries=False, ): - if _table_oid != table_oid: + if _table_oid != table_oid or return_record_summaries is False: raise AssertionError('incorrect parameters passed') return { "count": 50123, @@ -50,7 +51,8 @@ def mock_list_records( {"id": 3, "count": 8, "results_eq": {"1": "lsfj", "2": 3422}} ] }, - "preview_data": {"2": {"12345": "blkjdfslkj"}} + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"} } monkeypatch.setattr(records, 'connect', mock_connect) @@ -64,11 +66,15 @@ def mock_list_records( {"id": 3, "count": 8, "results_eq": {"1": "lsfj", "2": 3422}} ] }, - "preview_data": {"2": {"12345": "blkjdfslkj"}}, + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"}, "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', } actual_records_list = records.list_( - table_oid=table_oid, database_id=database_id, request=request + table_oid=table_oid, + database_id=database_id, + return_record_summaries=True, + request=request ) assert actual_records_list == expect_records_list @@ -96,15 +102,17 @@ def mock_get_record( conn, _record_id, _table_oid, + return_record_summaries=False, ): - if _table_oid != table_oid or _record_id != record_id: + if _table_oid != table_oid or _record_id != record_id or return_record_summaries is False: raise AssertionError('incorrect parameters passed') return { "count": 1, "results": [{"1": "abcde", "2": 12345}, {"1": "fghij", "2": 67890}], "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', "grouping": None, - "preview_data": {"2": {"12345": "blkjdfslkj"}}, + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"}, } monkeypatch.setattr(records, 'connect', mock_connect) @@ -113,11 +121,16 @@ def mock_get_record( "count": 1, "results": [{"1": "abcde", "2": 12345}, {"1": "fghij", "2": 67890}], "grouping": None, - "preview_data": {"2": {"12345": "blkjdfslkj"}}, + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"}, "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', } actual_record = records.get( - record_id=record_id, table_oid=table_oid, database_id=database_id, request=request + record_id=record_id, + table_oid=table_oid, + database_id=database_id, + return_record_summaries=True, + request=request ) assert actual_record == expect_record @@ -145,22 +158,29 @@ def mock_add_record( conn, _record_def, _table_oid, + return_record_summaries=False, ): - if _table_oid != table_oid or _record_def != record_def: + if _table_oid != table_oid or _record_def != record_def or return_record_summaries is False: raise AssertionError('incorrect parameters passed') return { "results": [_record_def], - "preview_data": {"2": {"12345": "blkjdfslkj"}}, + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"}, } monkeypatch.setattr(records, 'connect', mock_connect) monkeypatch.setattr(records.record_insert, 'add_record_to_table', mock_add_record) expect_record = { "results": [record_def], - "preview_data": {"2": {"12345": "blkjdfslkj"}}, + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"}, } actual_record = records.add( - record_def=record_def, table_oid=table_oid, database_id=database_id, request=request + record_def=record_def, + table_oid=table_oid, + database_id=database_id, + return_record_summaries=True, + request=request ) assert actual_record == expect_record @@ -190,25 +210,34 @@ def mock_patch_record( _record_def, _record_id, _table_oid, + return_record_summaries=False, ): - if _table_oid != table_oid or _record_def != record_def or _record_id != record_id: + if ( + _table_oid != table_oid + or _record_def != record_def + or _record_id != record_id + or return_record_summaries is False + ): raise AssertionError('incorrect parameters passed') return { "results": [_record_def | {"3": "another"}], - "preview_data": {"2": {"12345": "blkjdfslkj"}}, + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"}, } monkeypatch.setattr(records, 'connect', mock_connect) monkeypatch.setattr(records.record_update, 'patch_record_in_table', mock_patch_record) expect_record = { "results": [record_def | {"3": "another"}], - "preview_data": {"2": {"12345": "blkjdfslkj"}}, + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"}, } actual_record = records.patch( record_def=record_def, record_id=record_id, table_oid=table_oid, database_id=database_id, + return_record_summaries=True, request=request ) assert actual_record == expect_record @@ -274,13 +303,15 @@ def mock_search_records( _table_oid, search=[], limit=10, + return_record_summaries=False, ): - if _table_oid != table_oid: + if _table_oid != table_oid or return_record_summaries is False: raise AssertionError('incorrect parameters passed') return { "count": 50123, "results": [{"1": "abcde", "2": 12345}, {"1": "fghij", "2": 67890}], - "preview_data": {"2": {"12345": "blkjdfslkj"}}, + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"}, "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', } @@ -290,10 +321,14 @@ def mock_search_records( "count": 50123, "results": [{"1": "abcde", "2": 12345}, {"1": "fghij", "2": 67890}], "grouping": None, - "preview_data": {"2": {"12345": "blkjdfslkj"}}, + "linked_record_summaries": {"2": {"12345": "blkjdfslkj"}}, + "record_summaries": {"3": "abcde"}, "query": 'SELECT mycol AS "1", anothercol AS "2" FROM mytable LIMIT 2', } actual_records_list = records.search( - table_oid=table_oid, database_id=database_id, request=request + table_oid=table_oid, + database_id=database_id, + return_record_summaries=True, + request=request ) assert actual_records_list == expect_records_list From 930c81e16d706ad406be0caa7d760b474b9e904f Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Fri, 6 Sep 2024 19:09:33 +0530 Subject: [PATCH 0823/1141] test moving columns when having a single or multicol fk reference --- db/sql/00_msar.sql | 7 +-- db/sql/test_00_msar.sql | 115 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 118 insertions(+), 4 deletions(-) diff --git a/db/sql/00_msar.sql b/db/sql/00_msar.sql index 70a3c5abd1..bfd181ccb3 100644 --- a/db/sql/00_msar.sql +++ b/db/sql/00_msar.sql @@ -3899,9 +3899,10 @@ DECLARE move_con_defs CONSTANT jsonb := msar.get_extracted_con_def_jsonb(source_tab_id, move_col_ids); added_col_ids smallint[]; BEGIN - -- TODO test to make sure no multi-col fkeys reference the moved columns - -- TODO just throw error if _any_ multicol constraint references the moved columns. - -- TODO check behavior if one of the moving columns is referenced by another table (should raise) + -- TODO Add a custom validator that throws pretty errors in these scenario: + -- test to make sure no multi-col fkeys reference the moved columns + -- just throw error if _any_ multicol constraint references the moved columns. + -- check behavior if one of the moving columns is referenced by another table (should raise) SELECT conkey, confkey INTO source_join_col_id, target_join_col_id FROM msar.get_fkey_map_table(source_tab_id) WHERE target_oid = target_tab_id; diff --git a/db/sql/test_00_msar.sql b/db/sql/test_00_msar.sql index e86e6c012e..67a4849895 100644 --- a/db/sql/test_00_msar.sql +++ b/db/sql/test_00_msar.sql @@ -5087,8 +5087,121 @@ BEGIN (5, NULL, 'Abu''l-Fazl', NULL, 5, 1), (6, 'Joe', 'Abercrombie', 'https://joeabercrombie.com/', 5, 2), (7, 'Joe', 'Abercrombie', 'https://joeabercrombie.com/',NULL,1), - (8, 'Daniel', 'Abraham', 'https://www.danielabraham.com/', 5, 2); + (8, 'Daniel', 'Abraham', 'https://www.danielabraham.com/', 5, 2), + (9, 'Daniel', 'Abraham', 'https://www.danielabraham.com/', 5, NULL), + (10, NULL, 'Abu''l-Fazl', NULL, 10, 1); $w$ ); END; $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION __setup_move_columns_multicol_fk() RETURNS SETOF TEXT AS $$ +BEGIN +CREATE TABLE target_table( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + x text, + y integer +); +CREATE TABLE source_table( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + c1 integer, + c2 integer, + c3 integer, + c4 integer REFERENCES target_table(id), + UNIQUE (c1, c2) +); +CREATE TABLE t1 ( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + a integer, + b integer, + c integer, + FOREIGN KEY (b, c) REFERENCES source_table (c1, c2) +); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_move_columns_not_referenced_by_multicol_fk() RETURNS SETOF TEXT AS $$ +BEGIN + PERFORM __setup_move_columns_multicol_fk(); + PERFORM msar.move_columns_to_referenced_table( + 'source_table'::regclass, 'target_table'::regclass, ARRAY[4]::smallint[] + ); + RETURN NEXT columns_are( + 'source_table', + ARRAY['id', 'c1', 'c2', 'c4'] + ); + RETURN NEXT columns_are( + 'target_table', + ARRAY['id', 'x', 'y', 'c3'] + ); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_move_columns_referenced_by_multicol_fk() RETURNS SETOF TEXT AS $$ +BEGIN + PERFORM __setup_move_columns_multicol_fk(); + RETURN NEXT throws_ok( + $w$SELECT msar.move_columns_to_referenced_table( + 'source_table'::regclass, 'target_table'::regclass, ARRAY[2, 3, 4]::smallint[] + );$w$, + '2BP01', + 'cannot drop column c1 of table source_table because other objects depend on it' + ); + RETURN NEXT columns_are( + 'source_table', + ARRAY['id', 'c1', 'c2', 'c3', 'c4'] + ); + RETURN NEXT columns_are( + 'target_table', + ARRAY['id', 'x', 'y'] + ); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION __setup_move_columns_singlecol_fk() RETURNS SETOF TEXT AS $$ +BEGIN +CREATE TABLE target_table( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + x text, + y integer +); +CREATE TABLE source_table( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + c1 integer, + c2 integer REFERENCES target_table(id), + UNIQUE (c1) +); +CREATE TABLE t1 ( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + a integer, + b integer, + FOREIGN KEY (b) REFERENCES source_table (c1) +); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_move_columns_referenced_by_singlecol_fk() RETURNS SETOF TEXT AS $$ +BEGIN + PERFORM __setup_move_columns_singlecol_fk(); + RETURN NEXT throws_ok( + $w$SELECT msar.move_columns_to_referenced_table( + 'source_table'::regclass, 'target_table'::regclass, ARRAY[2]::smallint[] + );$w$, + '2BP01', + 'cannot drop column c1 of table source_table because other objects depend on it' + ); + RETURN NEXT columns_are( + 'source_table', + ARRAY['id', 'c1', 'c2'] + ); + RETURN NEXT columns_are( + 'target_table', + ARRAY['id', 'x', 'y'] + ); +END; +$$ LANGUAGE plpgsql; From 664c57690db3bf849af32a8fe2fd04353c46d59f Mon Sep 17 00:00:00 2001 From: pavish Date: Mon, 9 Sep 2024 19:47:11 +0530 Subject: [PATCH 0824/1141] Allow setting custom privileges without modifying access level and retain data when privileges are removed --- mathesar_ui/src/api/rpc/databases.ts | 9 +- .../checkbox-group/CheckboxGroup.svelte | 7 +- .../collapsible/Collapsible.scss | 2 +- .../component-library/tabs/TabContainer.scss | 4 +- .../DatabasePermissionsModal.svelte | 8 +- .../permissions/DirectPrivilegeRow.svelte | 91 +++++++----- .../permissions/PrivilegesSection.svelte | 47 ++++-- .../src/pages/database/permissions/utils.ts | 135 ++++++++++++++++++ 8 files changed, 249 insertions(+), 54 deletions(-) create mode 100644 mathesar_ui/src/pages/database/permissions/utils.ts diff --git a/mathesar_ui/src/api/rpc/databases.ts b/mathesar_ui/src/api/rpc/databases.ts index 38855fe6bd..6b4e567ca5 100644 --- a/mathesar_ui/src/api/rpc/databases.ts +++ b/mathesar_ui/src/api/rpc/databases.ts @@ -1,5 +1,4 @@ import { rpcMethodTypeContainer } from '@mathesar/packages/json-rpc-client-builder'; -import { ImmutableSet } from '@mathesar-component-library'; import type { RawConfiguredRole, RawRole } from './roles'; import type { RawServer } from './servers'; @@ -10,8 +9,12 @@ export interface RawDatabase { server_id: RawServer['id']; } -export const databasePrivileges = ['CREATE', 'CONNECT', 'TEMPORARY'] as const; -export type DatabasePrivilege = (typeof databasePrivileges)[number]; +export const allDatabasePrivileges = [ + 'CREATE', + 'CONNECT', + 'TEMPORARY', +] as const; +export type DatabasePrivilege = (typeof allDatabasePrivileges)[number]; export interface RawUnderlyingDatabase { oid: number; diff --git a/mathesar_ui/src/component-library/checkbox-group/CheckboxGroup.svelte b/mathesar_ui/src/component-library/checkbox-group/CheckboxGroup.svelte index 9ed775b439..77a37add71 100644 --- a/mathesar_ui/src/component-library/checkbox-group/CheckboxGroup.svelte +++ b/mathesar_ui/src/component-library/checkbox-group/CheckboxGroup.svelte @@ -1,5 +1,5 @@ diff --git a/mathesar_ui/src/component-library/collapsible/Collapsible.scss b/mathesar_ui/src/component-library/collapsible/Collapsible.scss index 84064edf4d..2a5208b418 100644 --- a/mathesar_ui/src/component-library/collapsible/Collapsible.scss +++ b/mathesar_ui/src/component-library/collapsible/Collapsible.scss @@ -15,7 +15,7 @@ .collapsible-header-title { flex-grow: 1; overflow: hidden; - font-weight: 600; + font-weight: var(--Collapsible_header-font-weight, 600); } svg { flex-shrink: 0; diff --git a/mathesar_ui/src/component-library/tabs/TabContainer.scss b/mathesar_ui/src/component-library/tabs/TabContainer.scss index 3f8f2900af..7ad38e1266 100644 --- a/mathesar_ui/src/component-library/tabs/TabContainer.scss +++ b/mathesar_ui/src/component-library/tabs/TabContainer.scss @@ -25,7 +25,7 @@ border-radius: 2px 2px 0px 0px; border-bottom: 2px solid transparent; margin-bottom: -1px; - margin-right: 2rem; + margin-right: var(--Tab_margin-right, 2rem); font-size: var(--text-size-large); > div, @@ -113,7 +113,7 @@ > li.tab { font-size: var(--text-size-base); text-align: center; - margin-right: 0; + margin-right: var(--Tab_margin-right, 0); > div { padding: 0.5rem 0; } diff --git a/mathesar_ui/src/pages/database/permissions/DatabasePermissionsModal.svelte b/mathesar_ui/src/pages/database/permissions/DatabasePermissionsModal.svelte index b4b5c9cdf6..b7f3572d36 100644 --- a/mathesar_ui/src/pages/database/permissions/DatabasePermissionsModal.svelte +++ b/mathesar_ui/src/pages/database/permissions/DatabasePermissionsModal.svelte @@ -38,7 +38,7 @@ {$_('database_permissions')} -
+
+ + diff --git a/mathesar_ui/src/pages/database/permissions/DirectPrivilegeRow.svelte b/mathesar_ui/src/pages/database/permissions/DirectPrivilegeRow.svelte index 738314cce4..a58691e944 100644 --- a/mathesar_ui/src/pages/database/permissions/DirectPrivilegeRow.svelte +++ b/mathesar_ui/src/pages/database/permissions/DirectPrivilegeRow.svelte @@ -1,7 +1,6 @@
@@ -65,18 +85,18 @@
entry.id), customAccess]} @@ -120,20 +111,9 @@ display: grid; grid-template-columns: 2fr 1fr auto; gap: var(--size-ultra-small); - - .name-and-members { - .name { - span { - padding: var(--size-extreme-small) var(--size-xx-small); - background: var(--slate-100); - border-radius: var(--border-radius-xl); - font-weight: 500; - } - } - } } .role-permissions-section { - margin-top: var(--size-super-ultra-small); + margin-top: var(--size-xx-small); --Collapsible_trigger-padding: var(--size-extreme-small) 0; --Collapsible_header-font-weight: 400; diff --git a/mathesar_ui/src/systems/permissions/PermissionsModal.svelte b/mathesar_ui/src/systems/permissions/PermissionsModal.svelte new file mode 100644 index 0000000000..7d3cef3d1b --- /dev/null +++ b/mathesar_ui/src/systems/permissions/PermissionsModal.svelte @@ -0,0 +1,55 @@ + + + + +
+ +
+ {#if activeTab.id === 'share'} + + {:else} + + {/if} +
+
+
+
+ + diff --git a/mathesar_ui/src/pages/database/permissions/PrivilegesSection.svelte b/mathesar_ui/src/systems/permissions/PrivilegesSection.svelte similarity index 53% rename from mathesar_ui/src/pages/database/permissions/PrivilegesSection.svelte rename to mathesar_ui/src/systems/permissions/PrivilegesSection.svelte index 21c520dec8..fc5e920326 100644 --- a/mathesar_ui/src/pages/database/permissions/PrivilegesSection.svelte +++ b/mathesar_ui/src/systems/permissions/PrivilegesSection.svelte @@ -1,20 +1,13 @@
{#if isLoading} - {:else if isSuccess && $roles.resolvedValue} + {:else if isSuccess && $roles.resolvedValue && $objectOwnerAndCurrentRolePrivileges.resolvedValue}
{$_('owner')}
-
+
+ +
{$_('granted_access')}
- {#each dbPrivilegesWithAccess as roleAccessLevelAndPrivileges (roleAccessLevelAndPrivileges.roleOid)} + {#each objectPrivilegesWithAccess as roleAccessLevelAndPrivileges (roleAccessLevelAndPrivileges.roleOid)}
@@ -130,18 +123,28 @@ diff --git a/mathesar_ui/src/pages/database/permissions/TransferOwnershipSection.svelte b/mathesar_ui/src/systems/permissions/TransferOwnershipSection.svelte similarity index 100% rename from mathesar_ui/src/pages/database/permissions/TransferOwnershipSection.svelte rename to mathesar_ui/src/systems/permissions/TransferOwnershipSection.svelte diff --git a/mathesar_ui/src/systems/permissions/utils.ts b/mathesar_ui/src/systems/permissions/utils.ts new file mode 100644 index 0000000000..6be070a139 --- /dev/null +++ b/mathesar_ui/src/systems/permissions/utils.ts @@ -0,0 +1,150 @@ +import type { Readable } from 'svelte/store'; + +import type { Role } from '@mathesar/models/Role'; +import type { AsyncStoreValue } from '@mathesar/stores/AsyncStore'; +import { type ImmutableMap, ImmutableSet } from '@mathesar-component-library'; + +export type AccessLevelConfig = { id: A; privileges: Set

}; + +export const customAccess = 'custom' as const; + +function getAccessLevelBasedOnPrivileges( + accessLevelConfigs: AccessLevelConfig[], + privileges: ImmutableSet

, +): A | typeof customAccess { + const accessLevelObject = accessLevelConfigs.find((entry) => + privileges.equals(entry.privileges), + ); + return accessLevelObject ? accessLevelObject.id : customAccess; +} + +type Props = { + roleOid: Role['oid']; + accessLevelConfigs: AccessLevelConfig[]; + isAccessRemoved?: boolean; + savedPrivileges: P[]; +} & ( + | { + privileges: P[]; + } + | { + accessLevel: A; + } + | { + privileges: P[]; + accessLevel: typeof customAccess; + } + | { + accessLevel: undefined; + privileges: []; + } +); + +export class RoleAccessLevelAndPrivileges { + readonly roleOid; + + readonly accessLevelConfigs; + + readonly accessLevel: A | typeof customAccess | undefined; + + readonly privileges: ImmutableSet

; + + readonly savedPrivileges: P[]; + + constructor(props: Props) { + this.accessLevelConfigs = props.accessLevelConfigs; + this.roleOid = props.roleOid; + this.savedPrivileges = props.savedPrivileges; + if ('privileges' in props && 'accessLevel' in props) { + this.accessLevel = props.accessLevel; + this.privileges = new ImmutableSet(props.privileges); + } else if ('privileges' in props) { + this.privileges = new ImmutableSet(props.privileges); + this.accessLevel = getAccessLevelBasedOnPrivileges( + this.accessLevelConfigs, + this.privileges, + ); + } else { + this.accessLevel = props.accessLevel; + const aL = this.accessLevelConfigs.find( + (entry) => entry.id === this.accessLevel, + ); + if (!aL) { + throw new Error( + 'Access level not found in configuration. This should never occur.', + ); + } + this.privileges = new ImmutableSet(aL.privileges); + } + } + + private getCommonProps() { + return { + roleOid: this.roleOid, + accessLevelConfigs: this.accessLevelConfigs, + savedPrivileges: this.savedPrivileges, + }; + } + + withAccess(accessLevel: A) { + return new RoleAccessLevelAndPrivileges({ + ...this.getCommonProps(), + accessLevel, + }); + } + + withCustomAccess(privileges?: P[]) { + const customPrivileges = privileges ?? this.savedPrivileges; + return new RoleAccessLevelAndPrivileges({ + ...this.getCommonProps(), + accessLevel: customAccess, + privileges: customPrivileges, + }); + } + + withAccessRemoved() { + return new RoleAccessLevelAndPrivileges({ + ...this.getCommonProps(), + accessLevel: undefined, + privileges: [], + }); + } +} + +export interface RoleWithPrivileges { + role_oid: Role['oid']; + direct: Privilege[]; +} + +export function getObjectAccessPrivilegeMap( + accessLevelConfigs: AccessLevelConfig[], + dbPrivileges: ImmutableMap>, +) { + return dbPrivileges.mapValues( + (entry) => + new RoleAccessLevelAndPrivileges({ + roleOid: entry.role_oid, + accessLevelConfigs, + privileges: entry.direct, + savedPrivileges: entry.direct, + }), + ); +} + +export interface AsyncStoresValues { + roles: Readable, string>>; + objectPrivileges: Readable< + AsyncStoreValue>, string> + >; + objectOwnerAndCurrentRolePrivileges: Readable< + AsyncStoreValue< + { + oid: number; + owner_oid: Role['oid']; + current_role_priv: Privilege[]; + current_role_owns: boolean; + }, + string + > + >; +} From 88b91c65a4a500d90b5631c98dabbac099ac848e Mon Sep 17 00:00:00 2001 From: Anish Umale Date: Tue, 10 Sep 2024 18:22:57 +0530 Subject: [PATCH 0837/1141] add data_file_id field to TableMetaData --- .../0015_tablemetadata_data_file.py | 19 +++++++++++++++++++ mathesar/models/base.py | 1 + mathesar/rpc/tables/metadata.py | 14 +++++--------- mathesar/tests/rpc/tables/test_t_metadata.py | 9 +++++---- mathesar/utils/tables.py | 5 ++++- 5 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 mathesar/migrations/0015_tablemetadata_data_file.py diff --git a/mathesar/migrations/0015_tablemetadata_data_file.py b/mathesar/migrations/0015_tablemetadata_data_file.py new file mode 100644 index 0000000000..ab9c0e7eaf --- /dev/null +++ b/mathesar/migrations/0015_tablemetadata_data_file.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.11 on 2024-09-10 12:25 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('mathesar', '0014_remove_columnmetadata_duration_show_units_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='tablemetadata', + name='data_file', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='mathesar.datafile'), + ), + ] diff --git a/mathesar/models/base.py b/mathesar/models/base.py index d4502f4414..797665102b 100644 --- a/mathesar/models/base.py +++ b/mathesar/models/base.py @@ -132,6 +132,7 @@ class Meta: class TableMetaData(BaseModel): database = models.ForeignKey('Database', on_delete=models.CASCADE) table_oid = models.PositiveBigIntegerField() + data_file = models.ForeignKey("DataFile", on_delete=models.SET_NULL, null=True) import_verified = models.BooleanField(null=True) column_order = models.JSONField(null=True) record_summary_customized = models.BooleanField(null=True) diff --git a/mathesar/rpc/tables/metadata.py b/mathesar/rpc/tables/metadata.py index 7a6481b81b..8cc076c49f 100644 --- a/mathesar/rpc/tables/metadata.py +++ b/mathesar/rpc/tables/metadata.py @@ -20,6 +20,7 @@ class TableMetaDataRecord(TypedDict): id: The Django id of the TableMetaData object. database_id: The Django id of the database containing the table. table_oid: The OID of the table in the database. + data_file_id: Specifies the DataFile model id used for the import. import_verified: Specifies whether a file has been successfully imported into a table. column_order: The order in which columns of a table are displayed. record_summary_customized: Specifies whether the record summary has been customized. @@ -28,6 +29,7 @@ class TableMetaDataRecord(TypedDict): id: int database_id: int table_oid: int + data_file_id: Optional[int] import_verified: Optional[bool] column_order: Optional[list[int]] record_summary_customized: Optional[bool] @@ -39,6 +41,7 @@ def from_model(cls, model): id=model.id, database_id=model.database.id, table_oid=model.table_oid, + data_file_id=model.data_file.id if model.data_file is not None else None, import_verified=model.import_verified, column_order=model.column_order, record_summary_customized=model.record_summary_customized, @@ -51,25 +54,18 @@ class TableMetaDataBlob(TypedDict): The metadata fields which can be set on a table Attributes: + data_file_id: Specifies the DataFile model id used for the import. import_verified: Specifies whether a file has been successfully imported into a table. column_order: The order in which columns of a table are displayed. record_summary_customized: Specifies whether the record summary has been customized. record_summary_template: Record summary template for a referent column. """ + data_file_id: Optional[int] import_verified: Optional[bool] column_order: Optional[list[int]] record_summary_customized: Optional[bool] record_summary_template: Optional[str] - @classmethod - def from_model(cls, model): - return cls( - import_verified=model.import_verified, - column_order=model.column_order, - record_summary_customized=model.record_summary_customized, - record_summary_template=model.record_summary_template, - ) - @rpc_method(name="tables.metadata.list") @http_basic_auth_login_required diff --git a/mathesar/tests/rpc/tables/test_t_metadata.py b/mathesar/tests/rpc/tables/test_t_metadata.py index c25753ffbc..342e8e0511 100644 --- a/mathesar/tests/rpc/tables/test_t_metadata.py +++ b/mathesar/tests/rpc/tables/test_t_metadata.py @@ -5,6 +5,7 @@ monkeypatch(pytest): Lets you monkeypatch an object for testing. """ from mathesar.models.base import TableMetaData, Database, Server +from mathesar.models.deprecated import DataFile from mathesar.rpc.tables import metadata @@ -16,12 +17,12 @@ def mock_get_tables_meta_data(_database_id): db_model = Database(id=_database_id, name='mymathesardb', server=server_model) return [ TableMetaData( - id=1, database=db_model, table_oid=1234, + id=1, database=db_model, table_oid=1234, data_file=None, import_verified=True, column_order=[8, 9, 10], record_summary_customized=False, record_summary_template="{5555}" ), TableMetaData( - id=2, database=db_model, table_oid=4567, + id=2, database=db_model, table_oid=4567, data_file=DataFile(id=11), import_verified=False, column_order=[], record_summary_customized=True, record_summary_template="{5512} {1223}" ) @@ -30,12 +31,12 @@ def mock_get_tables_meta_data(_database_id): expect_metadata_list = [ metadata.TableMetaDataRecord( - id=1, database_id=database_id, table_oid=1234, + id=1, database_id=database_id, table_oid=1234, data_file_id=None, import_verified=True, column_order=[8, 9, 10], record_summary_customized=False, record_summary_template="{5555}" ), metadata.TableMetaDataRecord( - id=2, database_id=database_id, table_oid=4567, + id=2, database_id=database_id, table_oid=4567, data_file_id=11, import_verified=False, column_order=[], record_summary_customized=True, record_summary_template="{5512} {1223}" ) diff --git a/mathesar/utils/tables.py b/mathesar/utils/tables.py index 8c0664fa63..35d086c263 100644 --- a/mathesar/utils/tables.py +++ b/mathesar/utils/tables.py @@ -4,7 +4,7 @@ from db.tables.operations.infer_types import infer_table_column_types from mathesar.database.base import create_mathesar_engine from mathesar.imports.base import create_table_from_data_file -from mathesar.models.deprecated import Table +from mathesar.models.deprecated import Table, DataFile from mathesar.models.base import Database, TableMetaData from mathesar.state.django import reflect_columns_from_tables from mathesar.state import get_cached_metadata @@ -93,6 +93,9 @@ def get_tables_meta_data(database_id): def set_table_meta_data(table_oid, metadata, database_id): TableMetaData.objects.update_or_create( database=Database.objects.get(id=database_id), + data_file=DataFile.objects.get( + id=metadata.pop('data_file_id') + ) if metadata.get('data_file_id') is not None else None, table_oid=table_oid, defaults=metadata, ) From deceb10a54d6a95cf1c0b0316094a1252e255852 Mon Sep 17 00:00:00 2001 From: Ghislaine Guerin Date: Wed, 4 Sep 2024 15:57:01 +0200 Subject: [PATCH 0838/1141] Table Inspector Improvements --- mathesar_ui/src/App.svelte | 2 +- .../src/component-library/button/Button.scss | 12 +-- .../common/styles/variables.scss | 23 +++--- .../labeled-input/LabeledInput.scss | 11 +++ .../labeled-input/LabeledInput.svelte | 5 ++ .../src/components/LinkedRecord.svelte | 1 + .../inspector/cell/CellInspector.svelte | 2 +- .../message-boxes/WarningBox.svelte | 2 +- mathesar_ui/src/i18n/languages/en/dict.json | 10 ++- .../column/ColumnNameAndDescription.svelte | 9 ++- .../column/ColumnOptions.svelte | 12 +-- .../record-summary/AppendColumn.svelte | 2 +- .../record-summary/RecordSummaryConfig.svelte | 76 +++++++++++-------- .../record-summary/TemplateInput.svelte | 1 + .../table/TableDescription.svelte | 5 +- .../table-inspector/table/TableName.svelte | 5 +- .../table/links/LinksSectionContainer.svelte | 2 +- 17 files changed, 112 insertions(+), 68 deletions(-) diff --git a/mathesar_ui/src/App.svelte b/mathesar_ui/src/App.svelte index 6c91c2cdba..a546ea39a7 100644 --- a/mathesar_ui/src/App.svelte +++ b/mathesar_ui/src/App.svelte @@ -53,7 +53,7 @@ --color-contrast-light: var(--color-blue-light); --color-link: var(--color-blue-dark); --color-text: #171717; - --color-text-muted: #5e6471; + --color-text-muted: #515662; --color-substring-match: rgb(254, 221, 72); --color-substring-match-light: rgba(254, 221, 72, 0.2); --text-size-xx-small: var(--size-xx-small); diff --git a/mathesar_ui/src/component-library/button/Button.scss b/mathesar_ui/src/component-library/button/Button.scss index f863e9b5c4..9cd2322834 100644 --- a/mathesar_ui/src/component-library/button/Button.scss +++ b/mathesar_ui/src/component-library/button/Button.scss @@ -13,7 +13,7 @@ #000 0 0 0 0, #000 0 0 0 0, rgba(0, 0, 0, 0.05) 0 1px 2px 0; - font-weight: inherit; + font-weight: var(--font-weight-medium); position: relative; padding: var(--input-padding); box-sizing: border-box; @@ -31,11 +31,11 @@ /* Appearances */ &.btn-default { background: var(--color-white); - border-color: var(--slate-200); + border-color: var(--slate-300); &:not(:disabled) { &:focus { - border-color: var(--slate-300); + border-color: var(--slate-400); } } } @@ -68,20 +68,20 @@ &.btn-secondary { background: var(--slate-100); - border-color: var(--slate-200); + border-color: var(--slate-300); color: var(--slate-800); &:not(:disabled) { &.focus, &:focus { - outline: 2px solid var(--slate-300); + outline: 2px solid var(--slate-400); outline-offset: 1px; } &:hover, &.hover { background: var(--slate-200); - border-color: var(--slate-300); + border-color: var(--slate-400); } &:active, diff --git a/mathesar_ui/src/component-library/common/styles/variables.scss b/mathesar_ui/src/component-library/common/styles/variables.scss index 159385f4ac..4495197957 100644 --- a/mathesar_ui/src/component-library/common/styles/variables.scss +++ b/mathesar_ui/src/component-library/common/styles/variables.scss @@ -36,26 +36,27 @@ --slate-800: #25292e; --sand-50: #fcfbf8; - --sand-100: #f9f8f6; - --sand-200: #efece7; - --sand-300: #e2dcd4; - --sand-400: #cec5b6; + --sand-100: #F9F8F5; + --sand-200: #EBE3D1; + --sand-300: #d7cab2; + --sand-400: #c6b39c; --sand-800: #776850; - --sky-200: #e8f1fd; - --sky-300: #b0ccf0; - --sky-500: #bcd9f9; - --sky-600: #1269cf; - --sky-700: #145bbe; + --sky-200: #d5ecfc; + --sky-300: #91d0f8; + --sky-500: #7ac5f4; + --sky-600: #0777d2; + --sky-700: #0662be; --sky-800: #36537a; // TODO: Get this incorporated in figma --slate-50-tentative: #f6f9f9; --sky-600-tentative: #87bcf6; - --yellow-100: #fdf7ed; + --yellow-100: #fff4d5; --yellow-200: #ffedaf; - --yellow-300: #f8ceb0; + --yellow-300: #f4c67c; + --yellow-400: #eda849; --white: white; diff --git a/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss b/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss index 9d5c482eb2..7db9af8b01 100644 --- a/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss +++ b/mathesar_ui/src/component-library/labeled-input/LabeledInput.scss @@ -27,6 +27,17 @@ } } + .description { + display: block; + font-size: var(--text-size-small); + color: var(--color-text-muted); + margin-top: var(--spacing-y, var(--spacing-y-default)); + margin-left: 1.5rem; + &:empty { + display: none; + } + } + &.layout-stacked { .input { margin-top: var(--spacing-y, var(--spacing-y-default)); diff --git a/mathesar_ui/src/component-library/labeled-input/LabeledInput.svelte b/mathesar_ui/src/component-library/labeled-input/LabeledInput.svelte index 21c8d02a88..af49c2d29a 100644 --- a/mathesar_ui/src/component-library/labeled-input/LabeledInput.svelte +++ b/mathesar_ui/src/component-library/labeled-input/LabeledInput.svelte @@ -6,6 +6,7 @@ export let label: string | undefined = undefined; export let help: string | undefined = undefined; export let layout: LabeledInputLayout = 'inline'; + export let description: string | undefined = undefined;

{label ?? ''} + {help ?? ''} + + +
diff --git a/mathesar_ui/src/components/LinkedRecord.svelte b/mathesar_ui/src/components/LinkedRecord.svelte index c673bcf423..9667b46813 100644 --- a/mathesar_ui/src/components/LinkedRecord.svelte +++ b/mathesar_ui/src/components/LinkedRecord.svelte @@ -145,6 +145,7 @@ overflow: hidden; white-space: nowrap; text-overflow: ellipsis; + } .delete-button { position: relative; diff --git a/mathesar_ui/src/components/inspector/cell/CellInspector.svelte b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte index 659a4859d9..082f754ec0 100644 --- a/mathesar_ui/src/components/inspector/cell/CellInspector.svelte +++ b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte @@ -47,7 +47,7 @@ diff --git a/mathesar_ui/src/i18n/languages/en/dict.json b/mathesar_ui/src/i18n/languages/en/dict.json index 095a5316dd..d715a258be 100644 --- a/mathesar_ui/src/i18n/languages/en/dict.json +++ b/mathesar_ui/src/i18n/languages/en/dict.json @@ -136,6 +136,7 @@ "create_user_via": "Create user via", "currently_installed": "Currently Installed", "custom": "Custom", + "use_custom_template": "Use Custom Template", "custom_default": "Custom Default", "customize_names_types_preview": "You can customize column names and types within the preview below.", "data_explorer": "Data Explorer", @@ -155,7 +156,8 @@ "databases_matching_search": "{count, plural, one {{count} database matches [searchValue]} other {{count} databases match [searchValue]}}", "days": "Days", "db_server": "DB server", - "default": "Default", + "use_default": "Use Default", + "default_template": "Default Template", "default_value": "Default Value", "delete": "Delete", "delete_account": "Delete Account", @@ -185,7 +187,7 @@ "disable_link_question": "Disable Link?", "disabled_connection_edit_fields_help": "Mathesar does not yet support editing a connection's database name, host, and port. We [issueLink](plan) to add support for this in the future. In the meantime, you can delete this connection and create a new one if needed.", "disallow_null_values": "Disallow [null] Values", - "disallow_null_values_help": "Enable this option to prevent null values in the column. Null values are empty values that are not the same as zero or an empty string.", + "disallow_null_values_help": "Prevents empty values in this column.", "discard_changes": "Discard Changes", "disconnect": "Disconnect", "disconnect_database": "Disconnect Database", @@ -438,7 +440,7 @@ "record_deleted_successfully": "Record deleted successfully!", "record_in_table": "Record in [tableName]", "record_summary": "Record Summary", - "record_summary_help": "Shows how links to [tableName] records will appear.", + "record_summary_help": "The record summary shown when linking to [tableName].", "record_summary_non_conformant_columns_help": "Because some column names contain curly braces, the following numerical values are used in place of column names within the above template", "recover_query_click_button": "You can attempt to recover the query by clicking on the button below.", "redirected_login_page_password_change": "You'll be redirected to the login page once you change your password.", @@ -462,7 +464,7 @@ "rename_schema": "Rename [schemaName] Schema", "reset": "Reset", "restrict_to_unique": "Restrict to Unique", - "restrict_to_unique_help": "Enable this option to make sure that the column only contains unique values. Useful for columns that contain identifiers, such as a person's ID number or emails.", + "restrict_to_unique_help": "Ensures all values in this column are unique.", "result": "Result", "result_could_not_be_displayed": "The result could not be displayed.", "retry": "Retry", diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnNameAndDescription.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnNameAndDescription.svelte index 4873d8cb04..3a2d239fc4 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnNameAndDescription.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnNameAndDescription.svelte @@ -50,7 +50,7 @@
- {$_('name')} + {$_('name')}
- {$_('description')} + {$_('description')} :global(* + *) { - margin-top: 0.5rem; + margin-top: 0.25rem; } } + .label { + font-weight: var(--font-weight-medium); + } diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnOptions.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnOptions.svelte index d4cac113b7..7b6bfaf7ca 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnOptions.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnOptions.svelte @@ -104,9 +104,9 @@ {$_('restrict_to_unique')} - - {$_('restrict_to_unique_help')} - + + + {$_('restrict_to_unique_help')} {#if isRequestingToggleAllowDuplicates} @@ -126,9 +126,9 @@ NULL {/if} - - {$_('disallow_null_values_help')} - + + + {$_('disallow_null_values_help')} {#if isRequestingToggleAllowNull} diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/AppendColumn.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/AppendColumn.svelte index 177d356ed7..9104b565cc 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/AppendColumn.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/AppendColumn.svelte @@ -34,7 +34,7 @@ } - + {#each columns as column (column.id)} {:else} - {#if previewRecordSummary} -
{$_('preview')}
-
-
- - {#if slotName === 'tableName'} - {table.name} - {/if} - -
- -
- {/if} - -
{$_('template')}
(v ? $_('custom') : $_('default'))} + getRadioLabel={(v) => (v ? $_('use_custom_template') : $_('use_default'))} ariaLabel={$_('template_type')} isInline bind:value={$customized} disabled={$customizedDisabled} /> + {#if previewRecordSummary} +
+
+ +
+
+ + {#if slotName === 'tableName'} + {table.name} + {/if} + +
+ +
+ {/if} + {#if $customized} + {#if $customized !== initialCustomized || $template !== initialTemplate} + + {/if} + {#if !previewRecordSummary} + {$_('no_record_summary_available')} + {/if}
- - {$_('no_record_summary_available')} + {/if}
diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/TemplateInput.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/TemplateInput.svelte index fdbef1188e..438466630d 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record-summary/TemplateInput.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record-summary/TemplateInput.svelte @@ -32,5 +32,6 @@ .column-insert { display: block; text-align: right; + margin-top: 0.5rem; } diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte index df2e3fba95..2fe3389cf6 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableDescription.svelte @@ -33,7 +33,10 @@ flex-direction: column; > :global(* + *) { - margin-top: 0.5rem; + margin-top: 0.25rem; } } + .label { + font-weight: var(--font-weight-medium); + } diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte index 1ffb43af18..a6f3b1fc1d 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/TableName.svelte @@ -43,7 +43,10 @@ flex-direction: column; > :global(* + *) { - margin-top: 0.5rem; + margin-top: 0.25rem; } } + .label { + font-weight: var(--font-weight-medium); + } diff --git a/mathesar_ui/src/systems/table-view/table-inspector/table/links/LinksSectionContainer.svelte b/mathesar_ui/src/systems/table-view/table-inspector/table/links/LinksSectionContainer.svelte index 24646bbfbf..280b904a18 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/table/links/LinksSectionContainer.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/table/links/LinksSectionContainer.svelte @@ -74,7 +74,7 @@ {$_('table_does_not_link')} {/if}
- From 42b59cd36f6d53f7211f24052c8fde06925af0b1 Mon Sep 17 00:00:00 2001 From: Ghislaine Guerin Date: Thu, 5 Sep 2024 09:02:51 +0200 Subject: [PATCH 0839/1141] update empty state padding --- .../src/components/inspector/cell/CellInspector.svelte | 10 ++++++++-- .../table-inspector/column/ColumnMode.svelte | 2 +- .../table-inspector/record/RecordMode.svelte | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mathesar_ui/src/components/inspector/cell/CellInspector.svelte b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte index 082f754ec0..e007842471 100644 --- a/mathesar_ui/src/components/inspector/cell/CellInspector.svelte +++ b/mathesar_ui/src/components/inspector/cell/CellInspector.svelte @@ -41,13 +41,15 @@
{:else} - {$_('select_cell_view_properties')} +
+ {$_('select_cell_view_properties')} +
{/if}
diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte index 7b4684b7cc..53cdb1e7fc 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnMode.svelte @@ -159,7 +159,7 @@ } .no-cell-selected { - padding: 2rem; + padding: var(--size-ultra-large); } .content-container { diff --git a/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte b/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte index 8f133a18ef..eec9c58c3b 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/record/RecordMode.svelte @@ -53,7 +53,7 @@ } .records-selected-count { - padding: 1rem; + padding: var(--size-large); } .content-container { From 341f478a992693564f96a1070282b8d564990804 Mon Sep 17 00:00:00 2001 From: Ghislaine Guerin Date: Thu, 5 Sep 2024 10:16:43 +0200 Subject: [PATCH 0840/1141] update action buttons --- .../src/component-library/common/styles/variables.scss | 4 ++-- .../table-view/table-inspector/column/ColumnActions.svelte | 4 ++-- .../table-view/table-inspector/record/RowActions.svelte | 4 ++-- .../table-view/table-inspector/table/AdvancedActions.svelte | 2 +- .../table-view/table-inspector/table/TableActions.svelte | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mathesar_ui/src/component-library/common/styles/variables.scss b/mathesar_ui/src/component-library/common/styles/variables.scss index 4495197957..c20cd616d6 100644 --- a/mathesar_ui/src/component-library/common/styles/variables.scss +++ b/mathesar_ui/src/component-library/common/styles/variables.scss @@ -29,10 +29,10 @@ --slate-100: #eff1f1; --slate-200: #d5d8dc; --slate-300: #b8bdc4; - --slate-400: #5e646e; + --slate-400: #6a707c; --slate-500: #4a4f58; --slate-600: #3a3f48; - --slate-700: #424952; + --slate-700: #373d44; --slate-800: #25292e; --sand-50: #fcfbf8; diff --git a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnActions.svelte b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnActions.svelte index 41957adf16..db041a5014 100644 --- a/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnActions.svelte +++ b/mathesar_ui/src/systems/table-view/table-inspector/column/ColumnActions.svelte @@ -55,7 +55,7 @@
- {#if canMoveToLinkedTable} -