diff --git a/src/routes/(console)/project-[region]-[project]/databases/database-[database]/collection-[collection]/+page.ts b/src/routes/(console)/project-[region]-[project]/databases/database-[database]/collection-[collection]/+page.ts index 9292eaf8cd..ee0764b6b0 100644 --- a/src/routes/(console)/project-[region]-[project]/databases/database-[database]/collection-[collection]/+page.ts +++ b/src/routes/(console)/project-[region]-[project]/databases/database-[database]/collection-[collection]/+page.ts @@ -2,7 +2,7 @@ import { Dependencies, SPREADSHEET_PAGE_LIMIT } from '$lib/constants'; import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/helpers/load'; import type { PageLoad } from './$types'; import { queries, queryParamToMap } from '$lib/components/filters'; -import { buildGridQueries, extractSortFromQueries } from '$database/store'; +import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store'; import { getCollectionService } from '$database/(entity)'; export const load: PageLoad = async ({ params, depends, url, route, parent }) => { @@ -22,6 +22,21 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) => const currentSort = extractSortFromQueries(parsedQueries); const collectionSdk = getCollectionService(params.region, params.project, database.type); + const documentsPage = await loadGridRows( + collection, + (includeRelationships) => + buildGridQueries(limit, offset, parsedQueries, collection, includeRelationships), + async (queries) => { + const response = await collectionSdk.listDocuments({ + databaseId: params.database, + collectionId: params.collection, + queries + }); + + return { total: response.total, rows: response.documents }; + } + ); + return { offset, limit, @@ -29,10 +44,6 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) => query, currentSort, parsedQueries, - documents: await collectionSdk.listDocuments({ - databaseId: params.database, - collectionId: params.collection, - queries: buildGridQueries(limit, offset, parsedQueries, collection) - }) + documents: { total: documentsPage.total, documents: documentsPage.rows } }; }; diff --git a/src/routes/(console)/project-[region]-[project]/databases/database-[database]/store.ts b/src/routes/(console)/project-[region]-[project]/databases/database-[database]/store.ts index 1a378fe435..c8c6ba7e33 100644 --- a/src/routes/(console)/project-[region]-[project]/databases/database-[database]/store.ts +++ b/src/routes/(console)/project-[region]-[project]/databases/database-[database]/store.ts @@ -3,7 +3,8 @@ import type { Column } from '$lib/helpers/types'; import { IconCloudUpload, IconCog } from '@appwrite.io/pink-icons-svelte'; import { resolveRoute, withPath } from '$lib/stores/navigation'; import type { Page } from '@sveltejs/kit'; -import { type Models, Query } from '@appwrite.io/console'; +import { AppwriteException, type Models, Query } from '@appwrite.io/console'; +import { chunks } from '$lib/helpers/array'; import type { Entity, Field } from '$database/(entity)'; import { isRelationship } from '$database/table-[table]/rows/store'; import type { TagValue } from '$lib/components/filters/store'; @@ -178,7 +179,8 @@ export function buildGridQueries( limit: number, offset: number, parsedQueries: Map, - table: Entity + table: Entity, + includeRelationships: boolean = true ) { const hasOrderQuery = Array.from(parsedQueries.values()).some( (q) => q.includes('orderAsc') || q.includes('orderDesc') @@ -191,7 +193,75 @@ export function buildGridQueries( queryArray.push(Query.orderDesc('')); } - queryArray.push(...parsedQueries.values(), ...buildWildcardEntitiesQuery(table)); + queryArray.push( + ...parsedQueries.values(), + ...(includeRelationships ? buildWildcardEntitiesQuery(table) : [Query.select(['*'])]) + ); return queryArray; } + +const RELATIONSHIP_CHUNK_SIZE = 10; + +function isTooManyQueryValues(error: unknown): boolean { + return ( + error instanceof AppwriteException && /greater than \d+ values/i.test(error.message ?? '') + ); +} + +type EntityRows = { total: number; rows: T[] }; + +/** + * Loads a page of the grid, falling back to smaller relationship batches when + * the API trips its own 500 value limit resolving them for the whole page. + */ +export async function loadGridRows( + entity: Entity, + buildQueries: (includeRelationships: boolean) => string[], + fetchRows: (queries: string[]) => Promise> +): Promise> { + try { + return await fetchRows(buildQueries(true)); + } catch (error) { + if (!isTooManyQueryValues(error)) throw error; + + const page = await fetchRows(buildQueries(false)); + + return { + total: page.total, + rows: await populateRelationships(entity, page.rows, fetchRows) + }; + } +} + +async function populateRelationships( + entity: Entity, + rows: T[], + fetchRows: (queries: string[]) => Promise> +): Promise { + const relationshipQueries = buildWildcardEntitiesQuery(entity); + + if (relationshipQueries.length <= 1 || rows.length === 0) return rows; + + const populated = new Map(); + + await Promise.all( + chunks(rows, RELATIONSHIP_CHUNK_SIZE).map(async (chunk) => { + const rowIds = chunk.map((row) => row.$id); + + try { + const response = await fetchRows([ + Query.equal('$id', rowIds), + Query.limit(rowIds.length), + ...relationshipQueries + ]); + + response.rows.forEach((row) => populated.set(row.$id, row)); + } catch { + // keep the unpopulated rows for this chunk! + } + }) + ); + + return rows.map((row) => populated.get(row.$id) ?? row); +} diff --git a/src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/+page.ts b/src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/+page.ts index 7ccffdf996..a584f612b3 100644 --- a/src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/+page.ts +++ b/src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/+page.ts @@ -3,7 +3,7 @@ import { getLimit, getPage, getQuery, getView, pageToOffset, View } from '$lib/h import { sdk } from '$lib/stores/sdk'; import type { PageLoad } from './$types'; import { queries, queryParamToMap } from '$lib/components/filters'; -import { buildGridQueries, extractSortFromQueries } from '$database/store'; +import { buildGridQueries, extractSortFromQueries, loadGridRows } from '$database/store'; export const load: PageLoad = async ({ params, depends, url, route, parent }) => { const { table } = await parent(); @@ -28,10 +28,16 @@ export const load: PageLoad = async ({ params, depends, url, route, parent }) => query, currentSort, parsedQueries, - rows: await sdk.forProject(params.region, params.project).tablesDB.listRows({ - databaseId: params.database, - tableId: params.table, - queries: buildGridQueries(limit, offset, parsedQueries, table) - }) + rows: await loadGridRows( + table, + (includeRelationships) => + buildGridQueries(limit, offset, parsedQueries, table, includeRelationships), + (queries) => + sdk.forProject(params.region, params.project).tablesDB.listRows({ + databaseId: params.database, + tableId: params.table, + queries + }) + ) }; }; diff --git a/src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/spreadsheet.svelte b/src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/spreadsheet.svelte index 9187728227..c6523114bb 100644 --- a/src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/spreadsheet.svelte +++ b/src/routes/(console)/project-[region]-[project]/databases/database-[database]/table-[table]/spreadsheet.svelte @@ -98,6 +98,7 @@ expandTabs, type Columns, buildWildcardEntitiesQuery, + loadGridRows, type SortState, randomDataModalState, spreadsheetLoading, @@ -722,22 +723,26 @@ } const parsedQueries = data.parsedQueries; - const filterQueries = parsedQueries.size ? data.parsedQueries.values() : []; + // materialized: the fallback in `loadGridRows` builds the queries more than once. + const filterQueries = parsedQueries.size ? Array.from(data.parsedQueries.values()) : []; $paginatedRowsLoading = true; - const loadedRows = await sdk - .forProject(page.params.region, page.params.project) - .tablesDB.listRows({ - databaseId, - tableId, - queries: [ - getCorrectOrderQuery(), - Query.limit(SPREADSHEET_PAGE_LIMIT), - Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)), - ...filterQueries /* filter queries */, - ...buildWildcardEntitiesQuery(table) - ] - }); + const loadedRows = await loadGridRows( + table, + (includeRelationships) => [ + getCorrectOrderQuery(), + Query.limit(SPREADSHEET_PAGE_LIMIT), + Query.offset(pageToOffset(pageNumber, SPREADSHEET_PAGE_LIMIT)), + ...filterQueries /* filter queries */, + ...(includeRelationships + ? buildWildcardEntitiesQuery(table) + : [Query.select(['*'])]) + ], + (queries) => + sdk + .forProject(page.params.region, page.params.project) + .tablesDB.listRows({ databaseId, tableId, queries }) + ); paginatedRows.setPage(pageNumber, loadedRows.rows); $paginatedRowsLoading = false; @@ -753,18 +758,21 @@ paginatedRows.setMaxPage(targetPageNum); $paginatedRowsLoading = true; - const loadedRows = await sdk - .forProject(page.params.region, page.params.project) - .tablesDB.listRows({ - databaseId, - tableId, - queries: [ - getCorrectOrderQuery(), - Query.limit(SPREADSHEET_PAGE_LIMIT), - Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)), - ...buildWildcardEntitiesQuery(table) - ] - }); + const loadedRows = await loadGridRows( + table, + (includeRelationships) => [ + getCorrectOrderQuery(), + Query.limit(SPREADSHEET_PAGE_LIMIT), + Query.offset(pageToOffset(targetPageNum, SPREADSHEET_PAGE_LIMIT)), + ...(includeRelationships + ? buildWildcardEntitiesQuery(table) + : [Query.select(['*'])]) + ], + (queries) => + sdk + .forProject(page.params.region, page.params.project) + .tablesDB.listRows({ databaseId, tableId, queries }) + ); paginatedRows.setPage(targetPageNum, loadedRows.rows); $paginatedRowsLoading = false;