[sync] feat: grid selection statistic backend aggregation (T3905)

Synced from teableio/teable-ee@11ebd64

Co-authored-by: Aries X <caoxing9@gmail.com>
Co-authored-by: Bieber <artist@teable.io>
Co-authored-by: Boris <boris2code@outlook.com>
Co-authored-by: Jocky-Teable <jocky@teable.ai>
Co-authored-by: Jun Lu <hammond@teable.io>
Co-authored-by: SkyHuang <sky.huang.fe@gmail.com>
Co-authored-by: Uno <uno@teable.ai>
Co-authored-by: nichenqin <nichenqin@hotmail.com>

Author: 9 people

apps/nestjs-backend/scripts/validate-dual-db-cutover.mjs

@@ -162,16 +162,24 @@ const main = async () => {
       ]);
 
     const schemaDiff = diffSets(sourceSchemas, targetSchemas);
-    const schemaTableCountDiffs = compareCountMaps(sourceSchemaTableCounts, targetSchemaTableCounts);
+    const schemaTableCountDiffs = compareCountMaps(
+      sourceSchemaTableCounts,
+      targetSchemaTableCounts
+    );
 
-    const [sourceDataCounts, targetDataCounts, sourceMetaCounts, targetMetaCounts, undoFunctionExists] =
-      await Promise.all([
-        getTableCounts(sourceClient, DATA_PLANE_TABLES),
-        getTableCounts(dataClient, DATA_PLANE_TABLES),
-        getTableCounts(sourceClient, META_PLANE_TABLES),
-        getTableCounts(metaClient, META_PLANE_TABLES),
-        getFunctionExists(dataClient, '__teable_capture_undo_row'),
-      ]);
+    const [
+      sourceDataCounts,
+      targetDataCounts,
+      sourceMetaCounts,
+      targetMetaCounts,
+      undoFunctionExists,
+    ] = await Promise.all([
+      getTableCounts(sourceClient, DATA_PLANE_TABLES),
+      getTableCounts(dataClient, DATA_PLANE_TABLES),
+      getTableCounts(sourceClient, META_PLANE_TABLES),
+      getTableCounts(metaClient, META_PLANE_TABLES),
+      getFunctionExists(dataClient, '__teable_capture_undo_row'),
+    ]);
 
     const dataCountDiffs = compareCountMaps(sourceDataCounts, targetDataCounts);
     const metaCountDiffs = compareCountMaps(sourceMetaCounts, targetMetaCounts);

apps/nestjs-backend/src/configs/threshold.config.ts

@@ -36,6 +36,7 @@ export const thresholdConfig = registerAs('threshold', () => ({
     jitter: Number(process.env.BACKEND_DB_DEADLOCK_JITTER ?? 1.0),
   },
   baseNodeMaxFolderDepth: Number(process.env.BASE_NODE_MAX_FOLDER_DEPTH ?? 2),
+  maxOwnedSpaceCount: Number(process.env.MAX_SPACE_OWNER_COUNT ?? 10),
   changeEmailSendCodeMailRate: Number(process.env.BACKEND_CHANGE_EMAIL_SEND_CODE_MAIL_RATE ?? 30),
   resetPasswordSendMailRate: Number(process.env.BACKEND_RESET_PASSWORD_SEND_MAIL_RATE ?? 30),
   signupVerificationSendCodeMailRate: Number(

apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts

@@ -1,4 +1,4 @@
-import type { IFilter, IGroup, StatisticsFunc } from '@teable/core';
+import type { IFilter, IGroup, ISortItem, StatisticsFunc } from '@teable/core';
 import type {
   IAggregationField,
   IQueryBaseRo,
@@ -32,6 +32,9 @@ export interface IAggregationService {
     withView?: IWithView;
     search?: [string, string?, boolean?];
     useQueryModel?: boolean;
+    skip?: number;
+    take?: number;
+    orderBy?: ISortItem[];
   }): Promise<IRawAggregationValue>;
 
   /**

apps/nestjs-backend/src/features/aggregation/aggregation.service.ts

@@ -7,13 +7,14 @@ import {
   identify,
   IdPrefix,
   mergeWithDefaultFilter,
+  mergeWithDefaultSort,
   nullsToUndefined,
   ViewType,
 } from '@teable/core';
-import type { IGridColumnMeta, IFilter, IGroup } from '@teable/core';
+import type { IGridColumnMeta, IFilter, IGroup, ISortItem } from '@teable/core';
+import { DataPrismaService } from '@teable/db-data-prisma';
 import type { Prisma } from '@teable/db-main-prisma';
 import { PrismaService } from '@teable/db-main-prisma';
-import { DataPrismaService } from '@teable/db-data-prisma';
 import { StatisticsFunc } from '@teable/openapi';
 import type {
   IAggregationField,
@@ -59,6 +60,10 @@ type IStatisticsData = {
   viewId?: string;
   filter?: IFilter;
   statisticFields?: IAggregationField[];
+  // Resolved view.sort merged with caller-supplied orderBy. Used for the BASE
+  // CTE order when skip/take is applied; aggregation itself is order-invariant
+  // so this is undefined unless the caller is paginating.
+  sort?: ISortItem[];
 };
 /**
  * Version 2 implementation of the aggregation service
@@ -92,21 +97,39 @@ export class AggregationService implements IAggregationService {
     withView?: IWithView;
     search?: [string, string?, boolean?];
     useQueryModel?: boolean;
+    // Optional row-range slice: when provided, the aggregation is computed
+    // over rows [skip, skip+take) of the view's filtered + sorted output. Used
+    // by the grid selection statistic endpoint; existing callers (footer
+    // aggregation, row count) leave them undefined for unchanged behavior.
+    skip?: number;
+    take?: number;
+    orderBy?: ISortItem[];
   }): Promise<IRawAggregationValue> {
-    const { tableId, withFieldIds, withView, search, useQueryModel } = params;
+    const { tableId, withFieldIds, withView, search, useQueryModel, skip, take, orderBy } = params;
     // Retrieve the current user's ID to build user-related query conditions
     const currentUserId = this.cls.get('user.id');
 
     const { statisticsData, fieldInstanceMap } = await this.fetchStatisticsParams({
       tableId,
       withView,
       withFieldIds,
+      extraOrderBy: orderBy,
     });
 
     const dbTableName = await this.getDbTableName(this.prisma, tableId);
 
-    const { filter, statisticFields } = statisticsData;
+    const { filter, statisticFields, sort: resolvedSort } = statisticsData;
     const groupBy = withView?.groupBy;
+
+    // When paginating, BASE CTE needs a deterministic ORDER BY so [skip, take)
+    // matches what the grid renders. Sort = group + view-sort, falling back to
+    // the view's row-order column (or __auto_number) for a stable tiebreaker.
+    const isPaginated = take !== undefined;
+    const baseSort = isPaginated ? [...(groupBy ?? []), ...(resolvedSort ?? [])] : undefined;
+    const defaultOrderField = isPaginated
+      ? await this.recordService.getBasicOrderIndexField(dbTableName, withView?.viewId)
+      : undefined;
+
     const rawAggregationData = await this.handleAggregation({
       dbTableName,
       fieldInstanceMap,
@@ -117,6 +140,10 @@ export class AggregationService implements IAggregationService {
       withUserId: currentUserId,
       withView,
       useQueryModel,
+      skip,
+      take,
+      sort: baseSort && baseSort.length ? baseSort : undefined,
+      defaultOrderField,
     });
 
     const aggregationResult = rawAggregationData && rawAggregationData[0];
@@ -210,6 +237,13 @@ export class AggregationService implements IAggregationService {
     withUserId?: string;
     withView?: IWithView;
     useQueryModel?: boolean;
+    // Optional row-range slice + ordering. Only set when the caller is
+    // paginating (selection aggregation); footer/row-count callers leave them
+    // undefined and the BASE CTE sees the full filtered set.
+    skip?: number;
+    take?: number;
+    sort?: ISortItem[];
+    defaultOrderField?: string;
   }) {
     const {
       dbTableName,
@@ -222,6 +256,10 @@ export class AggregationService implements IAggregationService {
       withView,
       tableId,
       useQueryModel,
+      skip,
+      take,
+      sort,
+      defaultOrderField,
     } = params;
 
     if (!statisticFields?.length) {
@@ -267,6 +305,10 @@ export class AggregationService implements IAggregationService {
         projection,
         useQueryModel,
         builder: permissionProbe.builder,
+        sort,
+        defaultOrderField,
+        limit: take,
+        offset: skip,
       }
     );
 
@@ -490,6 +532,7 @@ export class AggregationService implements IAggregationService {
     });
     return tableMeta.dbTableName;
   }
+
   private async handleRowCount(params: {
     tableId: string;
     dbTableName: string;
@@ -585,11 +628,12 @@ export class AggregationService implements IAggregationService {
     tableId: string;
     withView?: IWithView;
     withFieldIds?: string[];
+    extraOrderBy?: ISortItem[];
   }): Promise<{
     statisticsData: IStatisticsData;
     fieldInstanceMap: Record<string, IFieldInstance>;
   }> {
-    const { tableId, withView, withFieldIds } = params;
+    const { tableId, withView, withFieldIds, extraOrderBy } = params;
 
     const viewRaw = await this.findView(tableId, withView);
 
@@ -600,7 +644,12 @@ export class AggregationService implements IAggregationService {
       withFieldIds
     );
 
-    const statisticsData = this.buildStatisticsData(filteredFieldInstances, viewRaw, withView);
+    const statisticsData = this.buildStatisticsData(
+      filteredFieldInstances,
+      viewRaw,
+      withView,
+      extraOrderBy
+    );
 
     return { statisticsData, fieldInstanceMap };
   }
@@ -616,6 +665,7 @@ export class AggregationService implements IAggregationService {
           id: true,
           type: true,
           filter: true,
+          sort: true,
           group: true,
           options: true,
           columnMeta: true,
@@ -652,10 +702,12 @@ export class AggregationService implements IAggregationService {
           id: string | undefined;
           columnMeta: string | undefined;
           filter: string | undefined;
+          sort: string | undefined;
           group: string | undefined;
         }
       | undefined,
-    withView?: IWithView
+    withView?: IWithView,
+    extraOrderBy?: ISortItem[]
   ) {
     let statisticsData: IStatisticsData = {
       viewId: viewRaw?.id,
@@ -666,6 +718,12 @@ export class AggregationService implements IAggregationService {
       statisticsData = { ...statisticsData, filter };
     }
 
+    if (viewRaw?.sort || extraOrderBy) {
+      // Same recipe as record list: caller's orderBy overrides view.sort.
+      const sort = mergeWithDefaultSort(viewRaw?.sort, extraOrderBy);
+      statisticsData = { ...statisticsData, sort };
+    }
+
     if (viewRaw?.id || withView?.customFieldStats) {
       const statisticFields = this.getStatisticFields(
         filteredFieldInstances,

apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts

@@ -27,6 +27,8 @@ import {
   searchIndexByQueryRoSchema,
   IRecordIndexRo,
   recordIndexRoSchema,
+  ISelectionAggregationRo,
+  selectionAggregationRoSchema,
 } from '@teable/openapi';
 import { ClsService } from 'nestjs-cls';
 import { PerformanceCacheService } from '../../../performance-cache';
@@ -176,6 +178,16 @@ export class AggregationOpenApiController {
     );
   }
 
+  @Get('/selection')
+  @Permissions('table|read')
+  async getSelectionAggregation(
+    @Param('tableId') tableId: string,
+    @Query(new ZodValidationPipe(selectionAggregationRoSchema), TqlPipe)
+    query: ISelectionAggregationRo
+  ): Promise<IAggregationVo> {
+    return await this.aggregationOpenApiService.getSelectionAggregation(tableId, query);
+  }
+
   @Get('/task-status-collection')
   @Permissions('table|read')
   async getTaskStatusCollection(

apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.module.ts

@@ -1,11 +1,12 @@
 import { Module } from '@nestjs/common';
+import { RecordModule } from '../../record/record.module';
 import { AggregationModule } from '../aggregation.module';
 import { AggregationOpenApiController } from './aggregation-open-api.controller';
 import { AggregationOpenApiService } from './aggregation-open-api.service';
 
 @Module({
   controllers: [AggregationOpenApiController],
-  imports: [AggregationModule],
+  imports: [AggregationModule, RecordModule],
   providers: [AggregationOpenApiService],
   exports: [AggregationOpenApiService],
 })

apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts

@@ -14,16 +14,19 @@ import type {
   ISearchCountRo,
   IRecordIndexRo,
   IRecordIndexVo,
+  ISelectionAggregationRo,
 } from '@teable/openapi';
 import { forIn, isEmpty, map } from 'lodash';
+import { RecordService } from '../../record/record.service';
 import { IAggregationService } from '../aggregation.service.interface';
 import type { IWithView } from '../aggregation.service.interface';
 import { InjectAggregationService } from '../aggregation.service.provider';
 
 @Injectable()
 export class AggregationOpenApiService {
   constructor(
-    @InjectAggregationService() private readonly aggregationService: IAggregationService
+    @InjectAggregationService() private readonly aggregationService: IAggregationService,
+    private readonly recordService: RecordService
   ) {}
 
   async getAggregation(tableId: string, query?: IAggregationRo): Promise<IAggregationVo> {
@@ -133,4 +136,88 @@ export class AggregationOpenApiService {
   ) {
     return await this.aggregationService.getRecordIndexBySearchOrder(tableId, queryRo, projection);
   }
+
+  // Selection aggregation = the existing aggregation flow + a row-range slice.
+  // Same recipe as getAggregation: build customFieldStats, validate them, then
+  // delegate to performAggregation. Two deltas:
+  //   1. skip/take/orderBy thread through to scope the BASE CTE to the slice.
+  //   2. groupBy (if any) is folded INTO orderBy as a sort prefix and NOT
+  //      passed via withView. Two reasons:
+  //        a. `performGroupedAggregation` keys aggregations by fieldId, so a
+  //           request asking multiple funcs for the same field (chip asks
+  //           Sum + Filled) loses all but the last entry. Bypassing it keeps
+  //           every (fieldId, aggFunc) result intact.
+  //        b. The same routine re-runs handleAggregation without skip/take,
+  //           which would compute group totals over the whole view instead of
+  //           the slice — pointless work for the chip, which only reads
+  //           `total`.
+  //      The group prefix in orderBy preserves grid row order (records list
+  //      uses [...groupBy, ...orderBy] for its sort, mirrored here).
+  async getSelectionAggregation(
+    tableId: string,
+    query: ISelectionAggregationRo
+  ): Promise<IAggregationVo> {
+    const {
+      viewId,
+      filter: customFilter,
+      field: aggregationFields,
+      groupBy,
+      collapsedGroupIds,
+      ignoreViewQuery,
+      skip,
+      take,
+      orderBy,
+    } = query;
+
+    const sortWithGroup = [...(groupBy ?? []), ...(orderBy ?? [])];
+
+    // Translate collapsedGroupIds into a SQL filter (records in collapsed
+    // groups are excluded from the BASE CTE) so skip/take indexes the same
+    // visible-record sequence the grid renders. Same recipe records list uses.
+    let filterWithCollapsed = customFilter;
+    if (groupBy?.length && collapsedGroupIds?.length) {
+      const { filter } = await this.recordService.getGroupRelatedData(tableId, {
+        viewId,
+        ignoreViewQuery,
+        filter: customFilter,
+        groupBy,
+        collapsedGroupIds,
+        search: query.search,
+      });
+      filterWithCollapsed = filter;
+    }
+
+    let withView: IWithView = {
+      viewId: ignoreViewQuery ? undefined : viewId,
+      customFilter: filterWithCollapsed,
+      // Intentionally NOT passing groupBy (folded into orderBy above).
+    };
+
+    const fieldStatistics: Array<{ fieldId: string; statisticFunc: StatisticsFunc }> = [];
+    forIn(aggregationFields, (value: string[], key) => {
+      const fieldStats = map(value, (item) => ({
+        fieldId: item,
+        statisticFunc: key as StatisticsFunc,
+      }));
+      fieldStatistics.push(...fieldStats);
+    });
+
+    const validFieldStats = await this.validFieldStats(tableId, fieldStatistics);
+    if (validFieldStats) {
+      withView = { ...withView, customFieldStats: validFieldStats };
+    }
+
+    const result = await this.aggregationService.performAggregation({
+      tableId,
+      withView,
+      search: query.search,
+      // useQueryModel must stay false here: the tableCache path skips BASE CTE
+      // pagination, which would silently aggregate the entire view.
+      useQueryModel: false,
+      skip,
+      take,
+      orderBy: sortWithGroup.length ? sortWithGroup : undefined,
+    });
+    return { aggregations: result?.aggregations };
+  }
 }