refactor out of bulkAggregate

packages/nocodb/src/db/BaseModelSqlv2.ts

@@ -871,217 +871,11 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 {
     }>,
     view?: View,
   ) {
-    try {
-      if (!bulkFilterList?.length) {
-        return {};
-      }
-
-      const { where, aggregation } = this._getListArgs(args as any);
-
-      const columns = await this.model.getColumns(this.context);
-
-      let viewColumns: any[];
-      if (this.viewId) {
-        viewColumns = (
-          await GridViewColumn.list(this.context, this.viewId)
-        ).filter((c) => {
-          const col = this.model.columnsById[c.fk_column_id];
-          return c.show && (view?.show_system_fields || !isSystemColumn(col));
-        });
-
-        // By default, the aggregation is done based on the columns configured in the view
-        // If the aggregation parameter is provided, only the columns mentioned in the aggregation parameter are considered
-        // Also the aggregation type from the parameter is given preference over the aggregation type configured in the view
-        if (aggregation?.length) {
-          viewColumns = viewColumns
-            .map((c) => {
-              const agg = aggregation.find((a) => a.field === c.fk_column_id);
-              return new GridViewColumn({
-                ...c,
-                show: !!agg,
-                aggregation: agg ? agg.type : c.aggregation,
-              });
-            })
-            .filter((c) => c.show);
-        }
-      } else {
-        // If no viewId, use all model columns or those specified in aggregation
-        if (aggregation?.length) {
-          viewColumns = aggregation
-            .map((agg) => {
-              const col = this.model.columnsById[agg.field];
-              if (!col) return null;
-              return {
-                fk_column_id: col.id,
-                aggregation: agg.type,
-                show: true,
-              };
-            })
-            .filter(Boolean);
-        } else {
-          viewColumns = [];
-        }
-      }
-
-      const aliasColObjMap = await this.model.getAliasColObjMap(
-        this.context,
-        columns,
-      );
-
-      const qb = this.dbDriver(this.tnPath);
-
-      const aggregateExpressions = {};
-
-      // Construct aggregate expressions for each view column
-      for (const viewColumn of viewColumns) {
-        const col = this.model.columnsById[viewColumn.fk_column_id];
-        if (
-          !col ||
-          !viewColumn.aggregation ||
-          (isLinksOrLTAR(col) && col.system)
-        )
-          continue;
-
-        const aliasFieldName = col.id;
-        const aggSql = await applyAggregation({
-          baseModelSqlv2: this,
-          aggregation: viewColumn.aggregation,
-          column: col,
-        });
-
-        if (aggSql) {
-          aggregateExpressions[aliasFieldName] = aggSql;
-        }
-      }
-
-      if (!Object.keys(aggregateExpressions).length) {
-        return {};
-      }
-
-      let viewFilterList = [];
-      if (this.viewId) {
-        viewFilterList = await Filter.rootFilterList(this.context, {
-          viewId: this.viewId,
-        });
-      }
-
-      const selectors = [] as Array<Knex.Raw>;
-      // Generate a knex raw query for each filter in the bulkFilterList
-      for (const f of bulkFilterList) {
-        const tQb = this.dbDriver(this.tnPath);
-        const { filters: aggFilter } = extractFilterFromXwhere(
-          this.context,
-          f.where,
-          aliasColObjMap,
-        );
-        let aggFilterJson = f.filterArrJson;
-        try {
-          aggFilterJson = JSON.parse(aggFilterJson as any);
-        } catch (_e) {}
-
-        await conditionV2(
-          this,
-          [
-            ...(this.viewId
-              ? [
-                  new Filter({
-                    children: viewFilterList || [],
-                    is_group: true,
-                  }),
-                ]
-              : []),
-            new Filter({
-              children: args.filterArr || [],
-              is_group: true,
-              logical_op: 'and',
-            }),
-            new Filter({
-              children: extractFilterFromXwhere(
-                this.context,
-                where,
-                aliasColObjMap,
-              ).filters,
-              is_group: true,
-              logical_op: 'and',
-            }),
-            new Filter({
-              children: aggFilter,
-              is_group: true,
-              logical_op: 'and',
-            }),
-            ...(aggFilterJson
-              ? [
-                  new Filter({
-                    children: aggFilterJson as Filter[],
-                    is_group: true,
-                  }),
-                ]
-              : []),
-          ],
-          tQb,
-        );
-
-        let jsonBuildObject;
-
-        switch (this.dbDriver.client.config.client) {
-          case 'pg': {
-            jsonBuildObject = this.dbDriver.raw(
-              `JSON_BUILD_OBJECT(${Object.keys(aggregateExpressions)
-                .map((key) => {
-                  return `'${key}', ${aggregateExpressions[key]}`;
-                })
-                .join(', ')})`,
-            );
-
-            break;
-          }
-          case 'mysql2': {
-            jsonBuildObject = this.dbDriver.raw(`JSON_OBJECT(
-              ${Object.keys(aggregateExpressions)
-                .map((key) => `'${key}', ${aggregateExpressions[key]}`)
-                .join(', ')})`);
-            break;
-          }
-
-          case 'sqlite3': {
-            jsonBuildObject = this.dbDriver.raw(`json_object(
-                ${Object.keys(aggregateExpressions)
-                  .map((key) => `'${key}', ${aggregateExpressions[key]}`)
-                  .join(', ')})`);
-            break;
-          }
-          default:
-            NcError.notImplemented(
-              'This database is not supported for bulk aggregation',
-            );
-        }
-
-        tQb.select(jsonBuildObject);
-
-        if (this.dbDriver.client.config.client === 'mysql2') {
-          selectors.push(
-            this.dbDriver.raw('JSON_UNQUOTE(??) as ??', [
-              jsonBuildObject,
-              `${f.alias}`,
-            ]),
-          );
-        } else {
-          selectors.push(this.dbDriver.raw('(??) as ??', [tQb, `${f.alias}`]));
-        }
-      }
-
-      qb.select(...selectors);
-
-      qb.limit(1);
-
-      return await this.execAndParse(qb, null, {
-        first: true,
-        bulkAggregate: true,
-      });
-    } catch (err) {
-      logger.log(err);
-      return [];
-    }
+    return baseModelGroupBy(this, logger).bulkAggregate(
+      args,
+      bulkFilterList,
+      view,
+    );
   }
 
   async aggregate(args: { filterArr?: Filter[]; where?: string }, view?: View) {

packages/nocodb/src/db/BaseModelSqlv2/group-by.ts

@@ -1,4 +1,10 @@
-import { extractFilterFromXwhere, FormulaDataTypes, UITypes } from 'nocodb-sdk';
+import {
+  extractFilterFromXwhere,
+  FormulaDataTypes,
+  isLinksOrLTAR,
+  isSystemColumn,
+  UITypes,
+} from 'nocodb-sdk';
 import type { Logger } from '@nestjs/common';
 import type { Knex } from 'knex';
 import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2';
@@ -9,6 +15,7 @@ import type {
   RollupColumn,
   View,
 } from '~/models';
+import applyAggregation from '~/db/aggregation';
 import { replaceDelimitedWithKeyValuePg } from '~/db/aggregations/pg';
 import { sanitize } from '~/helpers/sqlSanitize';
 import conditionV2 from '~/db/conditionV2';
@@ -21,8 +28,8 @@ import {
   getAs,
   getColumnName,
 } from '~/helpers/dbHelpers';
-import { BaseUser, Column, Filter, Sort } from '~/models';
-import { getAliasGenerator, isOnPrem } from '~/utils';
+import { BaseUser, Column, Filter, GridViewColumn, Sort } from '~/models';
+import { getAliasGenerator } from '~/utils';
 import { replaceDelimitedWithKeyValueSqlite3 } from '~/db/aggregations/sqlite3';
 
 // Returns a SQL expression that converts blank (null or '') values to NULL
@@ -1439,10 +1446,237 @@ export const groupBy = (baseModel: IBaseModelSqlV2, logger: Logger) => {
     }
   };
 
+  const bulkAggregate = async (
+    args: {
+      filterArr?: Filter[];
+    },
+    bulkFilterList: Array<{
+      alias: string;
+      where?: string;
+      filterArrJson?: string | Filter[];
+    }>,
+    view?: View,
+  ) => {
+    try {
+      if (!bulkFilterList?.length) {
+        return {};
+      }
+
+      const { where, aggregation } = baseModel._getListArgs(args as any);
+
+      const columns = await baseModel.model.getColumns(baseModel.context);
+
+      let viewColumns: any[];
+      if (baseModel.viewId) {
+        viewColumns = (
+          await GridViewColumn.list(baseModel.context, baseModel.viewId)
+        ).filter((c) => {
+          const col = baseModel.model.columnsById[c.fk_column_id];
+          return c.show && (view?.show_system_fields || !isSystemColumn(col));
+        });
+
+        // By default, the aggregation is done based on the columns configured in the view
+        // If the aggregation parameter is provided, only the columns mentioned in the aggregation parameter are considered
+        // Also the aggregation type from the parameter is given preference over the aggregation type configured in the view
+        if (aggregation?.length) {
+          viewColumns = viewColumns
+            .map((c) => {
+              const agg = aggregation.find((a) => a.field === c.fk_column_id);
+              return new GridViewColumn({
+                ...c,
+                show: !!agg,
+                aggregation: agg ? agg.type : c.aggregation,
+              });
+            })
+            .filter((c) => c.show);
+        }
+      } else {
+        // If no viewId, use all model columns or those specified in aggregation
+        if (aggregation?.length) {
+          viewColumns = aggregation
+            .map((agg) => {
+              const col = baseModel.model.columnsById[agg.field];
+              if (!col) return null;
+              return {
+                fk_column_id: col.id,
+                aggregation: agg.type,
+                show: true,
+              };
+            })
+            .filter(Boolean);
+        } else {
+          viewColumns = [];
+        }
+      }
+
+      const aliasColObjMap = await baseModel.model.getAliasColObjMap(
+        baseModel.context,
+        columns,
+      );
+
+      const qb = baseModel.dbDriver(baseModel.tnPath);
+
+      const aggregateExpressions = {};
+
+      // Construct aggregate expressions for each view column
+      for (const viewColumn of viewColumns) {
+        const col = baseModel.model.columnsById[viewColumn.fk_column_id];
+        if (
+          !col ||
+          !viewColumn.aggregation ||
+          (isLinksOrLTAR(col) && col.system)
+        )
+          continue;
+
+        const aliasFieldName = col.id;
+        const aggSql = await applyAggregation({
+          baseModelSqlv2: baseModel,
+          aggregation: viewColumn.aggregation,
+          column: col,
+        });
+
+        if (aggSql) {
+          aggregateExpressions[aliasFieldName] = aggSql;
+        }
+      }
+
+      if (!Object.keys(aggregateExpressions).length) {
+        return {};
+      }
+
+      let viewFilterList = [];
+      if (baseModel.viewId) {
+        viewFilterList = await Filter.rootFilterList(baseModel.context, {
+          viewId: baseModel.viewId,
+        });
+      }
+
+      const selectors = [] as Array<Knex.Raw>;
+      // Generate a knex raw query for each filter in the bulkFilterList
+      for (const f of bulkFilterList) {
+        const tQb = baseModel.dbDriver(baseModel.tnPath);
+        const { filters: aggFilter } = extractFilterFromXwhere(
+          baseModel.context,
+          f.where,
+          aliasColObjMap,
+        );
+        let aggFilterJson = f.filterArrJson;
+        try {
+          aggFilterJson = JSON.parse(aggFilterJson as any);
+        } catch (_e) {}
+
+        await conditionV2(
+          baseModel,
+          [
+            ...(baseModel.viewId
+              ? [
+                  new Filter({
+                    children: viewFilterList || [],
+                    is_group: true,
+                  }),
+                ]
+              : []),
+            new Filter({
+              children: args.filterArr || [],
+              is_group: true,
+              logical_op: 'and',
+            }),
+            new Filter({
+              children: extractFilterFromXwhere(
+                baseModel.context,
+                where,
+                aliasColObjMap,
+              ).filters,
+              is_group: true,
+              logical_op: 'and',
+            }),
+            new Filter({
+              children: aggFilter,
+              is_group: true,
+              logical_op: 'and',
+            }),
+            ...(aggFilterJson
+              ? [
+                  new Filter({
+                    children: aggFilterJson as Filter[],
+                    is_group: true,
+                  }),
+                ]
+              : []),
+          ],
+          tQb,
+        );
+
+        let jsonBuildObject;
+
+        switch (baseModel.dbDriver.client.config.client) {
+          case 'pg': {
+            jsonBuildObject = baseModel.dbDriver.raw(
+              `JSON_BUILD_OBJECT(${Object.keys(aggregateExpressions)
+                .map((key) => {
+                  return `'${key}', ${aggregateExpressions[key]}`;
+                })
+                .join(', ')})`,
+            );
+
+            break;
+          }
+          case 'mysql2': {
+            jsonBuildObject = baseModel.dbDriver.raw(`JSON_OBJECT(
+                  ${Object.keys(aggregateExpressions)
+                    .map((key) => `'${key}', ${aggregateExpressions[key]}`)
+                    .join(', ')})`);
+            break;
+          }
+
+          case 'sqlite3': {
+            jsonBuildObject = baseModel.dbDriver.raw(`json_object(
+                    ${Object.keys(aggregateExpressions)
+                      .map((key) => `'${key}', ${aggregateExpressions[key]}`)
+                      .join(', ')})`);
+            break;
+          }
+          default:
+            NcError.get(baseModel.context).notImplemented(
+              'This database is not supported for bulk aggregation',
+            );
+        }
+
+        tQb.select(jsonBuildObject);
+
+        if (baseModel.dbDriver.client.config.client === 'mysql2') {
+          selectors.push(
+            baseModel.dbDriver.raw('JSON_UNQUOTE(??) as ??', [
+              jsonBuildObject,
+              `${f.alias}`,
+            ]),
+          );
+        } else {
+          selectors.push(
+            baseModel.dbDriver.raw('(??) as ??', [tQb, `${f.alias}`]),
+          );
+        }
+      }
+
+      qb.select(...selectors);
+
+      qb.limit(1);
+
+      return await baseModel.execAndParse(qb, null, {
+        first: true,
+        bulkAggregate: true,
+      });
+    } catch (err) {
+      logger.log(err);
+      return [];
+    }
+  };
+
   return {
     count,
     list,
     bulkCount,
     bulkList,
+    bulkAggregate,
   };
 };

packages/nocodb/src/db/aggregation.ts

@@ -7,8 +7,8 @@ import {
   NumericalAggregations,
   UITypes,
 } from 'nocodb-sdk';
+import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2';
 import type { NcContext } from 'nocodb-sdk';
-import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
 import type { BarcodeColumn, QrCodeColumn } from '~/models';
 import { Column } from '~/models';
 import { NcError } from '~/helpers/catchError';
@@ -84,7 +84,7 @@ export default async function applyAggregation({
   column,
   alias,
 }: {
-  baseModelSqlv2: BaseModelSqlv2;
+  baseModelSqlv2: IBaseModelSqlV2;
   aggregation: string;
   column: Column;
   alias?: string;

packages/nocodb/src/db/aggregations/mysql2.ts

@@ -9,8 +9,8 @@ import {
   UITypes,
 } from 'nocodb-sdk';
 import type { Column } from '~/models';
-import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
 import type { Knex } from 'knex';
+import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2';
 
 export function genMysql2AggregatedQuery({
   column,
@@ -22,7 +22,7 @@ export function genMysql2AggregatedQuery({
   alias,
 }: {
   column: Column;
-  baseModelSqlv2: BaseModelSqlv2;
+  baseModelSqlv2: IBaseModelSqlV2;
   aggregation: string;
   column_query: string | Knex.QueryBuilder;
   parsedFormulaType?: FormulaDataTypes;

packages/nocodb/src/db/aggregations/pg.ts

@@ -9,9 +9,9 @@ import {
   UITypes,
 } from 'nocodb-sdk';
 import type CustomKnex from '~/db/CustomKnex';
-import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
 import type { Knex } from 'knex';
 import type { Column } from '~/models';
+import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2';
 
 export function genPgAggregateQuery({
   column,
@@ -24,7 +24,7 @@ export function genPgAggregateQuery({
 }: {
   column: Column;
   column_query: string | Knex.QueryBuilder;
-  baseModelSqlv2: BaseModelSqlv2;
+  baseModelSqlv2: IBaseModelSqlV2;
   aggregation: string;
   parsedFormulaType?: FormulaDataTypes;
   aggType:

packages/nocodb/src/db/aggregations/sqlite3.ts

@@ -9,10 +9,10 @@ import {
   UITypes,
 } from 'nocodb-sdk';
 import type { Column } from '~/models';
-import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
 import type { Knex } from 'knex';
 
 import type CustomKnex from '~/db/CustomKnex';
+import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2';
 
 export function genSqlite3AggregateQuery({
   column,
@@ -24,7 +24,7 @@ export function genSqlite3AggregateQuery({
   alias,
 }: {
   column: Column;
-  baseModelSqlv2: BaseModelSqlv2;
+  baseModelSqlv2: IBaseModelSqlV2;
   aggregation: string;
   column_query: string | Knex.QueryBuilder;
   parsedFormulaType?: FormulaDataTypes;

packages/nocodb/src/db/getColumnNameQuery.ts

@@ -1,4 +1,5 @@
 import { UITypes } from 'nocodb-sdk';
+import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2';
 import type { Knex } from 'knex';
 import type {
   BarcodeColumn,
@@ -7,7 +8,6 @@ import type {
   RollupColumn,
 } from '~/models';
 import type { NcContext } from '~/interface/config';
-import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
 import { Column } from '~/models';
 import generateLookupSelectQuery from '~/db/generateLookupSelectQuery';
 import genRollupSelectv2 from '~/db/genRollupSelectv2';
@@ -27,7 +27,7 @@ export async function getColumnNameQuery({
   column,
   context,
 }: {
-  baseModelSqlv2: BaseModelSqlv2;
+  baseModelSqlv2: IBaseModelSqlV2;
   column: Column;
   context: NcContext;
 }): Promise<{