diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 4a94e6ba98..4572d7cfdc 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -871,217 +871,11 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { }>, view?: View, ) { - try { - if (!bulkFilterList?.length) { - return {}; - } - - const { where, aggregation } = this._getListArgs(args as any); - - const columns = await this.model.getColumns(this.context); - - let viewColumns: any[]; - if (this.viewId) { - viewColumns = ( - await GridViewColumn.list(this.context, this.viewId) - ).filter((c) => { - const col = this.model.columnsById[c.fk_column_id]; - return c.show && (view?.show_system_fields || !isSystemColumn(col)); - }); - - // By default, the aggregation is done based on the columns configured in the view - // If the aggregation parameter is provided, only the columns mentioned in the aggregation parameter are considered - // Also the aggregation type from the parameter is given preference over the aggregation type configured in the view - if (aggregation?.length) { - viewColumns = viewColumns - .map((c) => { - const agg = aggregation.find((a) => a.field === c.fk_column_id); - return new GridViewColumn({ - ...c, - show: !!agg, - aggregation: agg ? agg.type : c.aggregation, - }); - }) - .filter((c) => c.show); - } - } else { - // If no viewId, use all model columns or those specified in aggregation - if (aggregation?.length) { - viewColumns = aggregation - .map((agg) => { - const col = this.model.columnsById[agg.field]; - if (!col) return null; - return { - fk_column_id: col.id, - aggregation: agg.type, - show: true, - }; - }) - .filter(Boolean); - } else { - viewColumns = []; - } - } - - const aliasColObjMap = await this.model.getAliasColObjMap( - this.context, - columns, - ); - - const qb = this.dbDriver(this.tnPath); - - const aggregateExpressions = {}; - - // Construct aggregate expressions for each view column - for (const viewColumn of viewColumns) { - const col = this.model.columnsById[viewColumn.fk_column_id]; - if ( - !col || - !viewColumn.aggregation || - (isLinksOrLTAR(col) && col.system) - ) - continue; - - const aliasFieldName = col.id; - const aggSql = await applyAggregation({ - baseModelSqlv2: this, - aggregation: viewColumn.aggregation, - column: col, - }); - - if (aggSql) { - aggregateExpressions[aliasFieldName] = aggSql; - } - } - - if (!Object.keys(aggregateExpressions).length) { - return {}; - } - - let viewFilterList = []; - if (this.viewId) { - viewFilterList = await Filter.rootFilterList(this.context, { - viewId: this.viewId, - }); - } - - const selectors = [] as Array; - // Generate a knex raw query for each filter in the bulkFilterList - for (const f of bulkFilterList) { - const tQb = this.dbDriver(this.tnPath); - const { filters: aggFilter } = extractFilterFromXwhere( - this.context, - f.where, - aliasColObjMap, - ); - let aggFilterJson = f.filterArrJson; - try { - aggFilterJson = JSON.parse(aggFilterJson as any); - } catch (_e) {} - - await conditionV2( - this, - [ - ...(this.viewId - ? [ - new Filter({ - children: viewFilterList || [], - is_group: true, - }), - ] - : []), - new Filter({ - children: args.filterArr || [], - is_group: true, - logical_op: 'and', - }), - new Filter({ - children: extractFilterFromXwhere( - this.context, - where, - aliasColObjMap, - ).filters, - is_group: true, - logical_op: 'and', - }), - new Filter({ - children: aggFilter, - is_group: true, - logical_op: 'and', - }), - ...(aggFilterJson - ? [ - new Filter({ - children: aggFilterJson as Filter[], - is_group: true, - }), - ] - : []), - ], - tQb, - ); - - let jsonBuildObject; - - switch (this.dbDriver.client.config.client) { - case 'pg': { - jsonBuildObject = this.dbDriver.raw( - `JSON_BUILD_OBJECT(${Object.keys(aggregateExpressions) - .map((key) => { - return `'${key}', ${aggregateExpressions[key]}`; - }) - .join(', ')})`, - ); - - break; - } - case 'mysql2': { - jsonBuildObject = this.dbDriver.raw(`JSON_OBJECT( - ${Object.keys(aggregateExpressions) - .map((key) => `'${key}', ${aggregateExpressions[key]}`) - .join(', ')})`); - break; - } - - case 'sqlite3': { - jsonBuildObject = this.dbDriver.raw(`json_object( - ${Object.keys(aggregateExpressions) - .map((key) => `'${key}', ${aggregateExpressions[key]}`) - .join(', ')})`); - break; - } - default: - NcError.notImplemented( - 'This database is not supported for bulk aggregation', - ); - } - - tQb.select(jsonBuildObject); - - if (this.dbDriver.client.config.client === 'mysql2') { - selectors.push( - this.dbDriver.raw('JSON_UNQUOTE(??) as ??', [ - jsonBuildObject, - `${f.alias}`, - ]), - ); - } else { - selectors.push(this.dbDriver.raw('(??) as ??', [tQb, `${f.alias}`])); - } - } - - qb.select(...selectors); - - qb.limit(1); - - return await this.execAndParse(qb, null, { - first: true, - bulkAggregate: true, - }); - } catch (err) { - logger.log(err); - return []; - } + return baseModelGroupBy(this, logger).bulkAggregate( + args, + bulkFilterList, + view, + ); } async aggregate(args: { filterArr?: Filter[]; where?: string }, view?: View) { diff --git a/packages/nocodb/src/db/BaseModelSqlv2/group-by.ts b/packages/nocodb/src/db/BaseModelSqlv2/group-by.ts index 1ca6bfbc4d..ed6a673648 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2/group-by.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2/group-by.ts @@ -1,4 +1,10 @@ -import { extractFilterFromXwhere, FormulaDataTypes, UITypes } from 'nocodb-sdk'; +import { + extractFilterFromXwhere, + FormulaDataTypes, + isLinksOrLTAR, + isSystemColumn, + UITypes, +} from 'nocodb-sdk'; import type { Logger } from '@nestjs/common'; import type { Knex } from 'knex'; import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2'; @@ -9,6 +15,7 @@ import type { RollupColumn, View, } from '~/models'; +import applyAggregation from '~/db/aggregation'; import { replaceDelimitedWithKeyValuePg } from '~/db/aggregations/pg'; import { sanitize } from '~/helpers/sqlSanitize'; import conditionV2 from '~/db/conditionV2'; @@ -21,8 +28,8 @@ import { getAs, getColumnName, } from '~/helpers/dbHelpers'; -import { BaseUser, Column, Filter, Sort } from '~/models'; -import { getAliasGenerator, isOnPrem } from '~/utils'; +import { BaseUser, Column, Filter, GridViewColumn, Sort } from '~/models'; +import { getAliasGenerator } from '~/utils'; import { replaceDelimitedWithKeyValueSqlite3 } from '~/db/aggregations/sqlite3'; // Returns a SQL expression that converts blank (null or '') values to NULL @@ -1439,10 +1446,237 @@ export const groupBy = (baseModel: IBaseModelSqlV2, logger: Logger) => { } }; + const bulkAggregate = async ( + args: { + filterArr?: Filter[]; + }, + bulkFilterList: Array<{ + alias: string; + where?: string; + filterArrJson?: string | Filter[]; + }>, + view?: View, + ) => { + try { + if (!bulkFilterList?.length) { + return {}; + } + + const { where, aggregation } = baseModel._getListArgs(args as any); + + const columns = await baseModel.model.getColumns(baseModel.context); + + let viewColumns: any[]; + if (baseModel.viewId) { + viewColumns = ( + await GridViewColumn.list(baseModel.context, baseModel.viewId) + ).filter((c) => { + const col = baseModel.model.columnsById[c.fk_column_id]; + return c.show && (view?.show_system_fields || !isSystemColumn(col)); + }); + + // By default, the aggregation is done based on the columns configured in the view + // If the aggregation parameter is provided, only the columns mentioned in the aggregation parameter are considered + // Also the aggregation type from the parameter is given preference over the aggregation type configured in the view + if (aggregation?.length) { + viewColumns = viewColumns + .map((c) => { + const agg = aggregation.find((a) => a.field === c.fk_column_id); + return new GridViewColumn({ + ...c, + show: !!agg, + aggregation: agg ? agg.type : c.aggregation, + }); + }) + .filter((c) => c.show); + } + } else { + // If no viewId, use all model columns or those specified in aggregation + if (aggregation?.length) { + viewColumns = aggregation + .map((agg) => { + const col = baseModel.model.columnsById[agg.field]; + if (!col) return null; + return { + fk_column_id: col.id, + aggregation: agg.type, + show: true, + }; + }) + .filter(Boolean); + } else { + viewColumns = []; + } + } + + const aliasColObjMap = await baseModel.model.getAliasColObjMap( + baseModel.context, + columns, + ); + + const qb = baseModel.dbDriver(baseModel.tnPath); + + const aggregateExpressions = {}; + + // Construct aggregate expressions for each view column + for (const viewColumn of viewColumns) { + const col = baseModel.model.columnsById[viewColumn.fk_column_id]; + if ( + !col || + !viewColumn.aggregation || + (isLinksOrLTAR(col) && col.system) + ) + continue; + + const aliasFieldName = col.id; + const aggSql = await applyAggregation({ + baseModelSqlv2: baseModel, + aggregation: viewColumn.aggregation, + column: col, + }); + + if (aggSql) { + aggregateExpressions[aliasFieldName] = aggSql; + } + } + + if (!Object.keys(aggregateExpressions).length) { + return {}; + } + + let viewFilterList = []; + if (baseModel.viewId) { + viewFilterList = await Filter.rootFilterList(baseModel.context, { + viewId: baseModel.viewId, + }); + } + + const selectors = [] as Array; + // Generate a knex raw query for each filter in the bulkFilterList + for (const f of bulkFilterList) { + const tQb = baseModel.dbDriver(baseModel.tnPath); + const { filters: aggFilter } = extractFilterFromXwhere( + baseModel.context, + f.where, + aliasColObjMap, + ); + let aggFilterJson = f.filterArrJson; + try { + aggFilterJson = JSON.parse(aggFilterJson as any); + } catch (_e) {} + + await conditionV2( + baseModel, + [ + ...(baseModel.viewId + ? [ + new Filter({ + children: viewFilterList || [], + is_group: true, + }), + ] + : []), + new Filter({ + children: args.filterArr || [], + is_group: true, + logical_op: 'and', + }), + new Filter({ + children: extractFilterFromXwhere( + baseModel.context, + where, + aliasColObjMap, + ).filters, + is_group: true, + logical_op: 'and', + }), + new Filter({ + children: aggFilter, + is_group: true, + logical_op: 'and', + }), + ...(aggFilterJson + ? [ + new Filter({ + children: aggFilterJson as Filter[], + is_group: true, + }), + ] + : []), + ], + tQb, + ); + + let jsonBuildObject; + + switch (baseModel.dbDriver.client.config.client) { + case 'pg': { + jsonBuildObject = baseModel.dbDriver.raw( + `JSON_BUILD_OBJECT(${Object.keys(aggregateExpressions) + .map((key) => { + return `'${key}', ${aggregateExpressions[key]}`; + }) + .join(', ')})`, + ); + + break; + } + case 'mysql2': { + jsonBuildObject = baseModel.dbDriver.raw(`JSON_OBJECT( + ${Object.keys(aggregateExpressions) + .map((key) => `'${key}', ${aggregateExpressions[key]}`) + .join(', ')})`); + break; + } + + case 'sqlite3': { + jsonBuildObject = baseModel.dbDriver.raw(`json_object( + ${Object.keys(aggregateExpressions) + .map((key) => `'${key}', ${aggregateExpressions[key]}`) + .join(', ')})`); + break; + } + default: + NcError.get(baseModel.context).notImplemented( + 'This database is not supported for bulk aggregation', + ); + } + + tQb.select(jsonBuildObject); + + if (baseModel.dbDriver.client.config.client === 'mysql2') { + selectors.push( + baseModel.dbDriver.raw('JSON_UNQUOTE(??) as ??', [ + jsonBuildObject, + `${f.alias}`, + ]), + ); + } else { + selectors.push( + baseModel.dbDriver.raw('(??) as ??', [tQb, `${f.alias}`]), + ); + } + } + + qb.select(...selectors); + + qb.limit(1); + + return await baseModel.execAndParse(qb, null, { + first: true, + bulkAggregate: true, + }); + } catch (err) { + logger.log(err); + return []; + } + }; + return { count, list, bulkCount, bulkList, + bulkAggregate, }; }; diff --git a/packages/nocodb/src/db/aggregation.ts b/packages/nocodb/src/db/aggregation.ts index f5d7fca864..be93c81c9f 100644 --- a/packages/nocodb/src/db/aggregation.ts +++ b/packages/nocodb/src/db/aggregation.ts @@ -7,8 +7,8 @@ import { NumericalAggregations, UITypes, } from 'nocodb-sdk'; +import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2'; import type { NcContext } from 'nocodb-sdk'; -import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; import type { BarcodeColumn, QrCodeColumn } from '~/models'; import { Column } from '~/models'; import { NcError } from '~/helpers/catchError'; @@ -84,7 +84,7 @@ export default async function applyAggregation({ column, alias, }: { - baseModelSqlv2: BaseModelSqlv2; + baseModelSqlv2: IBaseModelSqlV2; aggregation: string; column: Column; alias?: string; diff --git a/packages/nocodb/src/db/aggregations/mysql2.ts b/packages/nocodb/src/db/aggregations/mysql2.ts index a6dfb2d49a..955890173b 100644 --- a/packages/nocodb/src/db/aggregations/mysql2.ts +++ b/packages/nocodb/src/db/aggregations/mysql2.ts @@ -9,8 +9,8 @@ import { UITypes, } from 'nocodb-sdk'; import type { Column } from '~/models'; -import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; import type { Knex } from 'knex'; +import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2'; export function genMysql2AggregatedQuery({ column, @@ -22,7 +22,7 @@ export function genMysql2AggregatedQuery({ alias, }: { column: Column; - baseModelSqlv2: BaseModelSqlv2; + baseModelSqlv2: IBaseModelSqlV2; aggregation: string; column_query: string | Knex.QueryBuilder; parsedFormulaType?: FormulaDataTypes; diff --git a/packages/nocodb/src/db/aggregations/pg.ts b/packages/nocodb/src/db/aggregations/pg.ts index 2c857dc421..901e04f27b 100644 --- a/packages/nocodb/src/db/aggregations/pg.ts +++ b/packages/nocodb/src/db/aggregations/pg.ts @@ -9,9 +9,9 @@ import { UITypes, } from 'nocodb-sdk'; import type CustomKnex from '~/db/CustomKnex'; -import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; import type { Knex } from 'knex'; import type { Column } from '~/models'; +import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2'; export function genPgAggregateQuery({ column, @@ -24,7 +24,7 @@ export function genPgAggregateQuery({ }: { column: Column; column_query: string | Knex.QueryBuilder; - baseModelSqlv2: BaseModelSqlv2; + baseModelSqlv2: IBaseModelSqlV2; aggregation: string; parsedFormulaType?: FormulaDataTypes; aggType: diff --git a/packages/nocodb/src/db/aggregations/sqlite3.ts b/packages/nocodb/src/db/aggregations/sqlite3.ts index a15145dff4..b5fa8cd567 100644 --- a/packages/nocodb/src/db/aggregations/sqlite3.ts +++ b/packages/nocodb/src/db/aggregations/sqlite3.ts @@ -9,10 +9,10 @@ import { UITypes, } from 'nocodb-sdk'; import type { Column } from '~/models'; -import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; import type { Knex } from 'knex'; import type CustomKnex from '~/db/CustomKnex'; +import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2'; export function genSqlite3AggregateQuery({ column, @@ -24,7 +24,7 @@ export function genSqlite3AggregateQuery({ alias, }: { column: Column; - baseModelSqlv2: BaseModelSqlv2; + baseModelSqlv2: IBaseModelSqlV2; aggregation: string; column_query: string | Knex.QueryBuilder; parsedFormulaType?: FormulaDataTypes; diff --git a/packages/nocodb/src/db/getColumnNameQuery.ts b/packages/nocodb/src/db/getColumnNameQuery.ts index 49185c6d6d..87a56a77cf 100644 --- a/packages/nocodb/src/db/getColumnNameQuery.ts +++ b/packages/nocodb/src/db/getColumnNameQuery.ts @@ -1,4 +1,5 @@ import { UITypes } from 'nocodb-sdk'; +import type { IBaseModelSqlV2 } from '~/db/IBaseModelSqlV2'; import type { Knex } from 'knex'; import type { BarcodeColumn, @@ -7,7 +8,6 @@ import type { RollupColumn, } from '~/models'; import type { NcContext } from '~/interface/config'; -import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; import { Column } from '~/models'; import generateLookupSelectQuery from '~/db/generateLookupSelectQuery'; import genRollupSelectv2 from '~/db/genRollupSelectv2'; @@ -27,7 +27,7 @@ export async function getColumnNameQuery({ column, context, }: { - baseModelSqlv2: BaseModelSqlv2; + baseModelSqlv2: IBaseModelSqlV2; column: Column; context: NcContext; }): Promise<{