diff --git a/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md b/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md index 5ee5f93157..a8c7d8bd3d 100644 --- a/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md +++ b/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md @@ -102,7 +102,6 @@ To update **either** `NC_ADMIN_EMAIL` or `NC_ADMIN_PASSWORD`, you must set **bot | `NC_CONNECT_TO_EXTERNAL_DB_DISABLED` | No | Disables the ability to create bases on external databases. | | | `NC_INVITE_ONLY_SIGNUP` | No | Disables public signup; signup is possible only via invitations. Integrated into the [super admin settings menu](/account-settings/oss-specific-details#enable--disable-signup) as of version 0.99.0. | | | `NC_REQUEST_BODY_SIZE` | No | Maximum bytes allowed in the request body, based on [ExpressJS limits](https://expressjs.com/en/resources/middleware/body-parser.html#limit). | Defaults to `1048576` (1 MB). | -| `NC_EXPORT_MAX_TIMEOUT` | No | Sets a timeout in milliseconds for downloading CSVs in batches if not completed within this period. | Defaults to `5000` (5 seconds). | | `NC_ALLOW_LOCAL_HOOKS` | No | Allows webhooks to call local network links, posing potential security risks. Set to `true` to enable; all other values are considered `false`. | Defaults to `false`. | | `NC_SANITIZE_COLUMN_NAME` | No | Enables sanitization of column names during their creation to prevent SQL injection and other security issues. | Defaults to `true`. | | `NC_TOOL_DIR` | No | Specifies the directory to store metadata and app-related files. In Docker setups, this maps to `/usr/app/data/` for mounting volumes. | Defaults to the current working directory. | diff --git a/packages/noco-docs/versioned_docs/version-0.109.7/020.getting-started/020.environment-variables.md b/packages/noco-docs/versioned_docs/version-0.109.7/020.getting-started/020.environment-variables.md index a8a69e3135..fa5b9a2195 100644 --- a/packages/noco-docs/versioned_docs/version-0.109.7/020.getting-started/020.environment-variables.md +++ b/packages/noco-docs/versioned_docs/version-0.109.7/020.getting-started/020.environment-variables.md @@ -29,7 +29,6 @@ For production usecases, it is **recommended** to configure | NC_INVITE_ONLY_SIGNUP | Removed since version 0.99.0 and now it's recommended to use [super admin settings menu](/0.109.7/setup-and-usages/account-settings#enable--disable-signup). Allow users to signup only via invite url, value should be any non-empty string. | | | NUXT_PUBLIC_NC_BACKEND_URL | Custom Backend URL | ``http://localhost:8080`` will be used | | NC_REQUEST_BODY_SIZE | Request body size [limit](https://expressjs.com/en/resources/middleware/body-parser.html#limit) | `1048576` | -| NC_EXPORT_MAX_TIMEOUT | After NC_EXPORT_MAX_TIMEOUT csv gets downloaded in batches | Default value 5000(in millisecond) will be used | | NC_DISABLE_TELE | Disable telemetry | | | NC_DISABLE_ERR_REPORTS | Disable default sentry error reporting | | | NC_DASHBOARD_URL | Custom dashboard url path | `/dashboard` | diff --git a/packages/nocodb/src/helpers/dataHelpers.ts b/packages/nocodb/src/helpers/dataHelpers.ts index 5c9c7527ff..cd9e32dd30 100644 --- a/packages/nocodb/src/helpers/dataHelpers.ts +++ b/packages/nocodb/src/helpers/dataHelpers.ts @@ -1,18 +1,11 @@ -import { convertMS2Duration, isSystemColumn, UITypes } from 'nocodb-sdk'; -import * as XLSX from 'xlsx'; -import papaparse from 'papaparse'; -import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; +import { convertMS2Duration, UITypes } from 'nocodb-sdk'; import type LinkToAnotherRecordColumn from '~/models/LinkToAnotherRecordColumn'; import type LookupColumn from '~/models/LookupColumn'; import type { NcContext } from '~/interface/config'; -import { nocoExecute } from '~/utils'; +import type Column from '~/models/Column'; import { NcError } from '~/helpers/catchError'; -import getAst from '~/helpers/getAst'; import { Model, View } from '~/models'; -import Source from '~/models/Source'; -import Column from '~/models/Column'; import Base from '~/models/Base'; -import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2'; export interface PathParams { baseName: string; @@ -54,118 +47,6 @@ export async function getViewAndModelByAliasOrId( return { model, view }; } -export async function extractXlsxData(context: NcContext, view: View, req) { - const source = await Source.get(context, view.source_id); - - await view.getModelWithInfo(context); - await view.getColumns(context); - - view.model.columns = view.columns - .filter((c) => c.show) - .map( - (c) => - new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any), - ) - .filter((column) => !isSystemColumn(column) || view.show_system_fields); - - const baseModel = await Model.getBaseModelSQL(context, { - id: view.model.id, - viewId: view?.id, - dbDriver: await NcConnectionMgrv2.get(source), - }); - - const { offset, dbRows, elapsed } = await getDbRows(context, { - baseModel, - view, - siteUrl: (req as any).ncSiteUrl, - query: req.query, - }); - - const fields = req.query.fields as string[]; - const columnFields = view.model.columns - .sort((c1, c2) => - Array.isArray(fields) - ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any) - : 0, - ) - .filter( - (c) => - !fields || !Array.isArray(fields) || fields.includes(c.title as any), - ) - .map((c) => c.title); - - const dataToSerialize = dbRows - .filter((k) => k) - .map((k) => - // remove all columns not defined in filter / columns - // because it's still getting added into xlsx even though undefined - Object.keys(k).reduce((obj, title) => { - if (columnFields.includes(title)) { - obj[title] = k[title]; - } - return obj; - }, {}), - ); - const data = XLSX.utils.json_to_sheet(dataToSerialize, { - header: columnFields, - }); - - return { offset, dbRows, elapsed, data }; -} - -export async function extractCsvData(context: NcContext, view: View, req) { - const source = await Source.get(context, view.source_id); - const fields = req.query.fields; - - await view.getModelWithInfo(context); - await view.getColumns(context); - - view.model.columns = view.columns - .filter((c) => c.show) - .map( - (c) => - new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any), - ) - .filter((column) => !isSystemColumn(column) || view.show_system_fields); - - const baseModel = await Model.getBaseModelSQL(context, { - id: view.model.id, - viewId: view?.id, - dbDriver: await NcConnectionMgrv2.get(source), - }); - - const { offset, dbRows, elapsed } = await getDbRows(context, { - baseModel, - view, - query: req.query, - siteUrl: (req as any).ncSiteUrl, - }); - - const data = papaparse.unparse( - { - fields: view.model.columns - .sort((c1, c2) => - Array.isArray(fields) - ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any) - : 0, - ) - .filter( - (c) => - !fields || - !Array.isArray(fields) || - fields.includes(c.title as any), - ) - .map((c) => c.title), - data: dbRows, - }, - { - escapeFormulae: true, - }, - ); - - return { offset, dbRows, elapsed, data }; -} - export async function serializeCellValue( context: NcContext, { @@ -292,71 +173,3 @@ export async function getColumnByIdOrName( return column; } - -export async function getDbRows( - context: NcContext, - param: { - baseModel: BaseModelSqlv2; - view: View; - query: any; - siteUrl: string; - }, -) { - const { baseModel, view, query = {}, siteUrl } = param; - let offset = +query.offset || 0; - const limit = 100; - // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; - const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; - const dbRows = []; - const startTime = process.hrtime(); - let elapsed, temp; - - const listArgs: any = { ...query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - - for ( - elapsed = 0; - elapsed < timeout; - offset += limit, - temp = process.hrtime(startTime), - elapsed = temp[0] * 1000 + temp[1] / 1000000 - ) { - const { ast, dependencyFields } = await getAst(context, { - query: query, - includePkByDefault: false, - model: view.model, - view, - }); - const rows = await nocoExecute( - ast, - await baseModel.list({ ...listArgs, ...dependencyFields, offset, limit }), - {}, - dependencyFields, - ); - - if (!rows?.length) { - offset = -1; - break; - } - - for (const row of rows) { - const dbRow = { ...row }; - - for (const column of view.model.columns) { - if (isSystemColumn(column) && !view.show_system_fields) continue; - dbRow[column.title] = await serializeCellValue(context, { - value: row[column.title], - column, - siteUrl, - }); - } - dbRows.push(dbRow); - } - } - return { offset, dbRows, elapsed }; -} diff --git a/packages/nocodb/src/services/datas.service.ts b/packages/nocodb/src/services/datas.service.ts index 01cd556a59..7740e2d100 100644 --- a/packages/nocodb/src/services/datas.service.ts +++ b/packages/nocodb/src/services/datas.service.ts @@ -1,7 +1,5 @@ import { Injectable, Logger } from '@nestjs/common'; -import { isLinksOrLTAR, isSystemColumn, NcSDKErrorV2 } from 'nocodb-sdk'; -import papaparse from 'papaparse'; -import * as XLSX from 'xlsx'; +import { isLinksOrLTAR, NcSDKErrorV2 } from 'nocodb-sdk'; import type { NcApiVersion } from 'nocodb-sdk'; import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; import type { PathParams } from '~/helpers/dataHelpers'; @@ -9,7 +7,7 @@ import type { NcContext } from '~/interface/config'; import type { Filter } from '~/models'; import type LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; import { NcBaseError, NcError } from '~/helpers/catchError'; -import { getDbRows, getViewAndModelByAliasOrId } from '~/helpers/dataHelpers'; +import { getViewAndModelByAliasOrId } from '~/helpers/dataHelpers'; import getAst from '~/helpers/getAst'; import { PagedResponseImpl } from '~/helpers/PagedResponse'; import { Base, Column, Model, Source, View } from '~/models'; @@ -1073,106 +1071,6 @@ export class DatasService { return { model, view }; } - async extractXlsxData( - context: NcContext, - param: { view: View; query: any; siteUrl: string }, - ) { - const { view, query, siteUrl } = param; - const source = await Source.get(context, view.source_id); - - await view.getModelWithInfo(context); - await view.getColumns(context); - - view.model.columns = view.columns - .filter((c) => c.show) - .map( - (c) => - new Column({ - ...c, - ...view.model.columnsById[c.fk_column_id], - } as any), - ) - .filter((column) => !isSystemColumn(column) || view.show_system_fields); - - const baseModel = await Model.getBaseModelSQL(context, { - id: view.model.id, - viewId: view?.id, - dbDriver: await NcConnectionMgrv2.get(source), - source, - }); - - const { offset, dbRows, elapsed } = await getDbRows(context, { - baseModel, - view, - query, - siteUrl, - }); - - const fields = query.fields as string[]; - - const data = XLSX.utils.json_to_sheet(dbRows, { header: fields }); - - return { offset, dbRows, elapsed, data }; - } - - async extractCsvData(context: NcContext, view: View, req) { - const source = await Source.get(context, view.source_id); - const fields = req.query.fields; - - await view.getModelWithInfo(context); - await view.getColumns(context); - - view.model.columns = view.columns - .filter((c) => c.show) - .map( - (c) => - new Column({ - ...c, - ...view.model.columnsById[c.fk_column_id], - } as any), - ) - .filter((column) => !isSystemColumn(column) || view.show_system_fields); - - const baseModel = await Model.getBaseModelSQL(context, { - id: view.model.id, - viewId: view?.id, - dbDriver: await NcConnectionMgrv2.get(source), - source, - }); - - const { offset, dbRows, elapsed } = await getDbRows(context, { - baseModel, - view, - query: req.query, - siteUrl: (req as any).ncSiteUrl, - }); - - const data = papaparse.unparse( - { - fields: view.model.columns - .sort((c1, c2) => - Array.isArray(fields) - ? fields.indexOf(c1.title as any) - - fields.indexOf(c2.title as any) - : 0, - ) - .filter( - (c) => - !fields || - !Array.isArray(fields) || - fields.includes(c.title as any), - ) - .map((c) => c.title), - data: dbRows, - }, - { - escapeFormulae: true, - }, - ); - - return { offset, dbRows, elapsed, data }; - } - async getColumnByIdOrName( context: NcContext, columnNameOrId: string,