Merge pull request #9517 from nocodb/nc-fix/at-stall

fix: at import stall issue
2026-02-01 23:48:33 +00:00 · 2024-09-21 00:45:39 +05:30
parent e8093293c4 37575f4094
commit 25352e96cc
4 changed files with 210 additions and 158 deletions
--- a/packages/nocodb/src/models/Column.ts
+++ b/packages/nocodb/src/models/Column.ts
@@ -816,6 +816,7 @@ export default class Column<T = any> implements ColumnType {

        if (button.type === 'url') {
          if (
+            button.formula &&
            addFormulaErrorIfMissingColumn({
              formula: button,
              columnId: id,
@@ -858,6 +859,7 @@ export default class Column<T = any> implements ColumnType {
          formulaCol,
        ).getColOptions<FormulaColumn>(context, ncMeta);
        if (
+          formula.formula &&
          addFormulaErrorIfMissingColumn({
            formula,
            columnId: id,
--- a/packages/nocodb/src/modules/jobs/jobs.module.ts
+++ b/packages/nocodb/src/modules/jobs/jobs.module.ts
@@ -54,6 +54,7 @@ export const JobsModuleMetadata = {
            name: JOBS_QUEUE,
            defaultJobOptions: {
              removeOnComplete: true,
+              attempts: 1,
            },
          }),
        ]
--- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
+++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
@@ -469,7 +469,7 @@ export class AtImportProcessor {
            if (
              options.find(
                (el) =>
-                  el.title.toLowerCase() === (value as any).name.toLowerCase(),
+                  el.title === (value as any).name,
              )
            ) {
              logWarning(
--- a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
+++ b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
@@ -1,5 +1,8 @@
 import { Logger } from '@nestjs/common';
 import axios from 'axios';
+import { streamObject } from 'stream-json/streamers/StreamObject';
+import { parser } from 'stream-json/Parser';
+import { ignore } from 'stream-json/filters/Ignore';

 const logger = new Logger('FetchAT');

@@ -17,45 +20,37 @@ async function initialize(shareId, appId?: string) {
  const url = `https://airtable.com/${appId ? `${appId}/` : ''}${shareId}`;

  try {
-    const hreq = await axios
-      .get(url, {
-        headers: {
-          accept:
-            'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-          'accept-language': 'en-US,en;q=0.9',
-          'sec-ch-ua':
-            '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
-          'sec-ch-ua-mobile': '?0',
-          'sec-ch-ua-platform': '"Linux"',
-          'sec-fetch-dest': 'document',
-          'sec-fetch-mode': 'navigate',
-          'sec-fetch-site': 'none',
-          'sec-fetch-user': '?1',
-          'upgrade-insecure-requests': '1',
-          'User-Agent':
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
-        },
-        // @ts-ignore
-        referrerPolicy: 'strict-origin-when-cross-origin',
-        body: null,
-        method: 'GET',
-      })
-      .then((response) => {
-        for (const ck of response.headers['set-cookie']) {
-          info.cookie += ck.split(';')[0] + '; ';
-        }
-        return response.data;
-      })
-      .catch((e) => {
-        logger.log(e);
-        throw {
-          message:
-            'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
-        };
-      });
+    const hreq = await axios.get(url, {
+      headers: {
+        accept:
+          'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+        'accept-language': 'en-US,en;q=0.9',
+        'sec-ch-ua':
+          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"Linux"',
+        'sec-fetch-dest': 'document',
+        'sec-fetch-mode': 'navigate',
+        'sec-fetch-site': 'none',
+        'sec-fetch-user': '?1',
+        'upgrade-insecure-requests': '1',
+        'User-Agent':
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+      },
+      // @ts-ignore
+      referrerPolicy: 'strict-origin-when-cross-origin',
+      body: null,
+      method: 'GET',
+    });

-    const headers = hreq.match(/(?<=var headers =)(.*)(?=;)/g);
-    const link = hreq.match(/(?<=fetch\(")(\\.*)(?=")/g);
+    for (const ck of hreq.headers['set-cookie']) {
+      info.cookie += ck.split(';')[0] + '; ';
+    }
+
+    const data = hreq.data;
+
+    const headers = data.match(/(?<=var headers =)(.*)(?=;)/g);
+    const link = data.match(/(?<=fetch\(")(\\.*)(?=")/g);

    if (!headers || !link) {
      throw {
@@ -65,11 +60,11 @@ async function initialize(shareId, appId?: string) {
    }

    info.headers = JSON.parse(
-      hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
+      data.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
    );

    info.link = unicodeToChar(
-      hreq.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
+      data.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
    );

    info.link = info.link.replace(
@@ -112,73 +107,8 @@ async function initialize(shareId, appId?: string) {

 async function read() {
  if (info.initialized) {
-    const resreq = await axios('https://airtable.com' + info.link, {
-      headers: {
-        accept: '*/*',
-        'accept-language': 'en-US,en;q=0.9',
-        'sec-ch-ua':
-          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
-        'sec-ch-ua-mobile': '?0',
-        'sec-ch-ua-platform': '"Linux"',
-        'sec-fetch-dest': 'empty',
-        'sec-fetch-mode': 'cors',
-        'sec-fetch-site': 'same-origin',
-        'User-Agent':
-          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
-        'x-time-zone': 'Europe/Berlin',
-        cookie: info.cookie,
-        ...info.headers,
-      },
-      // @ts-ignore
-      referrerPolicy: 'no-referrer',
-      body: null,
-      method: 'GET',
-    })
-      .then((response) => {
-        return response.data;
-      })
-      .catch((e) => {
-        logger.log(e);
-        throw {
-          message:
-            'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
-        };
-      });
-
-    return {
-      schema: resreq.data,
-      baseId: info.baseId,
-      baseInfo: info.baseInfo,
-    };
-  } else {
-    throw {
-      message: 'Error Initializing :: please try again !!',
-    };
-  }
-}
-
-async function readView(viewId) {
-  if (info.initialized) {
-    const resreq = await axios(
-      `https://airtable.com/v0.3/view/${viewId}/readData?` +
-        `stringifiedObjectParams=${encodeURIComponent(
-          JSON.stringify({
-            mayOnlyIncludeRowAndCellDataForIncludedViews: true,
-            mayExcludeCellDataForLargeViews: true,
-          }),
-        )}&requestId=${
-          info.baseInfo.requestId
-        }&accessPolicy=${encodeURIComponent(
-          JSON.stringify({
-            allowedActions: info.baseInfo.allowedActions,
-            shareId: info.baseInfo.shareId,
-            applicationId: info.baseInfo.applicationId,
-            generationNumber: info.baseInfo.generationNumber,
-            expires: info.baseInfo.expires,
-            signature: info.baseInfo.signature,
-          }),
-        )}`,
-      {
+    try {
+      const resreq = await axios('https://airtable.com' + info.link, {
        headers: {
          accept: '*/*',
          'accept-language': 'en-US,en;q=0.9',
@@ -199,19 +129,134 @@ async function readView(viewId) {
        referrerPolicy: 'no-referrer',
        body: null,
        method: 'GET',
-      },
-    )
-      .then((response) => {
-        return response.data;
-      })
-      .catch((e) => {
-        logger.log(e);
-        throw {
-          message:
-            'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
-        };
+        responseType: 'stream',
      });
-    return { view: resreq.data };
+
+      const data: any = await new Promise((resolve, reject) => {
+        const jsonStream = resreq.data
+          .pipe(parser())
+          .pipe(ignore({ filter: 'data.tableDatas' }))
+          .pipe(streamObject());
+
+        const fullObject = {};
+
+        jsonStream.on('data', (chunk) => {
+          if (chunk.key) fullObject[chunk.key] = chunk.value;
+        });
+
+        jsonStream.on('error', (err) => {
+          reject(err);
+        });
+
+        jsonStream.on('end', () => {
+          resolve(fullObject);
+        });
+      });
+
+      if (data?.data) {
+        return {
+          schema: data?.data,
+          baseId: info.baseId,
+          baseInfo: info.baseInfo,
+        };
+      } else {
+        throw new Error('Error Reading :: Data missing');
+      }
+    } catch (e) {
+      logger.log(e);
+      throw {
+        message:
+          'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+      };
+    }
+  } else {
+    throw {
+      message: 'Error Initializing :: please try again !!',
+    };
+  }
+}
+
+async function readView(viewId) {
+  if (info.initialized) {
+    try {
+      const resreq = await axios(
+        `https://airtable.com/v0.3/view/${viewId}/readData?` +
+          `stringifiedObjectParams=${encodeURIComponent(
+            JSON.stringify({
+              mayOnlyIncludeRowAndCellDataForIncludedViews: true,
+              mayExcludeCellDataForLargeViews: true,
+            }),
+          )}&requestId=${
+            info.baseInfo.requestId
+          }&accessPolicy=${encodeURIComponent(
+            JSON.stringify({
+              allowedActions: info.baseInfo.allowedActions,
+              shareId: info.baseInfo.shareId,
+              applicationId: info.baseInfo.applicationId,
+              generationNumber: info.baseInfo.generationNumber,
+              expires: info.baseInfo.expires,
+              signature: info.baseInfo.signature,
+            }),
+          )}`,
+        {
+          headers: {
+            accept: '*/*',
+            'accept-language': 'en-US,en;q=0.9',
+            'sec-ch-ua':
+              '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"Linux"',
+            'sec-fetch-dest': 'empty',
+            'sec-fetch-mode': 'cors',
+            'sec-fetch-site': 'same-origin',
+            'User-Agent':
+              'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+            'x-time-zone': 'Europe/Berlin',
+            cookie: info.cookie,
+            ...info.headers,
+          },
+          // @ts-ignore
+          referrerPolicy: 'no-referrer',
+          body: null,
+          method: 'GET',
+          responseType: 'stream',
+        },
+      );
+
+      const data: any = await new Promise((resolve, reject) => {
+        const jsonStream = resreq.data
+          .pipe(parser())
+          .pipe(ignore({ filter: 'data.rowOrder' }))
+          .pipe(streamObject());
+
+        const fullObject = {};
+
+        jsonStream.on('data', (chunk) => {
+          if (chunk.key) fullObject[chunk.key] = chunk.value;
+        });
+
+        jsonStream.on('error', (err) => {
+          reject(err);
+        });
+
+        jsonStream.on('end', () => {
+          resolve(fullObject);
+        });
+      });
+      
+
+      if (data?.data) {
+        return { view: data.data };
+      } else {
+        throw new Error('Error Reading :: Data missing');
+      }
+    } catch (e) {
+      logger.log(e);
+      throw {
+        message:
+          'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+      };
+    }
  } else {
    throw {
      message: 'Error Initializing :: please try again !!',
@@ -223,45 +268,49 @@ async function readTemplate(templateId) {
  if (!info.initialized) {
    await initialize('shrO8aYf3ybwSdDKn');
  }
-  const resreq = await axios(
-    `https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
-    {
-      headers: {
-        accept: '*/*',
-        'accept-language': 'en-US,en;q=0.9',
-        'sec-ch-ua':
-          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
-        'sec-ch-ua-mobile': '?0',
-        'sec-ch-ua-platform': '"Linux"',
-        'sec-fetch-dest': 'empty',
-        'sec-fetch-mode': 'cors',
-        'sec-fetch-site': 'same-origin',
-        'User-Agent':
-          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
-        'x-time-zone': 'Europe/Berlin',
-        cookie: info.cookie,
-        ...info.headers,
+
+  try {
+    const resreq = await axios(
+      `https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
+      {
+        headers: {
+          accept: '*/*',
+          'accept-language': 'en-US,en;q=0.9',
+          'sec-ch-ua':
+            '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+          'sec-ch-ua-mobile': '?0',
+          'sec-ch-ua-platform': '"Linux"',
+          'sec-fetch-dest': 'empty',
+          'sec-fetch-mode': 'cors',
+          'sec-fetch-site': 'same-origin',
+          'User-Agent':
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+          'x-time-zone': 'Europe/Berlin',
+          cookie: info.cookie,
+          ...info.headers,
+        },
+        // @ts-ignore
+        referrer: 'https://www.airtable.com/',
+        referrerPolicy: 'same-origin',
+        body: null,
+        method: 'GET',
+        mode: 'cors',
+        credentials: 'include',
      },
-      // @ts-ignore
-      referrer: 'https://www.airtable.com/',
-      referrerPolicy: 'same-origin',
-      body: null,
-      method: 'GET',
-      mode: 'cors',
-      credentials: 'include',
-    },
-  )
-    .then((response) => {
-      return response.data;
-    })
-    .catch((e) => {
-      logger.log(e);
-      throw {
-        message:
-          'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
-      };
-    });
-  return { template: resreq };
+    );
+
+    if (resreq?.data) {
+      return { template: resreq.data };
+    } else {
+      throw new Error('Error Reading :: Data missing');
+    }
+  } catch (e) {
+    logger.log(e);
+    throw {
+      message:
+        'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
+    };
+  }
 }

 function unicodeToChar(text) {