clone/nocodb
1
0
Fork 0
mirror of https://github.com/nocodb/nocodb.git synced 2026-04-25 12:05:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

mock airtable import when playwright

Browse Source
This commit is contained in:
Fendy Heryanto
2026-01-22 07:14:35 +00:00
parent c190e04b67
commit 51f9c3e765
27 changed files with 13563 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
View File

@@ -1,6 +1,5 @@
import moment from 'moment';
import { AuditV1OperationTypes, SqlUiFactory, UITypes } from 'nocodb-sdk';
import Airtable from 'airtable';
import hash from 'object-hash';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc';
@@ -13,6 +12,7 @@ import { JobsLogService } from '../jobs-log.service';
import FetchAT from './helpers/fetchAT';
import { importData } from './helpers/readAndProcessData';
import EntityMap from './helpers/EntityMap';
import { ATImportEngine } from './engine';
import type {
AirtableImportFailPayload,
AirtableImportPayload,
@@ -292,7 +292,6 @@ export class AtImportProcessor {
const getAirtableSchema = async (sDB) => {
const start = Date.now();
if (!sDB.shareId)
throw {
message:
@@ -319,7 +318,10 @@ export class AtImportProcessor {
const file = ft.schema;
atBaseId = ft.baseId;
atBase = new Airtable({ apiKey: sDB.apiKey }).base(atBaseId);
atBase = ATImportEngine.get().atBase({
apiKey: sDB.apiKey,
baseId: atBaseId,
});
// store copy of airtable schema globally
g_aTblSchema = file.tableSchemas;

122
packages/nocodb/src/modules/jobs/jobs/at-import/engine/index.ts Normal file
View File

@@ -0,0 +1,122 @@
import axios from 'axios';
import Airtable from 'airtable';
import { ATMockImportEngine, MockAirtable } from './mock';
import type { AirtableBase } from 'airtable/lib/airtable_base';
import { isPlayWrightNode } from '~/helpers/utils';
export class ATImportEngine extends ATMockImportEngine {
static get() {
if (isPlayWrightNode()) {
return new ATMockImportEngine();
}
return new ATImportEngine();
}
async initialize({ appId, shareId }: { appId: string; shareId: string }) {
const url = `https://airtable.com/${appId ? `${appId}/` : ''}${shareId}`;
return await axios.get(url, {
headers: {
accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
},
// @ts-ignore
referrerPolicy: 'strict-origin-when-cross-origin',
body: null,
method: 'GET',
});
}
async read(info: { link: string; cookie: string; headers: any }) {
return await axios('https://airtable.com' + info.link, {
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
// @ts-ignore
referrerPolicy: 'no-referrer',
body: null,
method: 'GET',
responseType: 'stream',
});
}
async readView(
viewId: string,
info: { cookie: string; headers: any; baseInfo: any },
) {
return await axios(
`https://airtable.com/v0.3/view/${viewId}/readData?` +
`stringifiedObjectParams=${encodeURIComponent(
JSON.stringify({
mayOnlyIncludeRowAndCellDataForIncludedViews: true,
mayExcludeCellDataForLargeViews: true,
}),
)}&requestId=${
info.baseInfo.requestId
}&accessPolicy=${encodeURIComponent(
JSON.stringify({
allowedActions: info.baseInfo.allowedActions,
shareId: info.baseInfo.shareId,
applicationId: info.baseInfo.applicationId,
generationNumber: info.baseInfo.generationNumber,
expires: info.baseInfo.expires,
signature: info.baseInfo.signature,
}),
)}`,
{
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
// @ts-ignore
referrerPolicy: 'no-referrer',
body: null,
method: 'GET',
responseType: 'stream',
},
);
}
atBase({ apiKey, baseId }: { apiKey: string; baseId: string }) {
if (process.env.DEBUG_MOCK_AIRTABLE_IMPORT === 'true') {
return ((title) => new MockAirtable(title)) as any as AirtableBase;
}
return new Airtable({ apiKey: apiKey }).base(baseId);
}
}

74
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mock.ts Normal file
View File

@@ -0,0 +1,74 @@
import { Readable } from 'node:stream';
import {
initializeHeader,
initializeHtml,
readResponse,
viewsResponse,
} from './mockResponses';
import { mockResponseData } from './mockResponseData';
import type { AirtableBase } from 'airtable/lib/airtable_base';
import type { FieldSet, Records } from 'airtable';
import type { AxiosResponse } from 'axios';
export class ATMockImportEngine {
async initialize(_param: { appId: string; shareId: string }) {
return {
data: initializeHtml, // ← this is what axios returns
status: 200,
statusText: 'OK',
headers: initializeHeader,
config: {},
} as AxiosResponse;
}
async read(_info: { link: string; cookie: string; headers: any }) {
const stream = Readable.from([JSON.stringify(readResponse)]);
return {
data: stream, // ← this is what axios returns
status: 200,
statusText: 'OK',
headers: {},
config: {},
} as AxiosResponse;
}
async readView(viewId: string, _info: { baseInfo: any }) {
const stream = Readable.from([JSON.stringify(viewsResponse[viewId])]);
return {
data: stream, // ← this is what axios returns
status: 200,
statusText: 'OK',
headers: {},
config: {},
} as AxiosResponse;
}
atBase(_param: { apiKey: string; baseId: string }) {
return ((title) => new MockAirtable(title)) as any as AirtableBase;
}
}
export class MockAirtable {
constructor(protected readonly title: string) {}
select(_selectParams: any) {
return this;
}
eachPage(
pageHandle: (
records: Records<FieldSet>,
fetchNextPage: () => void,
) => Promise<void>,
done: (err: any) => Promise<void>,
) {
void (async () => {
let cursor = 0;
let currentRecord = mockResponseData[this.title][cursor++];
while (currentRecord) {
await pageHandle(currentRecord.records, () => {
currentRecord = mockResponseData[this.title][cursor++];
});
}
done(undefined);
})();
}
}

6346
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponseData/Actor.json Normal file
View File

File diff suppressed because it is too large Load Diff

1735
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponseData/Film.json Normal file
View File

File diff suppressed because it is too large Load Diff

81
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponseData/Producer.json Normal file
View File

@@ -0,0 +1,81 @@
[
{
"records": [
{
"_table": {
"_base": {
"_airtable": {},
"_id": "apprb2mFeIlQitU0o"
},
"id": null,
"name": "Producer"
},
"id": "rec7CflqABElKe2iz",
"_rawJson": {
"id": "rec7CflqABElKe2iz",
"createdTime": "2022-06-04T10:21:14.000Z",
"fields": {
"Name": "P2",
"Notes": "Notes of P2",
"Status": "In progress"
}
},
"fields": {
"Name": "P2",
"Notes": "Notes of P2",
"Status": "In progress"
}
},
{
"_table": {
"_base": {
"_airtable": {},
"_id": "apprb2mFeIlQitU0o"
},
"id": null,
"name": "Producer"
},
"id": "rectTeNHuBbaFBjAv",
"_rawJson": {
"id": "rectTeNHuBbaFBjAv",
"createdTime": "2022-06-04T10:21:14.000Z",
"fields": {
"Name": "P1",
"Notes": "Notes of P1",
"Status": "Todo"
}
},
"fields": {
"Name": "P1",
"Notes": "Notes of P1",
"Status": "Todo"
}
},
{
"_table": {
"_base": {
"_airtable": {},
"_id": "apprb2mFeIlQitU0o"
},
"id": null,
"name": "Producer"
},
"id": "recxenRrH5sVUoV8M",
"_rawJson": {
"id": "recxenRrH5sVUoV8M",
"createdTime": "2022-06-04T10:21:14.000Z",
"fields": {
"Name": "P3",
"Notes": "Notes of P3",
"Status": "Done"
}
},
"fields": {
"Name": "P3",
"Notes": "Notes of P3",
"Status": "Done"
}
}
]
}
]

9
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponseData/index.ts Normal file
View File

@@ -0,0 +1,9 @@
import ActorData from './Actor.json';
import FilmData from './Film.json';
import ProducerData from './Producer.json';
export const mockResponseData = {
Actor: ActorData,
Film: FilmData,
Producer: ProducerData,
};

38
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/index.ts Normal file
View File

@@ -0,0 +1,38 @@
export {
responseData as initializeHtml,
responseHeaders as initializeHeader,
} from './initialize';
export { responseData as readResponse } from './read';
import readView_viw5f7BhLA3OkthNv from './readView_viw5f7BhLA3OkthNv.json';
import readView_viw9HVTL3HmZ3NCDd from './readView_viw9HVTL3HmZ3NCDd.json';
import readView_viwsF12Zep2IrUQ4L from './readView_viwsF12Zep2IrUQ4L.json';
import readView_viwc6YeYMlRUFJUYa from './readView_viwc6YeYMlRUFJUYa.json';
import readView_viwvu6dBtJcPFvl4b from './readView_viwvu6dBtJcPFvl4b.json';
import readView_viwebeyBYaiCnNwwO from './readView_viwebeyBYaiCnNwwO.json';
import readView_viwWiQnKWgbL2TGLw from './readView_viwWiQnKWgbL2TGLw.json';
import readView_viwMKGcEPNkD797BF from './readView_viwMKGcEPNkD797BF.json';
import readView_viwWYcqdMbeOx3Eg3 from './readView_viwWYcqdMbeOx3Eg3.json';
import readView_viw8p8zWYmN4Kwine from './readView_viw8p8zWYmN4Kwine.json';
import readView_viwqB059qatxSF8tI from './readView_viwqB059qatxSF8tI.json';
import readView_viwXfVqIcM25FVTbd from './readView_viwXfVqIcM25FVTbd.json';
import readView_viw8Xse051lREItSC from './readView_viw8Xse051lREItSC.json';
import readView_viwRRK2UljiQB3kko from './readView_viwRRK2UljiQB3kko.json';
import readView_viwZnwvKkOBnDqpul from './readView_viwZnwvKkOBnDqpul.json';
export const viewsResponse = {
viw5f7BhLA3OkthNv: { data: readView_viw5f7BhLA3OkthNv },
viw9HVTL3HmZ3NCDd: { data: readView_viw9HVTL3HmZ3NCDd },
viwsF12Zep2IrUQ4L: { data: readView_viwsF12Zep2IrUQ4L },
viwc6YeYMlRUFJUYa: { data: readView_viwc6YeYMlRUFJUYa },
viwvu6dBtJcPFvl4b: { data: readView_viwvu6dBtJcPFvl4b },
viwebeyBYaiCnNwwO: { data: readView_viwebeyBYaiCnNwwO },
viwWiQnKWgbL2TGLw: { data: readView_viwWiQnKWgbL2TGLw },
viwMKGcEPNkD797BF: { data: readView_viwMKGcEPNkD797BF },
viwWYcqdMbeOx3Eg3: { data: readView_viwWYcqdMbeOx3Eg3 },
viw8p8zWYmN4Kwine: { data: readView_viw8p8zWYmN4Kwine },
viwqB059qatxSF8tI: { data: readView_viwqB059qatxSF8tI },
viwXfVqIcM25FVTbd: { data: readView_viwXfVqIcM25FVTbd },
viw8Xse051lREItSC: { data: readView_viw8Xse051lREItSC },
viwRRK2UljiQB3kko: { data: readView_viwRRK2UljiQB3kko },
viwZnwvKkOBnDqpul: { data: readView_viwZnwvKkOBnDqpul },
};

470
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/initialize.ts Normal file
View File

File diff suppressed because one or more lines are too long

4661
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/read.ts Normal file
View File

File diff suppressed because it is too large Load Diff

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viw5f7BhLA3OkthNv.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viw5f7BhLA3OkthNv","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":false},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"filters":null,"lastSortsApplied":null,"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"grid","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viw8Xse051lREItSC.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viw8Xse051lREItSC","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"filters":null,"lastSortsApplied":null,"groupLevels":null,"colorConfig":null,"sharesById":{},"metadata":{"gallery":{"coverColumnId":"fldeuBlvCNuVLTpag"}},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"gallery","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viw8p8zWYmN4Kwine.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viw8p8zWYmN4Kwine","frozenColumnCount":1,"columnOrder":[{"columnId":"flduxFfvDassOS6sm","visibility":true},{"columnId":"fldBPTD14QIXSNarb","visibility":true},{"columnId":"fldn3XBgPr3SEbsXv","visibility":false},{"columnId":"fldXxq8VN0UhMy1xR","visibility":true},{"columnId":"fldrqN5ajLiEwNy0t","visibility":true}],"filters":{"filterSet":[{"id":"fltn1dZmC4SnYcCVt","columnId":"flduxFfvDassOS6sm","operator":"contains","value":"1"},{"id":"fltQefbSq0634Cxom","columnId":"flduxFfvDassOS6sm","operator":"contains","value":"2"}],"conjunction":"or"},"lastSortsApplied":{"sortSet":[{"id":"srtz6yNGgfc2hORy3","columnId":"flduxFfvDassOS6sm","ascending":true}],"shouldAutoSort":true,"appliedTime":"2022-06-04T10:32:22.736Z"},"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"grid","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viw9HVTL3HmZ3NCDd.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viw9HVTL3HmZ3NCDd","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"filters":null,"lastSortsApplied":null,"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"grid","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwMKGcEPNkD797BF.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwMKGcEPNkD797BF","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"form","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwRRK2UljiQB3kko.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwRRK2UljiQB3kko","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"filters":null,"lastSortsApplied":null,"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"grid","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwWYcqdMbeOx3Eg3.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwWYcqdMbeOx3Eg3","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"form","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwWiQnKWgbL2TGLw.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwWiQnKWgbL2TGLw","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"filters":null,"lastSortsApplied":null,"groupLevels":null,"colorConfig":null,"sharesById":{},"metadata":{"gallery":{"coverColumnId":"fldeuBlvCNuVLTpag"}},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"gallery","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwXfVqIcM25FVTbd.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwXfVqIcM25FVTbd","frozenColumnCount":1,"columnOrder":[{"columnId":"fldH7RPO1Ik2ienk7","visibility":false,"width":180},{"columnId":"fldSx61dqgUzHTn1l","visibility":false},{"columnId":"fld6r7a0q9hodzUgY","visibility":true},{"columnId":"fld6IgZRozMKeFyc5","visibility":false},{"columnId":"fldqMqzYMQ7i85g5O","visibility":false},{"columnId":"fldlEkObSXpshw2wk","visibility":false},{"columnId":"fldqSMGqzoIKvexVL","visibility":false},{"columnId":"fldC1wGc9klzZw8nE","visibility":false},{"columnId":"fldpGVOU1gA9CZUP9","visibility":true},{"columnId":"fldg6nJJBJAxGnb8m","visibility":false},{"columnId":"fldBvDdsV1k24yKDu","visibility":false,"width":168},{"columnId":"fld2cA23dqfpiQ37p","visibility":false},{"columnId":"fldlebEyEK11MAXqP","visibility":false},{"columnId":"fld62jI4iMGIWBuAH","visibility":false},{"columnId":"fldwMZogQefTVfwfT","visibility":false},{"columnId":"fldmVzyNoFUOktFba","visibility":false},{"columnId":"fldoQ9fOkIU5Mhrtp","visibility":false},{"columnId":"fldBAkWGWoEQaM5uK","visibility":false},{"columnId":"fldrUlRlnWNRHa5Wn","visibility":false}],"groupLevels":null,"colorConfig":null,"sharesById":{},"metadata":{"form":{"description":"FormDescription","fieldsByColumnId":{"fld6r7a0q9hodzUgY":{"title":"DisplayName","description":"HelpText","required":true}},"refreshAfterSubmit":"AUTO_REFRESH_AND_REFRESH_BUTTON"}},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"form","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwZnwvKkOBnDqpul.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwZnwvKkOBnDqpul","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"form","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwc6YeYMlRUFJUYa.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwc6YeYMlRUFJUYa","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"form","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwebeyBYaiCnNwwO.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwebeyBYaiCnNwwO","frozenColumnCount":1,"columnOrder":[{"columnId":"fld6r7a0q9hodzUgY","visibility":true},{"columnId":"fld6IgZRozMKeFyc5","visibility":true},{"columnId":"fldqMqzYMQ7i85g5O","visibility":false},{"columnId":"fldlEkObSXpshw2wk","visibility":true},{"columnId":"fldH7RPO1Ik2ienk7","visibility":true,"width":180},{"columnId":"fldSx61dqgUzHTn1l","visibility":true},{"columnId":"fldqSMGqzoIKvexVL","visibility":true},{"columnId":"fldC1wGc9klzZw8nE","visibility":true},{"columnId":"fldpGVOU1gA9CZUP9","visibility":true},{"columnId":"fldg6nJJBJAxGnb8m","visibility":true},{"columnId":"fldBvDdsV1k24yKDu","visibility":true,"width":168},{"columnId":"fld2cA23dqfpiQ37p","visibility":true},{"columnId":"fldlebEyEK11MAXqP","visibility":true},{"columnId":"fld62jI4iMGIWBuAH","visibility":true},{"columnId":"fldwMZogQefTVfwfT","visibility":true},{"columnId":"fldmVzyNoFUOktFba","visibility":true},{"columnId":"fldoQ9fOkIU5Mhrtp","visibility":true,"width":252},{"columnId":"fldBAkWGWoEQaM5uK","visibility":true},{"columnId":"fldrUlRlnWNRHa5Wn","visibility":true}],"filters":null,"lastSortsApplied":null,"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"grid","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwqB059qatxSF8tI.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwqB059qatxSF8tI","frozenColumnCount":1,"columnOrder":[{"columnId":"flduxFfvDassOS6sm","visibility":true},{"columnId":"fldBPTD14QIXSNarb","visibility":true},{"columnId":"fldn3XBgPr3SEbsXv","visibility":true},{"columnId":"fldXxq8VN0UhMy1xR","visibility":true},{"columnId":"fldrqN5ajLiEwNy0t","visibility":true}],"filters":null,"lastSortsApplied":null,"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"grid","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwsF12Zep2IrUQ4L.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwsF12Zep2IrUQ4L","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"filters":null,"lastSortsApplied":null,"groupLevels":null,"colorConfig":null,"sharesById":{},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"grid","applicationTransactionNumber":4,"signedUserContentUrls":{}}

1
packages/nocodb/src/modules/jobs/jobs/at-import/engine/mockResponses/readView_viwvu6dBtJcPFvl4b.json Normal file
View File

@@ -0,0 +1 @@
{"id":"viwvu6dBtJcPFvl4b","frozenColumnCount":1,"columnOrder":[{"columnId":"fldaKGmXYDqycI6FP","visibility":true},{"columnId":"fldJ4MZo6XbVBK8Cm","visibility":true},{"columnId":"fldeuBlvCNuVLTpag","visibility":true},{"columnId":"fldi3aP7nI5d7Th0S","visibility":true}],"filters":null,"lastSortsApplied":null,"groupLevels":null,"colorConfig":null,"sharesById":{},"metadata":{"gallery":{"coverColumnId":"fldeuBlvCNuVLTpag"}},"description":null,"createdByUserId":"usr9GWEED0hhrQ3P7","type":"gallery","applicationTransactionNumber":4,"signedUserContentUrls":{}}

94
packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
View File

@@ -3,6 +3,7 @@ import axios from 'axios';
import { streamObject } from 'stream-json/streamers/StreamObject';
import { parser } from 'stream-json/Parser';
import { ignore } from 'stream-json/filters/Ignore';
import { ATImportEngine } from '../engine';
const logger = new Logger('FetchAT');
@@ -17,31 +18,8 @@ async function initialize(shareId, appId?: string) {
appId = null;
}
const url = `https://airtable.com/${appId ? `${appId}/` : ''}${shareId}`;
try {
const hreq = await axios.get(url, {
headers: {
accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
},
// @ts-ignore
referrerPolicy: 'strict-origin-when-cross-origin',
body: null,
method: 'GET',
});
const hreq = await ATImportEngine.get().initialize({ appId, shareId });
for (const ck of hreq.headers['set-cookie']) {
info.cookie += ck.split(';')[0] + '; ';
@@ -108,29 +86,7 @@ async function initialize(shareId, appId?: string) {
async function read() {
if (info.initialized) {
try {
const resreq = await axios('https://airtable.com' + info.link, {
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
// @ts-ignore
referrerPolicy: 'no-referrer',
body: null,
method: 'GET',
responseType: 'stream',
});
const resreq = await ATImportEngine.get().read(info);
const data: any = await new Promise((resolve, reject) => {
const jsonStream = resreq.data
@@ -179,49 +135,7 @@ async function read() {
async function readView(viewId) {
if (info.initialized) {
try {
const resreq = await axios(
`https://airtable.com/v0.3/view/${viewId}/readData?` +
`stringifiedObjectParams=${encodeURIComponent(
JSON.stringify({
mayOnlyIncludeRowAndCellDataForIncludedViews: true,
mayExcludeCellDataForLargeViews: true,
}),
)}&requestId=${
info.baseInfo.requestId
}&accessPolicy=${encodeURIComponent(
JSON.stringify({
allowedActions: info.baseInfo.allowedActions,
shareId: info.baseInfo.shareId,
applicationId: info.baseInfo.applicationId,
generationNumber: info.baseInfo.generationNumber,
expires: info.baseInfo.expires,
signature: info.baseInfo.signature,
}),
)}`,
{
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
// @ts-ignore
referrerPolicy: 'no-referrer',
body: null,
method: 'GET',
responseType: 'stream',
},
);
const resreq = await ATImportEngine.get().readView(viewId, info);
const data: any = await new Promise((resolve, reject) => {
const jsonStream = resreq.data

6
tests/playwright/constants/index.ts
View File

@@ -1,7 +1,7 @@
const airtableApiBases = [
'https://airtable.com/appZtc5xOWQkUL4Ok/shrrfpcWoNa5QGGSI',
'https://airtable.com/appBnSP4VbJtKTTHB/shrfiNNg3yk9A0APu',
'https://airtable.com/appPFK0xj5jTQR6wQ/shr6zjJvrUbSvfGPq',
// 'https://airtable.com/appZtc5xOWQkUL4Ok/shrrfpcWoNa5QGGSI',
// 'https://airtable.com/appBnSP4VbJtKTTHB/shrfiNNg3yk9A0APu',
// 'https://airtable.com/appPFK0xj5jTQR6wQ/shr6zjJvrUbSvfGPq',
'https://airtable.com/apprb2mFeIlQitU0o/shrY6swT2FOy9X1Bl',
];
const today = new Date();