Nc feat/integrations (#8903)

* feat: integrations backend (WIP)

* feat: migration - source table

* feat: updated migration

* feat: integration APIs - WIP

* feat: integration - crud, acl, api tests

* feat: integration - crud, acl, api tests

* feat: integration - GUI integration

* feat: private integration config

* feat: integration GUI

* feat: delete api and source creation

* feat: add hint for input fields

* fix: source creation bugs

* refactor: placeholder text correction

* refactor: include context

* feat: integration delete with transaction

* refactor: permission scope correction and move ee logic

* refactor: migration correction and improvements

* feat: confirm dialog

* refactor: review comments

* refactor: meta service changes

* feat: add oss support - WIP

* feat: add oss support

* refactor: coderabbt suggestions

* refactor: exclude config from api response

* refactor: coderabbit review comments

* refactor: rename migration names

* fix: method name correction

* fix(nc-gui): integration ui changes

* fix(nc-gui): add edit integration ui changes

* fix(nc-gui): add shared badge in integrations list

* feat(nc-gui): duplicate integration

* fix(nocodb): add copy from id integration support in create integration api

* fix(nc-gui): update useIntegration store

* fix(nc-gui): test connection btn style update

* fix(nc-gui): update new integration modal

* feat(nc-gui): add sort integration list support

* fix(nc-gui): integration table to be center aligned

* fix(nc-gui): move form item required mark to right side

* fix: remove divider

* fix(nc-gui): add input shadow

* fix(nc-gui): base name validator error message

* fix(nc-gui): add border if search connection input has some value

* fix(nc-gui): add close btn in integration modal

* chore(nc-gui): lint

* fix(nc-gui): pr review changes

* chore(nc-gui): cleanup unused code

* chore(nc-gui): lint

* fix(nc-gui): integrationsType not found issue

* fix(nc-gui): update data source table

* fix(nc-gui): populate integration name only on input value change

* fix(nc-gui): create data source form update

* fix: type correction

* fix: label correction

* fix: font corrections

* fix: remove help text

* fix: grammar in help text

* fix(nc-gui): edit source ui changes

* fix(nc-gui): base settings modal changes & datasource search feat

* fix(nc-gui): update data source table

* fix(nc-gui): move integrations outside team & settings

* fix(nc-gui): make connections table full width

* fix(nc-gui): modal height issue in small screen

* fix(nc-gui): disable editing selected connection in edit data source

* fix(nc-gui): add data sources in base settings tab

* fix(nc-gui): ant design multiple warnings issue

* fix(nc-gui): create source page scrollbar issue

* feat(nc-gui): create connection from create source page

* chore(nc-gui): lint

* fix(nc-gui): update project members tab content margin

* chore: label text change

* fix: font changes

* chore: font corrections

* chore: integration => connection

* fix(nc-gui): disable auto editing database name on changing connection name

* fix(nc-gui): table header overflow issue

* fix(nc-gui): show connection crud operation messages in toast

* feat(nc-gui): request new integration ui

* fix(nc-gui): text area height adjust issue

* fix(nc-gui): add connection from source create issue

* fix(nc-gui): show data source details in modal

* fix(nc-gui): hide private connection option

* fix(nc-gui): user should able to edit & save connection without test connection if only title updated

* fix(nc-gui): add integration page in oss

* fix(nc-gui): typo currection

* fix(nc-gui): oss create base ui changes

* misc: minor formatting changes

* misc: formatting corrections

* fix(nc-gui): overlay close btn issue

* fix(nc-gui): some review changes

* fix(nc-gui): remove link beetween connection name & database name

* fix(nc-gui): update edit base/source modal oss

* fix(nc-gui): add db type icon in select connection

* chore(nc-gui): lint

* fix: integration list - allow access based on base level role

* fix(nc-gui): load integrations on creating integration from source create issue

* fix(nc-gui): add connection count in tab

* fix: correction in soft delete logic

* fix(nc-gui): reset use ssl on panel collapse

* fix(nc-gui): reduce select input font weight

* fix(nc-gui): update connection edit access control

* fix: integration read api correction

* fix(nc-gui): some review changes

* fix(nc-gui): labels update

* fix(nc-gui): udpate text in delete modal integration -> connection

* fix: remove permission from wrong scope

* refactor: swagger description correction

* fix(nc-gui): remove connection between source name & database name

* fix(nc-gui): test connection is not needed form source name. inflection field changes

* refactor: include integration title with source

* feat: integration pagination

* fix: remove unused prop

* fix(nc-gui): update all tables tab btns tooltip

* feat: new integration request

* refactor: replace delete statement and use assigning undefined for better performance

* feat(nc-gui): sync data support in project page

* fix(nc-gui): all sync data type list

* fix(nc-gui): close sync data modal issue

* fix(nc-gui): add bg gray color on db icon of tooltip

* fix(nc-gui): make connection as required field

* fix(nc-gui): show connection name if not found and reload page

* fix(nc-gui): show connection name in ds list

* fix(nc-gui): ssl related changes

* fix: oss permission

* fix(nc-gui): active tab issue on clicking source

* feat: include source count and sources in api response

* fix(nc-gui): add getIntegration fun in useIntegrationStore

* fix(nc-gui): source list udpate issue on updating source details

* fix(nc-gui): fix external source icon alignment

* feat: include base name and source count

* fix: query correction

* fix(nc-gui): show liked sources list in delete connection modal

* fix(nc-gui): display connection usage information in list

* fix(nc-gui): add sync data types icons

* fix(nc-gui): add pagination support in connection list

* fix(nc-gui): connection pagination issue

* fix(nc-gui): connection tab count update issue

* test(nc-gui): some of test cases updated

* fix(nc-gui): some minor review changes

* fix(nc-gui): minor ui changes

* fix(nc-gui): Cannot read properties of undefined (reading 'sub_type')

* fix(nc-gui): udpate all tables btn text

* fix(nc-gui): ui changes

* fix(nc-gui): overflow issue

* fix(nc-gui): add connection icon & back btn in modal

* fix(nc-gui): some minor ui changes

* test(nc-gui): update source restriction test cases

* chore(test): remove only from test

* fix(nc-gui): update style of delete connection modal

* test(nc-gui): update acl pw test cases

* fix(test): ws collaboration role accss test fail issue

* fix(nc-gui): add connection successfully added modal

* fix(nc-gui): update connection added modal

* fix(nc-gui): trigger sync request event on upvote

* chore(nc-gui): lint

* fix(nc-gui): add learn more btn in connection successfull modal

* fix(nc-gui): add integration docs link support

* fix(nc-gui): integration table name field text truncate issue

* fix: misc corrections

* misc: button width change

* fix(nc-gui): update icons

* fix(nc-gui): update test connection btn icons

* fix(nc-gui): all tables btn gap issue

* feat(nc-gui): search option in sync data modal

* feat(nc-gui): search connection through api

* fix(nc-gui): add base and source icon in delete connection modal

* fix: update sync request event

* fix(nc-gui): rebase conflict issue

* fix: connections text length

* fix(nc-gui): enable integration/create source supported docs option

* fix(nc-gui): update advanced option header style

---------

Co-authored-by: mertmit <mertmit99@gmail.com>
Co-authored-by: Ramesh Mane <101566080+rameshmane7218@users.noreply.github.com>
Co-authored-by: Raju Udava <86527202+dstala@users.noreply.github.com>

tests/playwright/tests/db/features/erd.spec.ts

@@ -45,10 +45,12 @@ test.describe('Erd', () => {
   };
 
   const openProjectErd = async () => {
-    await dashboard.treeView.baseSettings({ title: context.base.title });
-    await dashboard.settings.selectTab({ tab: 'dataSources' });
-    await dashboard.settings.dataSources.openErd({ rowIndex: 0 });
-    // await dashboard.baseView.dataSources.openERD({ rowIndex: 0 });
+    await dashboard.baseView.tab_dataSources.click();
+    await dashboard.baseView.dataSources.openERD({ rowIndex: 0 });
+
+    // await dashboard.treeView.baseSettings({ title: context.base.title });
+    // await dashboard.settings.selectTab({ tab: 'dataSources' });
+    // await dashboard.settings.dataSources.openErd({ rowIndex: 0 });
   };
 
   const openErdOfATable = async (tableName: string) => {

tests/playwright/tests/db/features/metaSync.spec.ts

@@ -36,12 +36,12 @@ test.describe('Meta sync', () => {
   test('Meta sync', async () => {
     test.setTimeout(process.env.CI ? 100000 : 70000);
 
-    // await dashboard.baseView.tab_dataSources.click();
-    // await dashboard.baseView.dataSources.openMetaSync({ rowIndex: 0 });
+    await dashboard.baseView.tab_dataSources.click();
+    await dashboard.baseView.dataSources.openMetaSync({ rowIndex: 0 });
 
-    await dashboard.treeView.baseSettings({ title: context.base.title });
-    await dashboard.settings.selectTab({ tab: 'dataSources' });
-    await dashboard.settings.dataSources.openMetaSync({ rowIndex: 0 });
+    // await dashboard.treeView.baseSettings({ title: context.base.title });
+    // await dashboard.settings.selectTab({ tab: 'dataSources' });
+    // await dashboard.settings.dataSources.openMetaSync({ rowIndex: 0 });
 
     await dbExec(`CREATE TABLE table1 (id INT NOT NULL, col1 INT NULL, PRIMARY KEY (id))`);
     await dbExec(`CREATE TABLE table2 (id INT NOT NULL, col1 INT NULL, PRIMARY KEY (id))`);
@@ -251,9 +251,12 @@ test.describe('Meta sync', () => {
       `INSERT INTO table1 (id, col1, col2, col3, col4) VALUES (1,1,1,1,1), (2,2,2,2,2), (3,3,3,3,3), (4,4,4,4,4), (5,5,5,5,5), (6,6,6,6,6), (7,7,7,7,7), (8,8,8,8,8), (9,9,9,9,9);`
     );
 
-    await dashboard.treeView.baseSettings({ title: context.base.title });
-    await dashboard.settings.selectTab({ tab: 'dataSources' });
-    await dashboard.settings.dataSources.openMetaSync({ rowIndex: 0 });
+    await dashboard.baseView.tab_dataSources.click();
+    await dashboard.baseView.dataSources.openMetaSync({ rowIndex: 0 });
+
+    // await dashboard.treeView.baseSettings({ title: context.base.title });
+    // await dashboard.settings.selectTab({ tab: 'dataSources' });
+    // await dashboard.settings.dataSources.openMetaSync({ rowIndex: 0 });
 
     await metaData.clickReload();
     await metaData.sync();

tests/playwright/tests/db/general/sourceRestrictions.spec.ts

@@ -2,11 +2,11 @@ import { expect, test } from '@playwright/test';
 import { DashboardPage } from '../../../pages/Dashboard';
 import setup, { NcContext, unsetup } from '../../../setup';
 import { Api } from 'nocodb-sdk';
-import { SettingsPage } from '../../../pages/Dashboard/Settings';
+import { DataSourcePage } from '../../../pages/Dashboard/ProjectView/DataSourcePage';
 
 test.describe('Source Restrictions', () => {
   let dashboard: DashboardPage;
-  let settingsPage: SettingsPage;
+  let dataSourcesPage: DataSourcePage;
   let context: NcContext;
   let api: Api<any>;
   test.setTimeout(150000);
@@ -15,7 +15,7 @@ test.describe('Source Restrictions', () => {
     page.setDefaultTimeout(70000);
     context = await setup({ page });
     dashboard = new DashboardPage(page, context.base);
-    settingsPage = new SettingsPage(dashboard);
+    dataSourcesPage = dashboard.baseView.dataSources;
     api = new Api({
       baseURL: `http://localhost:8080/`,
       headers: {
@@ -29,14 +29,13 @@ test.describe('Source Restrictions', () => {
   });
 
   test('Readonly data source', async () => {
-    await dashboard.treeView.openProjectSourceSettings({ title: context.base.title, context });
+    await dashboard.treeView.openProject({ title: context.base.title, context });
+    await dashboard.baseView.tab_dataSources.click();
 
-    await settingsPage.selectTab({ tab: 'dataSources' });
     await dashboard.rootPage.waitForTimeout(300);
 
-    await settingsPage.source.updateSchemaReadOnly({ sourceName: 'Default', readOnly: true });
-    await settingsPage.source.updateDataReadOnly({ sourceName: 'Default', readOnly: true });
-    await settingsPage.close();
+    await dataSourcesPage.source.updateSchemaReadOnly({ sourceName: 'Default', readOnly: true });
+    await dataSourcesPage.source.updateDataReadOnly({ sourceName: 'Default', readOnly: true });
 
     // reload page to reflect source changes
     await dashboard.rootPage.reload();
@@ -56,14 +55,12 @@ test.describe('Source Restrictions', () => {
   });
 
   test('Readonly schema source', async () => {
-    await dashboard.treeView.openProjectSourceSettings({ title: context.base.title, context });
+    await dashboard.treeView.openProject({ title: context.base.title, context });
+    await dashboard.baseView.tab_dataSources.click();
 
-    await settingsPage.selectTab({ tab: 'dataSources' });
     await dashboard.rootPage.waitForTimeout(300);
 
-    await settingsPage.source.updateSchemaReadOnly({ sourceName: 'Default', readOnly: true });
-    await settingsPage.close();
-
+    await dataSourcesPage.source.updateSchemaReadOnly({ sourceName: 'Default', readOnly: true });
     // reload page to reflect source changes
     await dashboard.rootPage.reload();
 
@@ -98,14 +95,12 @@ test.describe('Source Restrictions', () => {
       type: 'Rating',
     });
 
-    await dashboard.treeView.openProjectSourceSettings({ title: context.base.title, context });
+    await dashboard.treeView.openProject({ title: context.base.title, context });
+    await dashboard.baseView.tab_dataSources.click();
 
-    await settingsPage.selectTab({ tab: 'dataSources' });
     await dashboard.rootPage.waitForTimeout(300);
 
-    await settingsPage.source.updateSchemaReadOnly({ sourceName: 'Default', readOnly: true });
-    await settingsPage.close();
-
+    await dataSourcesPage.source.updateSchemaReadOnly({ sourceName: 'Default', readOnly: true });
     // reload page to reflect source changes
     await dashboard.rootPage.reload();
 

tests/playwright/tests/db/general/tableOperations.spec.ts

@@ -33,9 +33,12 @@ test.describe('Table Operations', () => {
 
       await dashboard.treeView.openProject({ title: context.base.title, context });
 
-      await dashboard.treeView.baseSettings({ title: context.base.title });
-      await dashboard.settings.selectTab({ tab: 'dataSources' });
-      await dashboard.settings.dataSources.openAudit({ rowIndex: 0 });
+      await dashboard.baseView.tab_dataSources.click();
+      await dashboard.baseView.dataSources.openAudit({ rowIndex: 0 });
+
+      // await dashboard.treeView.baseSettings({ title: context.base.title });
+      // await dashboard.settings.selectTab({ tab: 'dataSources' });
+      // await dashboard.settings.dataSources.openAudit({ rowIndex: 0 });
 
       await audit.verifyRow({
         index: 0,

tests/playwright/tests/db/usersAccounts/rolesPreview.spec.ts

@@ -2,12 +2,12 @@ import { test } from '@playwright/test';
 import { DashboardPage } from '../../../pages/Dashboard';
 import setup, { unsetup } from '../../../setup';
 import { ToolbarPage } from '../../../pages/Dashboard/common/Toolbar';
-import { SettingsPage, SettingTab } from '../../../pages/Dashboard/Settings';
 import { Api } from 'nocodb-sdk';
 import { AccountUsersPage } from '../../../pages/Account/Users';
 import { AccountPage } from '../../../pages/Account';
 import { LoginPage } from '../../../pages/LoginPage';
 import { isEE } from '../../../setup/db';
+import { DataSourcePage } from '../../../pages/Dashboard/ProjectView/DataSourcePage';
 let api: Api<any>;
 
 const roles = ['Editor', 'Commenter', 'Viewer'];
@@ -17,7 +17,7 @@ test.describe('Preview Mode', () => {
 
   let dashboard: DashboardPage;
   let toolbar: ToolbarPage;
-  let settings: SettingsPage;
+  let dataSources: DataSourcePage;
   let accountPage: AccountPage;
   let accountUsersPage: AccountUsersPage;
   let loginPage: LoginPage;
@@ -29,7 +29,7 @@ test.describe('Preview Mode', () => {
     context = await setup({ page, isEmptyProject: false });
     dashboard = new DashboardPage(page, context.base);
     toolbar = dashboard.grid.toolbar;
-    settings = dashboard.settings;
+    dataSources = dashboard.baseView.dataSources;
     accountPage = new AccountPage(page);
     accountUsersPage = new AccountUsersPage(accountPage);
     loginPage = new LoginPage(accountPage.rootPage);
@@ -88,34 +88,32 @@ test.describe('Preview Mode', () => {
 
     // configure ACL
     // configure access control
-    await dashboard.gotoSettings();
-    await settings.selectTab({
-      tab: SettingTab.DataSources,
-    });
-    await settings.dataSources.openAcl({ dataSourceName: 'Default' });
-    await settings.dataSources.acl.toggle({ table: 'Language', role: 'editor' });
-    await settings.dataSources.acl.toggle({ table: 'Language', role: 'commenter' });
-    await settings.dataSources.acl.toggle({ table: 'Language', role: 'viewer' });
-    await settings.dataSources.acl.toggle({ table: 'CustomerList', role: 'editor' });
-    await settings.dataSources.acl.toggle({ table: 'CustomerList', role: 'commenter' });
-    await settings.dataSources.acl.toggle({ table: 'CustomerList', role: 'viewer' });
-    await settings.dataSources.acl.save();
-    await settings.close();
+    await dashboard.treeView.openProject({ title: context.base.title, context });
+    await dashboard.baseView.tab_dataSources.click();
 
-    await dashboard.gotoSettings();
-    await settings.selectTab({
-      tab: SettingTab.DataSources,
-    });
-    await settings.dataSources.openAcl({ dataSourceName: 'Default' });
+    await dataSources.openAcl({ dataSourceName: 'Default' });
+    await dataSources.acl.toggle({ table: 'Language', role: 'editor' });
+    await dataSources.acl.toggle({ table: 'Language', role: 'commenter' });
+    await dataSources.acl.toggle({ table: 'Language', role: 'viewer' });
+    await dataSources.acl.toggle({ table: 'CustomerList', role: 'editor' });
+    await dataSources.acl.toggle({ table: 'CustomerList', role: 'commenter' });
+    await dataSources.acl.toggle({ table: 'CustomerList', role: 'viewer' });
+    await dataSources.acl.save();
+    await dataSources.closeDsDetailsModal();
 
-    await settings.dataSources.acl.verify({ table: 'Language', role: 'editor', expectedValue: false });
-    await settings.dataSources.acl.verify({ table: 'Language', role: 'commenter', expectedValue: false });
-    await settings.dataSources.acl.verify({ table: 'Language', role: 'viewer', expectedValue: false });
-    await settings.dataSources.acl.verify({ table: 'CustomerList', role: 'editor', expectedValue: false });
-    await settings.dataSources.acl.verify({ table: 'CustomerList', role: 'commenter', expectedValue: false });
-    await settings.dataSources.acl.verify({ table: 'CustomerList', role: 'viewer', expectedValue: false });
+    await dashboard.treeView.openProject({ title: context.base.title, context });
+    await dashboard.baseView.tab_dataSources.click();
 
-    await settings.close();
+    await dataSources.openAcl({ dataSourceName: 'Default' });
+
+    await dataSources.acl.verify({ table: 'Language', role: 'editor', expectedValue: false });
+    await dataSources.acl.verify({ table: 'Language', role: 'commenter', expectedValue: false });
+    await dataSources.acl.verify({ table: 'Language', role: 'viewer', expectedValue: false });
+    await dataSources.acl.verify({ table: 'CustomerList', role: 'editor', expectedValue: false });
+    await dataSources.acl.verify({ table: 'CustomerList', role: 'commenter', expectedValue: false });
+    await dataSources.acl.verify({ table: 'CustomerList', role: 'viewer', expectedValue: false });
+
+    await dataSources.closeDsDetailsModal();
     await dashboard.signOut();
 
     await loginPage.signIn({