feat(cypress): Made cypress tests independent for pg and integrated some tests

.gitignore

@@ -93,3 +93,4 @@ shared.json
 # NC_DBs
 #=========
 nc_minimal_dbs/
+test_noco.db

packages/nocodb/src/lib/meta/api/index.ts

@@ -28,6 +28,7 @@ import metaDiffApis from './metaDiffApis';
 import cacheApis from './cacheApis';
 import apiTokenApis from './apiTokenApis';
 import hookFilterApis from './hookFilterApis';
+import testApis from './testApis';
 import {
   bulkDataAliasApis,
   dataAliasApis,
@@ -57,6 +58,9 @@ export default function (router: Router, server) {
   projectApis(router);
   utilApis(router);
 
+  if(process.env['TEST'] === 'true') {
+    router.use(testApis);
+  }
   router.use(columnApis);
   router.use(exportApis);
   router.use(dataApis);

packages/nocodb/src/lib/meta/api/testApis.ts

@@ -0,0 +1,13 @@
+import { Request, Router } from 'express';
+import { TestResetService } from '../../services/test/TestResetService';
+
+export async function reset(_: Request<any, any>, res) {
+  const service = new TestResetService();
+
+  res.json(await service.process());
+}
+
+const router = Router({ mergeParams: true });
+
+router.get('/api/v1/meta/test/reset', reset);
+export default router;

packages/nocodb/src/lib/services/test/TestResetService/createProjects.ts

@@ -0,0 +1,65 @@
+import axios from 'axios';
+
+const extPgProject = {
+  title: 'pgExtREST',
+  bases: [
+    {
+      type: 'pg',
+      config: {
+        client: 'pg',
+        connection: {
+          host: 'localhost',
+          port: '5432',
+          user: 'postgres',
+          password: 'password',
+          database: 'postgres',
+        },
+        searchPath: ['public'],
+      },
+      inflection_column: 'camelize',
+      inflection_table: 'camelize',
+    },
+  ],
+  external: true,
+};
+
+// const extMysqlProject = {
+//   title: 'externalREST',
+//   bases: [
+//     {
+//       type: 'mysql2',
+//       config: {
+//         client: 'mysql2',
+//         connection: {
+//           host: 'localhost',
+//           port: '5432',
+//           user: 'root',
+//           password: 'password',
+//           database: 'sakila',
+//         },
+//       },
+//       inflection_column: 'camelize',
+//       inflection_table: 'camelize',
+//     },
+//   ],
+//   external: true,
+// };
+
+const createProjects = async (token) => {
+  return await Promise.all(
+    [extPgProject].map(async (projectAttr) => {
+      const response = await axios.post(
+        'http://localhost:8080/api/v1/db/meta/projects/',
+        projectAttr,
+        {
+          headers: {
+            'xc-auth': token,
+          },
+        }
+      );
+      return response.data;
+    })
+  );
+};
+
+export default createProjects;

packages/nocodb/src/lib/services/test/TestResetService/createUser.ts

@@ -0,0 +1,17 @@
+import axios from 'axios';
+
+const defaultUserArgs = {
+  email: 'user@nocodb.com',
+  password: 'Password123.',
+};
+
+const createUser = async () => {
+  const response = await axios.post(
+    'http://localhost:8080/api/v1/auth/user/signup',
+    defaultUserArgs
+  );
+
+  return { token: response.data.token };
+};
+
+export default createUser;

packages/nocodb/src/lib/services/test/TestResetService/index.ts

@@ -0,0 +1,39 @@
+import Noco from '../../../Noco';
+
+import Knex from 'knex';
+import NocoCache from '../../../cache/NocoCache';
+import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
+import createProjects from './createProjects';
+import { isPgSakilaToBeReset, resetPgSakila } from './resetPgSakila';
+import createUser from './createUser';
+import resetMeta from './resetMeta';
+
+export class TestResetService {
+  private knex: Knex | null = null;
+
+  constructor() {
+    this.knex = Noco.ncMeta.knex;
+  }
+
+  async process() {
+    try {
+      await NcConnectionMgrv2.destroyAll();
+
+      if (await isPgSakilaToBeReset()) {
+        await resetPgSakila();
+      }
+
+      await resetMeta(this.knex);
+
+      await NocoCache.destroy();
+
+      const { token } = await createUser();
+      const projects = await createProjects(token);
+
+      return { token, projects };
+    } catch (e) {
+      console.error('cleanupMeta', e);
+      return { error: e };
+    }
+  }
+}

packages/nocodb/src/lib/services/test/TestResetService/resetMeta.ts

@@ -0,0 +1,59 @@
+import Model from '../../../models/Model';
+import Project from '../../../models/Project';
+import { orderedMetaTables } from '../../../utils/globals';
+
+const disableForeignKeyChecks = async (knex) => {
+  await knex.raw('PRAGMA foreign_keys = OFF');
+  // await this.knex.raw(`SET FOREIGN_KEY_CHECKS = 0`);
+};
+
+const enableForeignKeyChecks = async (knex) => {
+  await knex.raw(`PRAGMA foreign_keys = ON;`);
+  // await this.knex.raw(`SET FOREIGN_KEY_CHECKS = 1`);
+};
+
+const dropTablesAllNonExternalProjects = async (knex) => {
+  const projects = await Project.list({});
+  const userCreatedTableNames: string[] = [];
+  await Promise.all(
+    projects
+      .filter((project) => project.is_meta)
+      .map(async (project) => {
+        await project.getBases();
+        const base = project.bases && project.bases[0];
+        if (!base) return;
+
+        const models = await Model.list({
+          project_id: project.id,
+          base_id: base.id!,
+        });
+        models.forEach((model) => {
+          userCreatedTableNames.push(model.table_name);
+        });
+      })
+  );
+
+  await disableForeignKeyChecks(knex);
+
+  for (const tableName of userCreatedTableNames) {
+    await knex.raw(`DROP TABLE ${tableName}`);
+  }
+
+  await enableForeignKeyChecks(knex);
+};
+
+const resetMeta = async (knex) => {
+  await dropTablesAllNonExternalProjects(knex);
+
+  await disableForeignKeyChecks(knex);
+  for (const tableName of orderedMetaTables) {
+    try {
+      await knex.raw(`DELETE FROM ${tableName}`);
+    } catch (e) {
+      console.error('cleanupMetaTables', e);
+    }
+  }
+  await enableForeignKeyChecks(knex);
+};
+
+export default resetMeta;

packages/nocodb/src/lib/services/test/TestResetService/resetPgSakila.ts

@@ -0,0 +1,56 @@
+import knex from 'knex';
+import fs from 'fs';
+import Project from '../../../models/Project';
+import Audit from '../../../models/Audit';
+
+const config = {
+  client: 'pg',
+  connection: {
+    host: 'localhost',
+    port: 5432,
+    user: 'postgres',
+    password: 'password',
+    database: 'postgres',
+  },
+  searchPath: ['public'],
+  meta: { dbtype: '' },
+  pool: { min: 0, max: 5 },
+};
+
+const isPgSakilaToBeReset = async () => {
+  const sakilaProject = await Project.getByTitle('pgExtREST');
+
+  const audits =
+    sakilaProject && (await Audit.projectAuditList(sakilaProject.id, {}));
+
+  return audits.length > 0;
+};
+
+const resetPgSakila = async () => {
+  const knexClient = knex(config);
+
+  try {
+    await knexClient.raw(`DROP SCHEMA public CASCADE`);
+  } catch (e) {
+    console.log('Error dropping pg schema', e);
+  }
+  await knexClient.raw(`CREATE SCHEMA public`);
+
+  const testsDir = __dirname.replace(
+    '/src/lib/services/test/TestResetService',
+    '/tests'
+  );
+
+  const schemaFile = fs
+    .readFileSync(`${testsDir}/pg-sakila-db/01-postgres-sakila-schema.sql`)
+    .toString();
+  const dataFile = fs
+    .readFileSync(`${testsDir}/pg-sakila-db/02-postgres-sakila-insert-data.sql`)
+    .toString();
+  await knexClient.raw(schemaFile);
+  await knexClient.raw(dataFile);
+
+  await knexClient.destroy();
+};
+
+export { resetPgSakila, isPgSakilaToBeReset };

packages/nocodb/src/lib/utils/NcConfigFactory.ts

@@ -626,7 +626,7 @@ export default class NcConfigFactory implements NcConfig {
     db: {
       client: 'sqlite3',
       connection: {
-        filename: 'noco.db',
+        filename: process.env['TEST'] !== 'true' ? 'noco.db': 'test_noco.db',
       },
     },
   };

scripts/cypress/cypress.json

@@ -14,7 +14,8 @@
         "test/pg-restRoles.js",
         "test/pg-restMisc.js",
         "test/quickTest.js",
-        "test/db-independent.js"
+        "test/db-independent.js",
+        "test/pg-restMiscV2.js"
     ],
     "defaultCommandTimeout": 13000,
     "pageLoadTimeout": 600000,

scripts/cypress/integration/common/6b_downloadCsv.js

@@ -9,10 +9,9 @@ export const genTest = (apiType, dbType) => {
   if (!isTestSuiteActive(apiType, dbType)) return;
 
   describe(`${apiType.toUpperCase()} Upload/ Download CSV`, () => {
-    // before(() => {
-    //     // standalone test
-    //     // loginPage.loginAndOpenProject(apiType, dbType);
-    // });
+    before(() => {
+      cy.setup({ dbType })
+    });
 
     beforeEach(() => {
       cy.restoreLocalStorage();

scripts/cypress/integration/common/6f_attachments.js

@@ -6,9 +6,9 @@ export const genTest = (apiType, dbType) => {
   if (!isTestSuiteActive(apiType, dbType)) return;
 
   describe(`${apiType.toUpperCase()} Columns of type attachment`, () => {
-    // before(() => {
-    //   // loginPage.loginAndOpenProject(apiType, dbType);
-    // });
+    before(() => {
+      cy.setup({ dbType })
+    });
 
     beforeEach(() => {
       cy.restoreLocalStorage();

scripts/cypress/integration/common/9b_ERD.js

@@ -15,22 +15,23 @@ export const genTest = (apiType, dbType) => {
     before(() => {
       cy.restoreLocalStorage();
       
-      // loginPage.loginAndOpenProject(apiType, dbType);
+      cy.setup({ dbType }).then(({project}) => {
+        cy.openTableTab("Country", 25);
+
+        projectId = project.id
+        if (dbType === "postgres") {
+          sakilaTables = pgSakilaTables;
+          sakilaSqlViews = pgSakilaSqlViews;
+        } else if(dbType === "mysql") {
+          sakilaTables = mysqlSakilaTables;
+          sakilaSqlViews = mysqlSakilaSqlViews;
+        } else if(dbType === "xcdb") {
+          sakilaTables = mysqlSakilaTables.map((tableName) => `${projectId}${tableName}`);
+          sakilaSqlViews = sqliteSakilaSqlViews.map((viewName) => `${projectId}${viewName}`);
+        }
+        cy.saveLocalStorage();
+      })
       
-      cy.openTableTab("Country", 25);
-      projectId = getProjectString()
-      cy.log('erd:getProjectString' + projectId)
-      if (dbType === "postgres") {
-        sakilaTables = pgSakilaTables;
-        sakilaSqlViews = pgSakilaSqlViews;
-      } else if(dbType === "mysql") {
-        sakilaTables = mysqlSakilaTables;
-        sakilaSqlViews = mysqlSakilaSqlViews;
-      } else if(dbType === "xcdb") {
-        sakilaTables = mysqlSakilaTables.map((tableName) => `${projectId}${tableName}`);
-        sakilaSqlViews = sqliteSakilaSqlViews.map((viewName) => `${projectId}${viewName}`);
-      }
-      cy.saveLocalStorage();
     });
 
     beforeEach(() => {
@@ -41,16 +42,10 @@ export const genTest = (apiType, dbType) => {
       cy.saveLocalStorage();
     })
 
-    after(() => {
-      cy.restoreLocalStorage();
-      cy.closeTableTab("Country");
-      cy.saveLocalStorage();
-    });
-
     // Test cases
 
     it(`Enable MM setting Open Table ERD`, () => {
-      // cy.openTableTab("Country", 25);
+      cy.openTableTab("Country", 25);
       mainPage.toggleShowMMSetting();
       
       mainPage.openErdTab();

scripts/cypress/integration/test/pg-restMiscV2.js

@@ -0,0 +1,51 @@
+// let t0 = require("./explicitLogin");
+// let t01 = require("../common/00_pre_configurations");
+let t6b = require("../common/6b_downloadCsv");
+// let t6c = require("../common/6c_swagger_api");
+// let t6d = require("../common/6d_language_validation");
+// let t6e = require("../common/6e_project_operations");
+let t6f = require("../common/6f_attachments");
+// let t6g = require("../common/6g_base_share");
+// let t7a = require("../common/7a_create_project_from_excel");
+const {
+  setCurrentMode,
+} = require("../../support/page_objects/projectConstants");
+// const t8a = require("../common/8a_webhook");
+const t9b = require("../common/9b_ERD");
+
+const nocoTestSuite = (apiType, dbType) => {
+  setCurrentMode(apiType, dbType);
+
+  // Download CSV
+  t6b.genTest(apiType, dbType);
+
+  // Attachment cell
+  t6f.genTest(apiType, dbType);
+
+  // ERD:
+  t9b.genTest(apiType, dbType);
+};
+
+nocoTestSuite("rest", "postgres");
+
+/**
+ * @copyright Copyright (c) 2021, Xgene Cloud Ltd
+ *
+ * @author Raju Udava <sivadstala@gmail.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */

scripts/cypress/support/commands.js

@@ -29,6 +29,40 @@ import { isXcdb, isPostgres } from "./page_objects/projectConstants";
 
 require("@4tw/cypress-drag-drop");
 
+let LOCAL_STORAGE_MEMORY = {};
+let LOCAL_STORAGE_MEMORY_v2 = {};
+
+Cypress.Commands.add('setup', ({ dbType }) => {
+  cy.request('GET', 'http://localhost:8080/api/v1/meta/test/reset').then((response) => {
+    LOCAL_STORAGE_MEMORY = {
+      "nocodb-gui-v2": JSON.stringify({
+        "token": response.body.token,
+        darkMode: false,
+      })
+    }
+    cy.restoreLocalStorage().then(() => {
+      console.log('setup done', localStorage.getItem('nocodb-gui-v2'))
+      let project;
+
+      if(dbType === "postgres") {
+        const pgProject = response.body.projects.find((project) => project.title === 'pgExtREST');
+        project = pgProject;
+      }
+
+      cy.visit(`http://localhost:3000/#/nc/${project.id}/auth`, {
+        retryOnNetworkFailure: true,
+        timeout: 1200000,
+        headers: {
+          "Accept-Encoding": "gzip, deflate",
+        }
+      }).then(() => {
+        return {token: response.body.token, project}
+      });
+
+    })
+  })
+});
+
 // recursively gets an element, returning only after it's determined to be attached to the DOM for good
 Cypress.Commands.add("getSettled", (selector, opts = {}) => {
   const retries = opts.retries || 3;
@@ -243,9 +277,6 @@ Cypress.Commands.add("openOrCreateGqlProject", (_args) => {
   cy.url({ timeout: 20000 }).should("contain", "#/nc/");
 });
 
-let LOCAL_STORAGE_MEMORY = {};
-let LOCAL_STORAGE_MEMORY_v2 = {};
-
 Cypress.Commands.add("saveLocalStorage", (name) => {
   LOCAL_STORAGE_MEMORY = {};
   Object.keys(localStorage).forEach((key) => {