feat(testing): Improved sqlite support and added retry logic in tableUpdate in sqlite client due it popping up on high number of writes in sqlite and configured WAL mode for meta db in testing

packages/nocodb/.gitignore

@@ -18,4 +18,6 @@ noco.db*
 /docker/main.js
 test_meta.db
 test_sakila.db
+test_noco.db-shm
+test_noco.db-wal
 .env

packages/nocodb/src/lib/db/sql-client/lib/sqlite/SqliteClient.ts

@@ -1443,6 +1443,107 @@ class SqliteClient extends KnexClient {
     return result;
   }
 
+  private async _tableUpdate(args) {
+    const result = new Result();
+
+    args.table = args.tn;
+    const originalColumns = args.originalColumns;
+    args.connectionConfig = this._connectionConfig;
+    args.sqlClient = this.sqlClient;
+
+    let upQuery = '';
+    let downQuery = '';
+
+    for (let i = 0; i < args.columns.length; ++i) {
+      const oldColumn = lodash.find(originalColumns, {
+        cn: args.columns[i].cno,
+      });
+
+      if (args.columns[i].altered & 4) {
+        // col remove
+        upQuery += await this.alterTableRemoveColumn(
+          args.table,
+          args.columns[i],
+          oldColumn,
+          upQuery
+        );
+        downQuery += this.alterTableAddColumn(
+          args.table,
+          oldColumn,
+          args.columns[i],
+          downQuery
+        );
+      } else if (args.columns[i].altered & 2 || args.columns[i].altered & 8) {
+        // col edit
+        upQuery += this.alterTableChangeColumn(
+          args.table,
+          args.columns[i],
+          oldColumn,
+          upQuery
+        );
+        downQuery += ';';
+        // downQuery += this.alterTableChangeColumn(
+        //   args.table,
+        //   oldColumn,
+        //   args.columns[i],
+        //   downQuery,
+        //             this.sqlClient
+        // );
+      } else if (args.columns[i].altered & 1) {
+        // col addition
+        upQuery += this.alterTableAddColumn(
+          args.table,
+          args.columns[i],
+          oldColumn,
+          upQuery
+        );
+        downQuery += ';';
+        // downQuery += alterTableRemoveColumn(
+        //   args.table,
+        //   args.columns[i],
+        //   oldColumn,
+        //   downQuery,
+        //             this.sqlClient
+        // );
+      }
+    }
+
+    upQuery += this.alterTablePK(
+      args.columns,
+      args.originalColumns,
+      upQuery,
+      this.sqlClient
+    );
+    //downQuery += alterTablePK(args.originalColumns, args.columns, downQuery);
+
+    if (upQuery) {
+      //upQuery = `ALTER TABLE ${args.columns[0].tn} ${upQuery};`;
+      //downQuery = `ALTER TABLE ${args.columns[0].tn} ${downQuery};`;
+    }
+
+    await Promise.all(
+      upQuery.split(';').map(async (query) => {
+        if (query.trim().length) await this.sqlClient.raw(query);
+      })
+    );
+
+    // await this.sqlClient.raw(upQuery);
+
+    console.log(upQuery);
+
+    const afterUpdate = await this.afterTableUpdate(args);
+
+    result.data.object = {
+      upStatement: [
+        { sql: this.querySeparator() + upQuery },
+        ...afterUpdate.upStatement,
+      ],
+      downStatement: [{ sql: ';' }],
+    };
+
+    return result;
+  }
+
   /**
    *
    * @param {Object} - args
@@ -1472,109 +1573,24 @@ class SqliteClient extends KnexClient {
    */
   async tableUpdate(args) {
     const _func = this.tableUpdate.name;
-    const result = new Result();
     log.api(`${_func}:args:`, args);
 
+    for (let retry = 0; retry < 3; retry++) {
+      try {
+        return await this._tableUpdate(args);
+      } catch (e) {
+        console.log('retrying:tableUpdate', e);
+      }
+      // Wait for 300ms
+      await new Promise((resolve) => setTimeout(resolve, 300));
+    }
+
     try {
-      args.table = args.tn;
-      const originalColumns = args.originalColumns;
-      args.connectionConfig = this._connectionConfig;
-      args.sqlClient = this.sqlClient;
-
-      let upQuery = '';
-      let downQuery = '';
-
-      for (let i = 0; i < args.columns.length; ++i) {
-        const oldColumn = lodash.find(originalColumns, {
-          cn: args.columns[i].cno,
-        });
-
-        if (args.columns[i].altered & 4) {
-          // col remove
-          upQuery += await this.alterTableRemoveColumn(
-            args.table,
-            args.columns[i],
-            oldColumn,
-            upQuery
-          );
-          downQuery += this.alterTableAddColumn(
-            args.table,
-            oldColumn,
-            args.columns[i],
-            downQuery
-          );
-        } else if (args.columns[i].altered & 2 || args.columns[i].altered & 8) {
-          // col edit
-          upQuery += this.alterTableChangeColumn(
-            args.table,
-            args.columns[i],
-            oldColumn,
-            upQuery
-          );
-          downQuery += ';';
-          // downQuery += this.alterTableChangeColumn(
-          //   args.table,
-          //   oldColumn,
-          //   args.columns[i],
-          //   downQuery,
-          //             this.sqlClient
-          // );
-        } else if (args.columns[i].altered & 1) {
-          // col addition
-          upQuery += this.alterTableAddColumn(
-            args.table,
-            args.columns[i],
-            oldColumn,
-            upQuery
-          );
-          downQuery += ';';
-          // downQuery += alterTableRemoveColumn(
-          //   args.table,
-          //   args.columns[i],
-          //   oldColumn,
-          //   downQuery,
-          //             this.sqlClient
-          // );
-        }
-      }
-
-      upQuery += this.alterTablePK(
-        args.columns,
-        args.originalColumns,
-        upQuery,
-        this.sqlClient
-      );
-      //downQuery += alterTablePK(args.originalColumns, args.columns, downQuery);
-
-      if (upQuery) {
-        //upQuery = `ALTER TABLE ${args.columns[0].tn} ${upQuery};`;
-        //downQuery = `ALTER TABLE ${args.columns[0].tn} ${downQuery};`;
-      }
-
-      await Promise.all(
-        upQuery.split(';').map(async (query) => {
-          if (query.trim().length) await this.sqlClient.raw(query);
-        })
-      );
-
-      // await this.sqlClient.raw(upQuery);
-
-      console.log(upQuery);
-
-      const afterUpdate = await this.afterTableUpdate(args);
-
-      result.data.object = {
-        upStatement: [
-          { sql: this.querySeparator() + upQuery },
-          ...afterUpdate.upStatement,
-        ],
-        downStatement: [{ sql: ';' }],
-      };
+      return await this._tableUpdate(args);
     } catch (e) {
       log.ppe(e, _func);
       throw e;
     }
-    return result;
   }
 
   /**

packages/nocodb/src/lib/services/test/TestResetService/resetMetaSakilaSqliteProject.ts

@@ -16,20 +16,16 @@ const sqliteSakilaSqlViews = [
 ];
 
 const dropTablesAndViews = async (metaKnex: Knex, prefix: string) => {
-  for (const view of sqliteSakilaSqlViews) {
-    try {
+  try {
+    for (const view of sqliteSakilaSqlViews) {
       await metaKnex.raw(`DROP VIEW IF EXISTS ${prefix}${view}`);
-    } catch (e) {
-      console.log('Error dropping sqlite view', e);
     }
-  }
 
-  for (const table of sakilaTableNames) {
-    try {
+    for (const table of sakilaTableNames) {
       await metaKnex.raw(`DROP TABLE IF EXISTS ${prefix}${table}`);
-    } catch (e) {
-      console.log('Error dropping sqlite table', e);
     }
+  } catch (e) {
+    console.error('Error dropping tables and views', e);
   }
 };
 
@@ -58,31 +54,34 @@ const resetMetaSakilaSqlite = async (metaKnex: Knex, prefix: string) => {
     '/tests'
   );
 
-  const trx = await metaKnex.transaction();
-
   try {
     const schemaFile = await fs.readFile(
       `${testsDir}/sqlite-sakila-db/03-sqlite-prefix-sakila-schema.sql`
     );
     const schemaFileStr = schemaFile.toString().replace(/prefix___/g, prefix);
 
-    const dataFile = await fs.readFile(
-      `${testsDir}/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql`
-    );
-    const dataFileStr = dataFile.toString().replace(/prefix___/g, prefix);
-
     const schemaSqlQueries = schemaFileStr.split(';');
     for (const sqlQuery of schemaSqlQueries) {
       if (sqlQuery.trim().length > 0) {
-        await trx.raw(
+        await metaKnex.raw(
           sqlQuery
             .trim()
             .replace(/WHERE rowid = new.rowid/g, 'WHERE rowid = new.rowid;')
         );
       }
     }
+  } catch (e) {
+    console.error('Error resetting meta sakila sqlite:db', e);
+  }
 
+  const trx = await metaKnex.transaction();
+  try {
+    const dataFile = await fs.readFile(
+      `${testsDir}/sqlite-sakila-db/04-sqlite-prefix-sakila-insert-data.sql`
+    );
+    const dataFileStr = dataFile.toString().replace(/prefix___/g, prefix);
     const dataSqlQueries = dataFileStr.split(';');
+
     for (const sqlQuery of dataSqlQueries) {
       if (sqlQuery.trim().length > 0) {
         await trx.raw(sqlQuery.trim());

packages/nocodb/src/lib/utils/NcConfigFactory.ts

@@ -590,6 +590,10 @@ export default class NcConfigFactory implements NcConfig {
         ...args.meta.db,
         connection: args.meta.db,
       });
+      if (process.env['TEST'] === 'true') {
+        await metaSqlClient.raw('PRAGMA journal_mode=WAL');
+        await metaSqlClient.raw('PRAGMA busy_timeout=60000');
+      }
       await metaSqlClient.createDatabaseIfNotExists({
         database: args.meta.db?.connection?.filename,
       });

scripts/playwright/playwright.config.ts

@@ -26,9 +26,9 @@ const config: PlaywrightTestConfig = {
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
-  retries: process.env.CI ? 2 : 0,
+  retries: 2,
   /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : 2,
+  workers: process.env.CI ? 2 : 4,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */