diff --git a/.run/Run NocoDB Sqlite.run.xml b/.run/Run NocoDB Sqlite.run.xml new file mode 100644 index 0000000000..58f55251c6 --- /dev/null +++ b/.run/Run NocoDB Sqlite.run.xml @@ -0,0 +1,15 @@ + + + + + + @@ -20,5 +19,40 @@ export default ` + `; diff --git a/packages/nocodb/src/lib/noco/meta/api/swagger/swaggerHtml.ts b/packages/nocodb/src/lib/noco/meta/api/swagger/swaggerHtml.ts index 9d50dc0258..08c876afa9 100644 --- a/packages/nocodb/src/lib/noco/meta/api/swagger/swaggerHtml.ts +++ b/packages/nocodb/src/lib/noco/meta/api/swagger/swaggerHtml.ts @@ -20,6 +20,39 @@ export default ` SwaggerUIBundle.SwaggerUIStandalonePreset ], }) + console.log('%c🚀 We are Hiring!!! 🚀%c\\n%cJoin the forces http://careers.nocodb.com', 'color:#1348ba;font-size:3rem;padding:20px;', 'display:none', 'font-size:1.5rem;padding:20px'); + const linkEl = document.createElement('a') + linkEl.setAttribute('href', "http://careers.nocodb.com") + linkEl.setAttribute('target', '_blank') + linkEl.setAttribute('class', 'we-are-hiring') + linkEl.innerHTML = '🚀 We are Hiring!!! 🚀' + const styleEl = document.createElement('style'); + styleEl.innerHTML = \` +.we-are-hiring { + position: fixed; + bottom: 50px; + right: -250px; + opacity: 0; + background: orange; + border-radius: 4px; + padding: 19px; + z-index: 200; + text-decoration: none; + text-transform: uppercase; + color: black; + transition: 1s opacity, 1s right; + display: block; + font-weight: bold; +} + +.we-are-hiring.active { + opacity: 1; + right:25px; +} + \` + document.body.appendChild(linkEl, document.body.firstChild) + document.body.appendChild(styleEl, document.body.firstChild) + setTimeout(() => linkEl.classList.add('active'), 2000) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index 2ea02d532c..4b8d029d0d 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -12,11 +12,12 @@ import hash from 'object-hash'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; + dayjs.extend(utc); export default async ( syncDB: AirtableSyncConfig, - progress: (msg: string) => void + progress: (data: { msg?: string; level?: any }) => void ) => { const sMap = { mapTbl: {}, @@ -47,6 +48,14 @@ export default async ( } }; + function logBasic(log) { + progress({ level: 0, msg: log }); + } + + function logDetailed(log) { + if (debugMode) progress({ level: 1, msg: log }); + } + let base, baseId; const start = Date.now(); const enableErrorLogs = false; @@ -59,7 +68,12 @@ export default async ( const ncLinkMappingTable: any[] = []; const nestedLookupTbl: any[] = []; const nestedRollupTbl: any[] = []; - const runTimeCounters = { + const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' }; + + const uniqueTableNameGen = getUniqueNameGenerator('sheet'); + + // run time counter (statistics) + const rtc = { sort: 0, filter: 0, view: { @@ -79,16 +93,12 @@ export default async ( }; function updateMigrationSkipLog(tbl, col, type, reason?) { - runTimeCounters.migrationSkipLog.count++; - runTimeCounters.migrationSkipLog.log.push( + rtc.migrationSkipLog.count++; + rtc.migrationSkipLog.log.push( `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}` ); } - let syncLog = _log => { - // console.log(log) - }; - // mapping table // @@ -102,8 +112,8 @@ export default async ( } const ft = await FetchAT.read(); const duration = Date.now() - start; - runTimeCounters.fetchAt.count++; - runTimeCounters.fetchAt.time += duration; + rtc.fetchAt.count++; + rtc.fetchAt.time += duration; const file = ft.schema; baseId = ft.baseId; @@ -120,8 +130,8 @@ export default async ( const start = Date.now(); const ft = await FetchAT.readView(viewId); const duration = Date.now() - start; - runTimeCounters.fetchAt.count++; - runTimeCounters.fetchAt.time += duration; + rtc.fetchAt.count++; + rtc.fetchAt.time += duration; if (debugMode) jsonfile.writeFileSync(`${viewId}.json`, ft, { spaces: 2 }); return ft.view; @@ -132,6 +142,7 @@ export default async ( foreignKey: UITypes.LinkToAnotherRecord, text: UITypes.SingleLineText, multilineText: UITypes.LongText, + richText: UITypes.LongText, multipleAttachment: UITypes.Attachment, checkbox: UITypes.Checkbox, multiSelect: UITypes.MultiSelect, @@ -157,31 +168,24 @@ export default async ( // function nc_sanitizeName(name) { - // knex complains use of '?' in field name - // good to replace all special characters by _ in one go - - // https://stackoverflow.com/questions/18862256/how-to-detect-emoji-using-javascript - const regex = /(?:[\u2700-\u27bf]|(?:\ud83c[\udde6-\uddff]){2}|[\ud800-\udbff][\udc00-\udfff]|[\u0023-\u0039]\ufe0f?\u20e3|\u3299|\u3297|\u303d|\u3030|\u24c2|\ud83c[\udd70-\udd71]|\ud83c[\udd7e-\udd7f]|\ud83c\udd8e|\ud83c[\udd91-\udd9a]|\ud83c[\udde6-\uddff]|\ud83c[\ude01-\ude02]|\ud83c\ude1a|\ud83c\ude2f|\ud83c[\ude32-\ude3a]|\ud83c[\ude50-\ude51]|\u203c|\u2049|[\u25aa-\u25ab]|\u25b6|\u25c0|[\u25fb-\u25fe]|\u00a9|\u00ae|\u2122|\u2139|\ud83c\udc04|[\u2600-\u26FF]|\u2b05|\u2b06|\u2b07|\u2b1b|\u2b1c|\u2b50|\u2b55|\u231a|\u231b|\u2328|\u23cf|[\u23e9-\u23f3]|[\u23f8-\u23fa]|\ud83c\udccf|\u2934|\u2935|[\u2190-\u21ff])/g; - - return name - .replace(/\?/g, '_') - .replace('.', '_') - .replace(regex, '_') - .trim(); + // replace all special characters by _ + return name.replace(/\W+/g, '_').trim(); } function nc_getSanitizedColumnName(table, name) { const col_name = nc_sanitizeName(name); + // for knex, replace . with _ + const col_alias = name.trim().replace(/\./g, '_'); // check if already a column exists with same name? - const duplicateColumn = table.columns.find(x => x.title === name.trim()); + const duplicateColumn = table.columns.find(x => x.title === col_alias); if (duplicateColumn) { - if (enableErrorLogs) console.log(`## Duplicate ${name.trim()}`); + if (enableErrorLogs) console.log(`## Duplicate ${col_alias}`); } return { // kludge: error observed in Nc with space around column-name - title: name.trim() + (duplicateColumn ? '_2' : ''), + title: col_alias + (duplicateColumn ? '_2' : ''), column_name: col_name + (duplicateColumn ? '_2' : '') }; } @@ -273,7 +277,7 @@ export default async ( if (sampleProj) { await api.project.delete(sampleProj.id); } - syncLog('Init'); + logDetailed('Init'); } // map UIDT @@ -304,9 +308,9 @@ export default async ( case 'formula': if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()') - ncType = UITypes.CreateTime; + ncType = UITypes.DateTime; else if (col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()') - ncType = UITypes.LastModifiedTime; + ncType = UITypes.DateTime; break; case 'computation': @@ -346,7 +350,9 @@ export default async ( ); } // const csvOpt = "'" + opt.join("','") + "'"; - const csvOpt = opt.map(v => `'${v.replace(/'/g, "\\'")}'`).join(','); + const csvOpt = opt + .map(v => `'${v.replace(/'/g, "\\'").replace(/,/g, '.')}'`) + .join(','); return { type: 'select', data: csvOpt }; } default: @@ -362,32 +368,34 @@ export default async ( for (let i = 0; i < tblSchema.length; ++i) { const table: any = {}; - syncLog(`Preparing base schema (sans relations): ${tblSchema[i].name}`); - runTimeCounters.view.total += tblSchema[i].views.length; + if (syncDB.syncViews) { + rtc.view.total += tblSchema[i].views.reduce( + (acc, cur) => + ['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc, + 0 + ); + } else { + rtc.view.total = tblSchema.length; + } // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id; table.title = tblSchema[i].name; - table.table_name = nc_sanitizeName(tblSchema[i].name); + table.table_name = uniqueTableNameGen(nc_sanitizeName(tblSchema[i].name)); - // insert _aTbl_nc_rec_id of type ID by default - table.columns = [ + const uniqueColNameGen = getUniqueNameGenerator('field'); + table.columns = []; + const sysColumns = [ { - title: '_aTbl_nc_rec_id', - column_name: '_aTbl_nc_rec_id', + title: ncSysFields.id, + column_name: ncSysFields.id, uidt: UITypes.ID, - // idType: 'AG' - // uidt: UITypes.SingleLineText, - // pk: true, - // // mysql additionally requires NOT-NULL to be explicitly set - // rqd: true, - // system: true, meta: { ag: 'nc' } }, { - title: '_aTbl_nc_rec_hash', - column_name: '_aTbl_nc_rec_hash', + title: ncSysFields.hash, + column_name: ncSysFields.hash, uidt: UITypes.SingleLineText, system: true } @@ -406,7 +414,7 @@ export default async ( const ncCol: any = { // Enable to use aTbl identifiers as is: id: col.id, title: ncName.title, - column_name: ncName.column_name, + column_name: uniqueColNameGen(ncName.column_name), uidt: getNocoType(col) }; @@ -423,9 +431,15 @@ export default async ( } // populate cdf (column default value) if configured - if (col?.default) { - ncCol.cdf = col.default; - } + // if (col?.default) { + // if (typeof col.default === 'string') + // ncCol.cdf = `'${col.default.replace?.(/'/g, "\\'")}'`; + // else ncCol.cdf = col.default; + // } + + // change from default 'tinytext' as airtable allows more than 255 characters + // for single line text column type + if (col.type === 'text') ncCol.dt = 'text'; // additional column parameters when applicable const colOptions = getNocoTypeOptions(col); @@ -440,6 +454,9 @@ export default async ( } table.columns.push(ncCol); } + table.columns.push(sysColumns[0]); + table.columns.push(sysColumns[1]); + tables.push(table); } return tables; @@ -449,17 +466,11 @@ export default async ( // base schema preparation: exclude const tables: any[] = tablesPrepare(aTblSchema); - syncLog(`Total tables: ${tables.length} `); - // for each table schema, create nc table for (let idx = 0; idx < tables.length; idx++) { - syncLog( - `[${idx + 1}/${tables.length}] Creating base table schema: ${ - tables[idx].title - }` - ); + logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`); - syncLog(`NC API: dbTable.create ${tables[idx].title}`); + logDetailed(`NC API: dbTable.create ${tables[idx].title}`); const table: any = await api.dbTable.create( ncCreatedProjectSchema.id, tables[idx] @@ -470,7 +481,7 @@ export default async ( await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title); for (let colIdx = 0; colIdx < table.columns.length; colIdx++) { const aId = aTblSchema[idx].columns.find( - x => x.name.trim() === table.columns[colIdx].title + x => x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title )?.id; if (aId) await sMap.addToMappingTbl( @@ -482,14 +493,11 @@ export default async ( } // update default view name- to match it to airtable view name - syncLog(`NC API: dbView.list ${table.id}`); + logDetailed(`NC API: dbView.list ${table.id}`); const view = await api.dbView.list(table.id); - syncLog( - `NC API: dbView.update ${view.list[0].id} ${aTblSchema[idx].views[0].name}` - ); const aTbl_grid = aTblSchema[idx].views.find(x => x.type === 'grid'); - // @ts-ignore + logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); await api.dbView.update(view.list[0].id, { title: aTbl_grid.name }); @@ -519,7 +527,7 @@ export default async ( // if (aTblLinkColumns.length) { for (let i = 0; i < aTblLinkColumns.length; i++) { - syncLog( + logDetailed( `[${idx + 1}/${aTblSchema.length}] Configuring Links :: [${i + 1}/${ aTblLinkColumns.length }] ${aTblSchema[idx].name}` @@ -531,8 +539,8 @@ export default async ( const dst = aTbl_getColumnName( aTblLinkColumns[i].typeOptions?.symmetricColumnId ); - syncLog( - ` LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst?.tn}:${dst?.cn}` + logDetailed( + `LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst?.tn}:${dst?.cn}` ); } @@ -562,6 +570,10 @@ export default async ( srcTbl, aTblLinkColumns[i].name ); + + logDetailed( + `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` + ); const ncTbl: any = await api.dbTableColumn.create(srcTableId, { uidt: UITypes.LinkToAnotherRecord, title: ncName.title, @@ -574,7 +586,6 @@ export default async ( // : 'hm' }); updateNcTblSchema(ncTbl); - syncLog(`NC API: dbTableColumn.create LinkToAnotherRecord`); const ncId = ncTbl.columns.find(x => x.title === ncName.title)?.id; await sMap.addToMappingTbl( @@ -683,6 +694,10 @@ export default async ( childTblSchema, aTblLinkColumns[i].name ); + + logDetailed( + `NC API: dbTableColumn.update rename symmetric column ${ncName.title}` + ); const ncTbl: any = await api.dbTableColumn.update( childLinkColumn.id, { @@ -704,7 +719,6 @@ export default async ( ); // console.log(res.columns.find(x => x.title === aTblLinkColumns[i].name)) - syncLog(`NC API: dbTableColumn.update rename symmetric column`); } } } @@ -726,7 +740,7 @@ export default async ( if (aTblColumns.length) { // Lookup for (let i = 0; i < aTblColumns.length; i++) { - syncLog( + logDetailed( `[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${i + 1}/${aTblColumns.length}] ${aTblSchema[idx].name}` ); @@ -764,6 +778,8 @@ export default async ( srcTableSchema, aTblColumns[i].name ); + + logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const ncTbl: any = await api.dbTableColumn.create(srcTableId, { uidt: UITypes.Lookup, title: ncName.title, @@ -781,8 +797,6 @@ export default async ( aTblColumns[i].name, ncTbl.id ); - - syncLog(`NC API: dbTableColumn.create LOOKUP`); } } } @@ -813,10 +827,6 @@ export default async ( // Nested lookup nestedCnt = nestedLookupTbl.length; for (let i = 0; i < nestedLookupTbl.length; i++) { - syncLog( - `Configuring Nested Lookup: Level-${level} [${i + 1}/${nestedCnt}]` - ); - const srcTableId = nestedLookupTbl[0].srcTableId; const srcTableSchema = ncSchema.tablesById[srcTableId]; @@ -835,6 +845,14 @@ export default async ( srcTableSchema, nestedLookupTbl[0].name ); + + logDetailed( + `Configuring Nested Lookup: Level-${level} [${i + 1}/${nestedCnt} ${ + ncName.title + }]` + ); + + logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const ncTbl: any = await api.dbTableColumn.create(srcTableId, { uidt: UITypes.Lookup, title: ncName.title, @@ -856,7 +874,6 @@ export default async ( // remove entry nestedLookupTbl.splice(0, 1); - syncLog(`NC API: dbTableColumn.create LOOKUP`); } level++; } @@ -898,7 +915,7 @@ export default async ( if (aTblColumns.length) { // rollup exist for (let i = 0; i < aTblColumns.length; i++) { - syncLog( + logDetailed( `[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${i + 1}/${aTblColumns.length}] ${aTblSchema[idx].name}` ); @@ -955,13 +972,14 @@ export default async ( if ( ncColSchema?.uidt === UITypes.Formula || ncColSchema?.uidt === UITypes.Lookup || - ncColSchema?.uidt === UITypes.Rollup + ncColSchema?.uidt === UITypes.Rollup || + ncColSchema?.uidt === UITypes.Checkbox ) { updateMigrationSkipLog( srcTableSchema.title, aTblColumns[i].name, aTblColumns[i].type, - 'rollup referring to a lookup column' + 'rollup referring to a column type not supported currently' ); continue; } @@ -970,6 +988,8 @@ export default async ( srcTableSchema, aTblColumns[i].name ); + + logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); const ncTbl: any = await api.dbTableColumn.create(srcTableId, { uidt: UITypes.Rollup, title: ncName.title, @@ -979,7 +999,6 @@ export default async ( rollup_function: ncRollupFn }); updateNcTblSchema(ncTbl); - syncLog(`NC API: dbTableColumn.create ROLLUP`); const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name) ?.id; @@ -992,14 +1011,12 @@ export default async ( } } } - syncLog(`Nested rollup: ${nestedRollupTbl.length}`); + logDetailed(`Nested rollup: ${nestedRollupTbl.length}`); } async function nocoLookupForRollup() { const nestedCnt = nestedLookupTbl.length; for (let i = 0; i < nestedLookupTbl.length; i++) { - syncLog(`Configuring Lookup over Rollup :: [${i + 1}/${nestedCnt}]`); - const srcTableId = nestedLookupTbl[0].srcTableId; const srcTableSchema = ncSchema.tablesById[srcTableId]; @@ -1018,6 +1035,14 @@ export default async ( srcTableSchema, nestedLookupTbl[0].name ); + + logDetailed( + `Configuring Lookup over Rollup :: [${i + 1}/${nestedCnt}] ${ + ncName.title + }` + ); + + logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); const ncTbl: any = await api.dbTableColumn.create(srcTableId, { uidt: UITypes.Lookup, title: ncName.title, @@ -1038,13 +1063,12 @@ export default async ( // remove entry nestedLookupTbl.splice(0, 1); - syncLog(`NC API: dbTableColumn.create LOOKUP`); } } async function nocoSetPrimary(aTblSchema) { for (let idx = 0; idx < aTblSchema.length; idx++) { - syncLog( + logDetailed( `[${idx + 1}/${aTblSchema.length}] Configuring Primary value : ${ aTblSchema[idx].name }` @@ -1054,8 +1078,8 @@ export default async ( const ncColId = sMap.getNcIdFromAtId(pColId); // skip primary column configuration if we field not migrated - syncLog(`NC API: dbTableColumn.primaryColumnSet`); if (ncColId) { + logDetailed(`NC API: dbTableColumn.primaryColumnSet`); await api.dbTableColumn.primaryColumnSet(ncColId); // update schema @@ -1088,7 +1112,7 @@ export default async ( if (refRowIdList.length) { for (let i = 0; i < refRowIdList[0].length; i++) { - syncLog( + logDetailed( `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}` ); @@ -1113,7 +1137,7 @@ export default async ( // trim spaces on either side of column name // leads to error in NocoDB Object.keys(rec).forEach(key => { - const replacedKey = key.replace(/\?/g, 'QQ').trim(); + const replacedKey = key.trim().replace(/\./g, '_'); if (key !== replacedKey) { rec[replacedKey] = rec[key]; delete rec[key]; @@ -1154,12 +1178,19 @@ export default async ( if (dt === UITypes.Barcode) rec[key] = value.text; if (dt === UITypes.Button) rec[key] = `${value?.label} <${value?.url}>`; - if (dt === UITypes.DateTime) { + if ( + dt === UITypes.DateTime || + dt === UITypes.CreateTime || + dt === UITypes.LastModifiedTime + ) { const atDateField = dayjs(value); rec[key] = atDateField.utc().format('YYYY-MM-DD HH:mm'); } - if (dt === UITypes.MultiSelect) rec[key] = value.join(','); + if (dt === UITypes.SingleSelect) rec[key] = value.replace(/,/g, '.'); + + if (dt === UITypes.MultiSelect) + rec[key] = value.map(v => `${v.replace(/,/g, '.')}`).join(','); if (dt === UITypes.Attachment) { const tempArr = []; @@ -1210,15 +1241,11 @@ export default async ( } // insert airtable record ID explicitly into each records - rec['_aTbl_nc_rec_id'] = record.id; - rec['_aTbl_nc_rec_hash'] = recordHash; - - // console.log(rec) - - syncLog(`NC API: dbTableRow.bulkCreate ${table.title} [${rec}]`); - // console.log(JSON.stringify(rec, null, 2)) + rec[ncSysFields.id] = record.id; + rec[ncSysFields.hash] = recordHash; // bulk Insert + logDetailed(`NC API: dbTableRow.bulkCreate ${table.title} [${rec}]`); await api.dbTableRow.bulkCreate( 'nc', sDB.projectName, @@ -1239,6 +1266,10 @@ export default async ( // console.log(JSON.stringify(records, null, 2)); // This function (`page`) will get called for each page of records. + logBasic( + `:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += 100)}` + ); + await Promise.all( records.map(record => callback(sDB, table, record)) ); @@ -1273,6 +1304,10 @@ export default async ( // This function (`page`) will get called for each page of records. // records.forEach(record => callback(table, record)); + logBasic( + `:: ${table.title} / ${fields} : ${recordCnt + + 1} ~ ${(recordCnt += 100)}` + ); await Promise.all( records.map(r => callback(projName, table, r, fields)) ); @@ -1302,18 +1337,16 @@ export default async ( } async function nocoCreateProject(projName) { - syncLog(`Create Project: ${projName}`); - // create empty project (XC-DB) + logDetailed(`Create Project: ${projName}`); ncCreatedProjectSchema = await api.project.create({ title: projName }); } async function nocoGetProject(projId) { - syncLog(`Getting project meta: ${projId}`); - // create empty project (XC-DB) + logDetailed(`Getting project meta: ${projId}`); ncCreatedProjectSchema = await api.project.read(projId); } @@ -1325,19 +1358,11 @@ export default async ( x => x.type === 'gallery' ); - const configuredViews = - runTimeCounters.view.grid + - runTimeCounters.view.gallery + - runTimeCounters.view.form; - runTimeCounters.view.gallery += galleryViews.length; + const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; + rtc.view.gallery += galleryViews.length; for (let i = 0; i < galleryViews.length; i++) { - syncLog( - `[${configuredViews + i + 1}/${ - runTimeCounters.view.total - }] Configuring view :: Gallery` - ); - syncLog(` Axios fetch view-data`); + logDetailed(` Axios fetch view-data`); // create view await getViewData(galleryViews[i].id); @@ -1345,12 +1370,18 @@ export default async ( x => x.id === galleryViews[i].id )?.name; - syncLog(` Create NC View :: ${viewName}`); + logBasic( + `:: [${configuredViews + i + 1}/${rtc.view.total}] Gallery : ${ + aTblSchema[idx].name + } / ${viewName}` + ); + + logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); await api.dbView.galleryCreate(tblId, { title: viewName }); await updateNcTblSchemaById(tblId); // syncLog(`[${idx+1}/${aTblSchema.length}][Gallery View][${i+1}/${galleryViews.length}] Create ${viewName}`) - // await nc_configureFields(g.id, vData.columnOrder, aTblSchema[idx].name, viewName, 'gallery'); + // await nc_configureFields(g.id, vData, aTblSchema[idx].name, viewName, 'gallery'); } } } @@ -1361,18 +1392,10 @@ export default async ( const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const formViews = aTblSchema[idx].views.filter(x => x.type === 'form'); - const configuredViews = - runTimeCounters.view.grid + - runTimeCounters.view.gallery + - runTimeCounters.view.form; - runTimeCounters.view.form += formViews.length; + const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; + rtc.view.form += formViews.length; for (let i = 0; i < formViews.length; i++) { - syncLog( - `[${configuredViews + i + 1}/${ - runTimeCounters.view.total - }] Configuring view :: Form` - ); - syncLog(` Axios fetch view-data`); + logDetailed(` Axios fetch view-data`); // create view const vData = await getViewData(formViews[i].id); @@ -1380,6 +1403,12 @@ export default async ( x => x.id === formViews[i].id )?.name; + logBasic( + `:: [${configuredViews + i + 1}/${rtc.view.total}] Form : ${ + aTblSchema[idx].name + } / ${viewName}` + ); + // everything is default let refreshMode = 'NO_REFRESH'; let msg = 'Thank you for submitting the form!'; @@ -1405,9 +1434,9 @@ export default async ( show_blank_form: refreshMode.includes('AUTO_REFRESH') }; - syncLog(` Create NC View :: ${viewName}`); + logDetailed(`NC API dbView.formCreate :: ${viewName}`); const f = await api.dbView.formCreate(tblId, formData); - syncLog( + logDetailed( `[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${ formViews.length }] Create ${viewName}` @@ -1415,10 +1444,10 @@ export default async ( await updateNcTblSchemaById(tblId); - syncLog(` Configure show/hide columns`); + logDetailed(` Configure show/hide columns`); await nc_configureFields( f.id, - vData.columnOrder, + vData, aTblSchema[idx].name, viewName, 'form' @@ -1432,19 +1461,13 @@ export default async ( const tblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const gridViews = aTblSchema[idx].views.filter(x => x.type === 'grid'); - const configuredViews = - runTimeCounters.view.grid + - runTimeCounters.view.gallery + - runTimeCounters.view.form; - runTimeCounters.view.grid += gridViews.length; + let viewCnt = idx; + if (syncDB.syncViews) + viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form; + rtc.view.grid += gridViews.length; for (let i = 0; i < (sDB.syncViews ? gridViews.length : 1); i++) { - syncLog( - `[${configuredViews + i + 1}/${ - runTimeCounters.view.total - }] Configuring view :: Grid` - ); - syncLog(` Axios fetch view-data`); + logDetailed(` Axios fetch view-data`); // fetch viewData JSON const vData = await getViewData(gridViews[i].id); @@ -1455,9 +1478,15 @@ export default async ( const viewList: any = await api.dbView.list(tblId); let ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id; + logBasic( + `:: [${viewCnt + i + 1}/${rtc.view.total}] Grid : ${ + aTblSchema[idx].name + } / ${viewName}` + ); + // create view (default already created) if (i > 0) { - syncLog(` Create NC View :: ${viewName}`); + logDetailed(`NC API dbView.gridCreate :: ${viewName}`); const viewCreated = await api.dbView.gridCreate(tblId, { title: viewName }); @@ -1473,10 +1502,10 @@ export default async ( } // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Hide columns ${viewName}`) - syncLog(` Configure show/hide columns`); + logDetailed(` Configure show/hide columns`); await nc_configureFields( ncViewId, - vData.columnOrder, + vData, aTblSchema[idx].name, viewName, 'grid' @@ -1485,7 +1514,7 @@ export default async ( // configure filters if (vData?.filters) { // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure filters ${viewName}`) - syncLog(` Configure filter set`); + logDetailed(` Configure filter set`); // skip filters if nested if (!vData.filters.filterSet.find(x => x?.type === 'nested')) { @@ -1496,7 +1525,7 @@ export default async ( // configure sort if (vData?.lastSortsApplied?.sortSet.length) { // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure sort ${viewName}`) - syncLog(` Configure sort set`); + logDetailed(` Configure sort set`); await nc_configureSort(ncViewId, vData.lastSortsApplied); } } @@ -1519,7 +1548,9 @@ export default async ( for (const [, value] of Object.entries( userList as { [key: string]: any } )) { - syncLog(`[${++cnt}/${totalUsers}] Configuring User :: ${value.email}`); + logDetailed( + `[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${value.email}` + ); await api.auth.projectUserAdd(ncCreatedProjectSchema.id, { email: value.email, roles: userRoles[value.permissionLevel] @@ -1634,23 +1665,23 @@ export default async ( return accumulator + object.nc.rollup; }, 0); - syncLog(`Quick Stats:`); - syncLog(` Total Tables: ${aTblSchema.length}`); - syncLog(` Total Columns: ${columnSum}`); - syncLog(` Links: ${linkSum}`); - syncLog(` Lookup: ${lookupSum}`); - syncLog(` Rollup: ${rollupSum}`); - syncLog(` Total Filters: ${runTimeCounters.filter}`); - syncLog(` Total Sort: ${runTimeCounters.sort}`); - syncLog(` Total Views: ${runTimeCounters.view.total}`); - syncLog(` Grid: ${runTimeCounters.view.grid}`); - syncLog(` Gallery: ${runTimeCounters.view.gallery}`); - syncLog(` Form: ${runTimeCounters.view.form}`); + logDetailed(`Quick Summary:`); + logDetailed(`:: Total Tables: ${aTblSchema.length}`); + logDetailed(`:: Total Columns: ${columnSum}`); + logDetailed(`:: Links: ${linkSum}`); + logDetailed(`:: Lookup: ${lookupSum}`); + logDetailed(`:: Rollup: ${rollupSum}`); + logDetailed(`:: Total Filters: ${rtc.filter}`); + logDetailed(`:: Total Sort: ${rtc.sort}`); + logDetailed(`:: Total Views: ${rtc.view.total}`); + logDetailed(`:: Grid: ${rtc.view.grid}`); + logDetailed(`:: Gallery: ${rtc.view.gallery}`); + logDetailed(`:: Form: ${rtc.view.form}`); const duration = Date.now() - start; - syncLog(`Migration time: ${duration}`); - syncLog(`Axios fetch count: ${runTimeCounters.fetchAt.count}`); - syncLog(`Axios fetch time: ${runTimeCounters.fetchAt.time}`); + logDetailed(`:: Migration time: ${duration}`); + logDetailed(`:: Axios fetch count: ${rtc.fetchAt.count}`); + logDetailed(`:: Axios fetch time: ${rtc.fetchAt.time}`); } ////////////////////////////// @@ -1692,7 +1723,7 @@ export default async ( const ncFilters = []; // console.log(filter) - if (datatype === UITypes.Date) { + if (datatype === UITypes.Date || datatype === UITypes.DateTime) { // skip filters over data datatype updateMigrationSkipLog( sMap.getNcNameFromAtId(viewId), @@ -1748,7 +1779,7 @@ export default async ( await api.dbTableFilter.create(viewId, { ...ncFilters[i] }); - runTimeCounters.filter++; + rtc.filter++; } } } @@ -1762,13 +1793,14 @@ export default async ( fk_column_id: columnId, direction: s.sortSet[i].ascending ? 'asc' : 'dsc' }); - runTimeCounters.sort++; + rtc.sort++; } } - async function nc_configureFields(_viewId, c, tblName, viewName, viewType?) { + async function nc_configureFields(_viewId, _c, tblName, viewName, viewType?) { // force hide PK column - const hiddenColumns = ['_aTbl_nc_rec_id', '_aTbl_nc_rec_hash']; + const hiddenColumns = [ncSysFields.id, ncSysFields.hash]; + const c = _c.columnOrder; // column order corrections // retrieve table schema @@ -1805,17 +1837,25 @@ export default async ( if (ncViewColumnId === undefined) continue; // first two positions held by record id & record hash - await api.dbViewColumn.update(viewId, ncViewColumnId, { - show: c[j].visibility, - order: j + 1 - }); + const configData = { show: c[j].visibility, order: j + 1 }; + if (viewType === 'form') { + if (_c?.metadata?.form?.fieldsByColumnId?.[c[j].columnId]) { + const x = _c.metadata.form.fieldsByColumnId[c[j].columnId]; + const formData = { ...configData }; + if (x?.title) formData[`label`] = x.title; + if (x?.required) formData[`required`] = x.required; + if (x?.description) formData[`description`] = x.description; + await api.dbView.formColumnUpdate(ncViewColumnId, formData); + } + } + await api.dbViewColumn.update(viewId, ncViewColumnId, configData); } } /////////////////////////////////////////////////////////////////////////////// + let recordCnt = 0; try { - syncLog = progress; - progress('SDK initialized'); + logBasic('SDK initialized'); api = new Api({ baseURL: syncDB.baseURL, headers: { @@ -1823,118 +1863,123 @@ export default async ( } }); - progress('Project initialization started'); + logDetailed('Project initialization started'); // delete project if already exists if (debugMode) await init(syncDB); - progress('Project initialized'); + logDetailed('Project initialized'); - progress('Project schema extraction started'); + logBasic('Retrieving Airtable schema'); // read schema file const schema = await getAirtableSchema(syncDB); const aTblSchema = schema.tableSchemas; - progress('Project schema extraction completed'); + logDetailed('Project schema extraction completed'); if (!syncDB.projectId) { if (!syncDB.projectName) throw new Error('Project name or id not provided'); // create empty project await nocoCreateProject(syncDB.projectName); - progress('Project created'); + logDetailed('Project created'); } else { await nocoGetProject(syncDB.projectId); syncDB.projectName = ncCreatedProjectSchema?.title; - progress('Getting existing project meta'); + logDetailed('Getting existing project meta'); } - progress('Table creation started'); + logBasic('Importing Tables...'); // prepare table schema (base) await nocoCreateBaseSchema(aTblSchema); - progress('Table creation completed'); + logDetailed('Table creation completed'); - progress('Migrating LTAR columns'); + logDetailed('Configuring Links'); // add LTAR await nocoCreateLinkToAnotherRecord(aTblSchema); - progress('Migrating LTAR columns completed'); + logDetailed('Migrating LTAR columns completed'); - progress('Migrating Lookup columns'); + logDetailed(`Configuring Lookup`); // add look-ups await nocoCreateLookups(aTblSchema); - progress('Migrating Lookup columns completed'); + logDetailed('Migrating Lookup columns completed'); - progress('Migrating Rollup columns'); + logDetailed('Configuring Rollup'); // add roll-ups await nocoCreateRollup(aTblSchema); - progress('Migrating Rollup columns completed'); + logDetailed('Migrating Rollup columns completed'); - progress('Migrating Lookup form Rollup columns'); + logDetailed('Migrating Lookup form Rollup columns'); // lookups for rollup await nocoLookupForRollup(); - progress('Migrating Lookup form Rollup columns completed'); + logDetailed('Migrating Lookup form Rollup columns completed'); - progress('Configuring primary value column'); + logDetailed('Configuring Primary value column'); // configure primary values await nocoSetPrimary(aTblSchema); - progress('Configuring primary value column completed'); + logDetailed('Configuring primary value column completed'); - progress('Adding users'); + logBasic('Configuring User(s)'); // add users await nocoAddUsers(schema); - progress('Adding users completed'); + logDetailed('Adding users completed'); // hide-fields // await nocoReconfigureFields(aTblSchema); - progress('Syncing views'); + logBasic('Syncing views'); // configure views await nocoConfigureGridView(syncDB, aTblSchema); await nocoConfigureFormView(syncDB, aTblSchema); await nocoConfigureGalleryView(syncDB, aTblSchema); - progress('Syncing views completed'); + logDetailed('Syncing views completed'); if (process_aTblData) { try { // await nc_DumpTableSchema(); const ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); + logBasic('Reading Records...'); + for (let i = 0; i < ncTblList.list.length; i++) { const ncTbl = await api.dbTable.read(ncTblList.list[i].id); - progress(`Reading data from ${ncTbl.title}`); - let c = 0; + + // not a migrated table, skip + if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) + continue; + + recordCnt = 0; await nocoReadData(syncDB, ncTbl, async (sDB, table, record) => { - progress( - `Processing records from ${ncTbl.title} : ${c} - ${(c += 25)}` - ); await nocoBaseDataProcessing(sDB, table, record); }); - progress(`Data inserted from ${ncTbl.title}`); + logDetailed(`Data inserted from ${ncTbl.title}`); } + logBasic('Configuring Record Links...'); // Configure link @ Data row's for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { const x = ncLinkMappingTable[idx]; const ncTbl = await nc_getTableSchema( aTbl_getTableName(x.aTbl.tblId).tn ); - progress(`Linking data to ${ncTbl.title}`); - let c = 0; + + // not a migrated table, skip + if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) + continue; + + recordCnt = 0; await nocoReadDataSelected( syncDB.projectName, ncTbl, async (projName, table, record, _field) => { - progress( - `Mapping LTAR records from ${ncTbl.title} : ${c} - ${(c += 25)}` - ); await nocoLinkProcessing(projName, table, record, _field); }, x.aTbl.name ); - progress(`Linked data to ${ncTbl.title}`); + logDetailed(`Linked data to ${ncTbl.title}`); } } catch (error) { - progress( + logDetailed( `There was an error while migrating data! Please make sure your API key (${syncDB.apiKey}) is correct.` ); - progress(`Error: ${error}`); + logDetailed(`Error: ${error}`); } } if (generate_migrationStats) { @@ -1948,6 +1993,20 @@ export default async ( } }; +export function getUniqueNameGenerator(defaultName = 'name') { + const namesRef = {}; + + return (initName: string = defaultName): string => { + let name = initName === '_' ? defaultName : initName; + let c = 0; + while (name in namesRef) { + name = `${initName}_${++c}`; + } + namesRef[name] = true; + return name; + }; +} + export interface AirtableSyncConfig { id: string; baseURL: string; diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/importApis.ts b/packages/nocodb/src/lib/noco/meta/api/sync/importApis.ts index 5855f48e8e..7d6e61a616 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/importApis.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/importApis.ts @@ -20,13 +20,14 @@ export default (router: Router, clients: { [id: string]: Socket }) => { NocoJobs.jobsMgr.addJobWorker(AIRTABLE_IMPORT_JOB, job); NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => { clients?.[payload?.id]?.emit('progress', { - msg: progress, + msg: progress?.msg, + level: progress?.level, status: SyncStatus.PROGRESS }); }); NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, payload => { clients?.[payload?.id]?.emit('progress', { - msg: 'completed', + msg: 'Complete!', status: SyncStatus.COMPLETED }); }); diff --git a/packages/nocodb/src/lib/noco/meta/api/tableApis.ts b/packages/nocodb/src/lib/noco/meta/api/tableApis.ts index 43852fc4b6..4c2b477b83 100644 --- a/packages/nocodb/src/lib/noco/meta/api/tableApis.ts +++ b/packages/nocodb/src/lib/noco/meta/api/tableApis.ts @@ -27,6 +27,7 @@ import NcConnectionMgrv2 from '../../common/NcConnectionMgrv2'; import getColumnUiType from '../helpers/getColumnUiType'; import LinkToAnotherRecordColumn from '../../../noco-models/LinkToAnotherRecordColumn'; import { metaApiMetrics } from '../helpers/apiMetrics'; + export async function tableGet(req: Request, res: Response) { const table = await Model.getWithInfo({ id: req.params.tableId @@ -102,6 +103,13 @@ export async function tableCreate(req: Request, res) { } } + // validate table name + if (/^\s+|\s+$/.test(req.body.table_name)) { + NcError.badRequest( + 'Leading or trailing whitespace not allowed in table names' + ); + } + if ( !(await Model.checkTitleAvailable({ table_name: req.body.table_name, @@ -176,6 +184,11 @@ export async function tableCreate(req: Request, res) { ...colMetaFromReq, uidt: colMetaFromReq?.uidt || c.uidt || getColumnUiType(base, c), ...c, + dtxp: [UITypes.MultiSelect, UITypes.SingleSelect].includes( + colMetaFromReq.uidt as any + ) + ? colMetaFromReq.dtxp + : c.dtxp, title: colMetaFromReq?.title || getColumnNameAlias(c.cn, base), column_name: c.cn, order: i + 1 diff --git a/packages/nocodb/src/lib/noco/meta/helpers/formulaHelpers.ts b/packages/nocodb/src/lib/noco/meta/helpers/formulaHelpers.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/nocodb/src/lib/noco/meta/helpers/webhookHelpers.ts b/packages/nocodb/src/lib/noco/meta/helpers/webhookHelpers.ts index dba524aab9..dfdbdade0c 100644 --- a/packages/nocodb/src/lib/noco/meta/helpers/webhookHelpers.ts +++ b/packages/nocodb/src/lib/noco/meta/helpers/webhookHelpers.ts @@ -132,7 +132,6 @@ export function axiosRequestMake(_apiMeta, user, data) { }); } catch (e) { apiMeta.body = parseBody(apiMeta.body, user, data, apiMeta); - console.log(e); } } if (apiMeta.auth) { @@ -144,7 +143,6 @@ export function axiosRequestMake(_apiMeta, user, data) { }); } catch (e) { apiMeta.auth = parseBody(apiMeta.auth, user, data, apiMeta); - console.log(e); } } apiMeta.response = {}; diff --git a/packages/nocodb/src/lib/utils/projectAcl.ts b/packages/nocodb/src/lib/utils/projectAcl.ts index 6e5ecd3528..b231b27ee8 100644 --- a/packages/nocodb/src/lib/utils/projectAcl.ts +++ b/packages/nocodb/src/lib/utils/projectAcl.ts @@ -54,6 +54,7 @@ export default { formViewGet: true, projectInfoGet: true, gridColumnUpdate: true, + galleryViewGet: true, // old xcTableAndViewList: true, @@ -161,6 +162,8 @@ export default { dataGroupBy: true, commentsCount: true, + galleryViewGet: true, + xcTableAndViewList: true, xcVirtualTableList: true, projectList: true, @@ -204,6 +207,8 @@ export default { sortList: true, projectInfoGet: true, + galleryViewGet: true, + mmList: true, hmList: true, commentList: true, diff --git a/packages/nocodb/src/lib/utils/weAreHiring.ts b/packages/nocodb/src/lib/utils/weAreHiring.ts new file mode 100644 index 0000000000..e409a72ad8 --- /dev/null +++ b/packages/nocodb/src/lib/utils/weAreHiring.ts @@ -0,0 +1,13 @@ +import boxen from 'boxen'; + +export default function() { + console.log(` + ${boxen(`Join the forces http://careers.nocodb.com`, { + title: '🚀 We are Hiring!!! 🚀', + padding: 1, + margin: 1, + titleAlignment: 'center', + borderColor: 'green' + })} +`); +} diff --git a/scripts/cypress/integration/common/5a_user_role.js b/scripts/cypress/integration/common/5a_user_role.js index 9e8c80d4b9..0c46ef67a7 100644 --- a/scripts/cypress/integration/common/5a_user_role.js +++ b/scripts/cypress/integration/common/5a_user_role.js @@ -189,7 +189,7 @@ export const genTest = (apiType, dbType) => { // right navigation menu bar // Editor/Viewer/Commenter : can only view 'existing' views // Rest: can create/edit - _viewMenu(roleType, false); + _viewMenu(roleType, false, 2); }); it(`[${roles[roleType].name}] Top Right Menu bar`, () => { diff --git a/scripts/cypress/integration/common/5b_preview_role.js b/scripts/cypress/integration/common/5b_preview_role.js index 12fe5947f9..21875a5f0f 100644 --- a/scripts/cypress/integration/common/5b_preview_role.js +++ b/scripts/cypress/integration/common/5b_preview_role.js @@ -115,7 +115,7 @@ export const genTest = (apiType, dbType, roleType) => { // right navigation menu bar // Editor/Viewer/Commenter : can only view 'existing' views // Rest: can create/edit - _viewMenu(roleType, true); + _viewMenu(roleType, true, 2); }); it(`Role preview: ${roleType}: Top Right Menu bar`, () => { diff --git a/scripts/cypress/integration/common/6g_base_share.js b/scripts/cypress/integration/common/6g_base_share.js index 3e583942ae..475e8bc50a 100644 --- a/scripts/cypress/integration/common/6g_base_share.js +++ b/scripts/cypress/integration/common/6g_base_share.js @@ -48,7 +48,7 @@ export const genTest = (apiType, dbType) => { }); it(`${roleType}: Validate access permissions: view's menu`, () => { - _viewMenu(roleType, false); + _viewMenu(roleType, false, 1); }); }; diff --git a/scripts/cypress/integration/spec/roleValidation.spec.js b/scripts/cypress/integration/spec/roleValidation.spec.js index 2d86df82d9..e88c6af853 100644 --- a/scripts/cypress/integration/spec/roleValidation.spec.js +++ b/scripts/cypress/integration/spec/roleValidation.spec.js @@ -212,9 +212,10 @@ export function _editComment(roleType, previewMode) { // right navigation menu bar // Editor/Viewer/Commenter : can only view 'existing' views // Rest: can create/edit -export function _viewMenu(roleType, previewMode) { +export function _viewMenu(roleType, previewMode, navDrawListCnt) { let columnName = "City"; - let navDrawListCnt = 2; + // let navDrawListCnt = 2; + // Download CSV let actionsMenuItemsCnt = 1;