Nc fix/progress meta sync (#10448)

* feat: progress panel component

* feat: meta sync progress support

* test: meta sync

---------

Co-authored-by: mertmit <mertmit99@gmail.com>

packages/nc-gui/components/dashboard/settings/Metadata.vue

@@ -13,16 +13,24 @@ const { base } = storeToRefs(baseStore)
 
 const { t } = useI18n()
 
+const progressRef = ref()
+
 const isLoading = ref(false)
 
 const isDifferent = ref(false)
 
+const triggeredSync = ref(false)
+
+const syncCompleted = ref(false)
+
 const metadiff = ref<any[]>([])
 
-async function loadMetaDiff() {
+async function loadMetaDiff(afterSync = false) {
   try {
     if (!base.value?.id) return
 
+    if (triggeredSync.value && !syncCompleted.value && !afterSync) return
+
     isLoading.value = true
     isDifferent.value = false
     metadiff.value = await $api.source.metaDiffGet(base.value?.id, props.sourceId)
@@ -36,16 +44,24 @@ async function loadMetaDiff() {
     console.error(e)
   } finally {
     isLoading.value = false
+    if (afterSync) syncCompleted.value = true
   }
 }
 
 const { $poller } = useNuxtApp()
 
+const onBack = () => {
+  triggeredSync.value = false
+  syncCompleted.value = false
+}
+
 async function syncMetaDiff() {
   try {
     if (!base.value?.id || !isDifferent.value) return
 
     isLoading.value = true
+    triggeredSync.value = true
+
     const jobData = await $api.source.metaDiffSync(base.value?.id, props.sourceId)
 
     $poller.subscribe(
@@ -65,13 +81,21 @@ async function syncMetaDiff() {
           if (data.status === JobStatus.COMPLETED) {
             // Table metadata recreated successfully
             message.info(t('msg.info.metaDataRecreated'))
+            progressRef.value.pushProgress('Done!', data.status)
+
+            isLoading.value = false
+
             await loadTables()
-            await loadMetaDiff()
+            await loadMetaDiff(true)
+
             emit('baseSynced')
+          } else if (data.status === JobStatus.FAILED) {
+            progressRef.value.pushProgress(data.data?.error?.message || 'Failed to sync base metadata', data.status)
+            syncCompleted.value = true
             isLoading.value = false
-          } else if (status === JobStatus.FAILED) {
-            message.error('Failed to sync base metadata')
-            isLoading.value = false
+          } else {
+            // Job is still in progress
+            progressRef.value.pushProgress(data.data?.message)
           }
         }
       },
@@ -136,7 +160,7 @@ const customRow = (record: Record<string, any>) => ({
         <a-button
           v-e="['a:proj-meta:meta-data:reload']"
           class="self-start !rounded-md nc-btn-metasync-reload"
-          @click="loadMetaDiff"
+          @click="loadMetaDiff()"
         >
           <div class="flex items-center gap-2 text-gray-600 font-light">
             <component :is="iconMap.reload" :class="{ 'animate-infinite animate-spin !text-success': isLoading }" />
@@ -144,7 +168,25 @@ const customRow = (record: Record<string, any>) => ({
           </div>
         </a-button>
       </div>
+      <div v-if="triggeredSync" class="flex flex-col justify-center items-center h-full overflow-y-auto">
+        <GeneralProgressPanel ref="progressRef" class="w-1/2 h-full" />
+        <div class="flex justify-center">
+          <NcButton
+            html-type="submit"
+            class="mt-4 mb-8"
+            :class="{
+              'sync-completed': syncCompleted,
+            }"
+            size="medium"
+            :disabled="!syncCompleted"
+            @click="onBack"
+          >
+            Back
+          </NcButton>
+        </div>
+      </div>
       <NcTable
+        v-else
         :columns="columns"
         :data="metadiff ?? []"
         row-height="44px"

packages/nc-gui/components/dlg/AirtableImport.vue

@@ -1,5 +1,4 @@
 <script setup lang="ts">
-import type { Card as AntCard } from 'ant-design-vue'
 import { JobStatus } from '#imports'
 
 const { modelValue, baseId, sourceId, transition } = defineProps<{
@@ -29,9 +28,9 @@ const showGoToDashboardButton = ref(false)
 
 const step = ref(1)
 
-const progress = ref<Record<string, any>[]>([])
+const progressRef = ref()
 
-const logRef = ref<typeof AntCard>()
+const lastProgress = ref()
 
 const enableAbort = ref(false)
 
@@ -62,35 +61,28 @@ const syncSource = ref({
   },
 })
 
-const pushProgress = async (message: string, status: JobStatus | 'progress') => {
-  progress.value.push({ msg: message, status })
-
-  await nextTick(() => {
-    const container: HTMLDivElement = logRef.value?.$el?.firstElementChild
-    if (!container) return
-    container.scrollTop = container.scrollHeight
-  })
+const onLog = (data: { message: string }) => {
+  progressRef.value?.pushProgress(data.message, 'progress')
+  lastProgress.value = { msg: data.message, status: 'progress' }
 }
 
 const onStatus = async (status: JobStatus, data?: any) => {
+  lastProgress.value = { msg: data?.message, status }
+
   if (status === JobStatus.COMPLETED) {
     showGoToDashboardButton.value = true
     await loadTables()
-    pushProgress('Done!', status)
+    progressRef.value?.pushProgress('Done!', status)
     refreshCommandPalette()
     // TODO: add tab of the first table
   } else if (status === JobStatus.FAILED) {
     await loadTables()
     goBack.value = true
-    pushProgress(data.error.message, status)
+    progressRef.value?.pushProgress(data.error.message, status)
     refreshCommandPalette()
   }
 }
 
-const onLog = (data: { message: string }) => {
-  pushProgress(data.message, 'progress')
-}
-
 const validators = computed(() => ({
   'details.apiKey': [fieldRequiredValidator()],
   'details.syncSourceUrlOrId': [fieldRequiredValidator()],
@@ -279,29 +271,8 @@ onMounted(async () => {
   await loadSyncSrc()
 })
 
-function downloadLogs(filename: string) {
-  let text = ''
-  for (const o of document.querySelectorAll('.nc-modal-airtable-import .log-message')) {
-    text += `${o.textContent}\n`
-  }
-  const element = document.createElement('a')
-  element.setAttribute('href', `data:text/plain;charset=utf-8,${encodeURIComponent(text)}`)
-  element.setAttribute('download', filename)
-
-  element.style.display = 'none'
-  document.body.appendChild(element)
-
-  element.click()
-
-  document.body.removeChild(element)
-}
-
 const isInProgress = computed(() => {
-  return (
-    !progress.value ||
-    !progress.value.length ||
-    ![JobStatus.COMPLETED, JobStatus.FAILED].includes(progress.value[progress.value.length - 1]?.status)
-  )
+  return !lastProgress.value || ![JobStatus.COMPLETED, JobStatus.FAILED].includes(lastProgress.value?.status)
 })
 
 const detailsIsShown = ref(false)
@@ -430,51 +401,15 @@ const collapseKey = ref('')
     </div>
 
     <div v-if="step === 2">
-      <a-card
-        v-if="detailsIsShown"
-        ref="logRef"
-        class="nc-import-logs-container"
-        :body-style="{
-          'backgroundColor': '#101015',
-          'height': '200px',
-          'overflow': 'auto',
-          'borderRadius': '0.5rem',
-          'padding': '16px !important',
-          'scrollbar-color': 'var(--scrollbar-thumb) var(--scrollbar-track)',
-          'scrollbar-width': 'thin',
-          '--scrollbar-thumb': '#E7E7E9',
-          '--scrollbar-track': 'transparent',
-        }"
-      >
-        <a-button
-          v-if="showGoToDashboardButton || goBack"
-          class="!absolute z-1 right-2 bottom-2 opacity-75 hover:opacity-100 !rounded-md !w-8 !h-8"
-          size="small"
-          @click="downloadLogs('at-import-logs.txt')"
-        >
-          <nc-tooltip>
-            <template #title>Download Logs</template>
-            <component :is="iconMap.download" />
-          </nc-tooltip>
-        </a-button>
-
-        <div v-for="({ msg, status }, i) in progress" :key="i" class="my-1">
-          <div v-if="status === JobStatus.FAILED" class="flex items-start">
-            <span class="text-red-400 ml-2 log-message">{{ msg }}</span>
-          </div>
-          <div v-else class="flex items-start">
-            <span class="text-green-400 ml-2 log-message">{{ msg }}</span>
-          </div>
-        </div>
-      </a-card>
-      <div v-else class="flex items-start gap-2">
+      <GeneralProgressPanel v-show="detailsIsShown" ref="progressRef" class="w-full h-[200px]" />
+      <div v-show="!detailsIsShown" class="flex items-start gap-2">
         <template v-if="isInProgress">
           <component :is="iconMap.loading" class="text-primary animate-spin mt-1" />
           <span>
-            {{ progress?.[progress?.length - 1]?.msg ?? '---' }}
+            {{ lastProgress?.msg ?? '---' }}
           </span>
         </template>
-        <template v-else-if="progress?.[progress?.length - 1]?.status === JobStatus.FAILED">
+        <template v-else-if="lastProgress?.status === JobStatus.FAILED">
           <a-alert class="!rounded-lg !bg-transparent !border-gray-200 !p-3 !w-full">
             >
             <template #message>
@@ -485,7 +420,7 @@ const collapseKey = ref('')
             </template>
             <template #description>
               <div class="text-gray-500 text-[13px] leading-5 ml-6">
-                {{ progress?.[progress?.length - 1]?.msg ?? '---' }}
+                {{ lastProgress?.msg ?? '---' }}
               </div>
             </template>
           </a-alert>
@@ -497,9 +432,7 @@ const collapseKey = ref('')
       </div>
 
       <div v-if="!isInProgress" class="text-right mt-4">
-        <nc-button v-if="progress?.[progress?.length - 1]?.status === JobStatus.FAILED" size="small" @click="step = 1">
-          Retry import
-        </nc-button>
+        <nc-button v-if="lastProgress?.status === JobStatus.FAILED" size="small" @click="step = 1"> Retry import </nc-button>
         <nc-button v-else size="small" @click="dialogShow = false"> Go to base </nc-button>
       </div>
     </div>

packages/nc-gui/components/general/ProgressPanel.vue

@@ -0,0 +1,134 @@
+<script lang="ts" setup>
+import type { Card as AntCard } from 'ant-design-vue'
+
+const progress = ref<Record<string, any>[]>([])
+
+const logRef = ref<typeof AntCard>()
+
+const autoScroll = ref(true)
+
+const scrollToBottom = () => {
+  const container: HTMLDivElement = logRef.value?.$el?.firstElementChild
+  if (!container) return
+  container.scrollTop = container.scrollHeight
+}
+
+const pushProgress = (message: string, status: JobStatus | 'progress') => {
+  if (!message?.trim()) return
+  progress.value.push({ msg: message, status })
+
+  if (autoScroll.value) {
+    nextTick(scrollToBottom)
+  }
+}
+
+const onUserScroll = (e: Event) => {
+  const { scrollTop, scrollHeight, clientHeight } = e.target as HTMLDivElement
+
+  // If user is not at the bottom, disable auto-scroll
+  autoScroll.value = scrollTop + clientHeight >= scrollHeight - 10
+}
+
+function downloadLogs(filename: string) {
+  let text = ''
+  for (const { msg } of progress.value) {
+    text += `${msg}\n`
+  }
+
+  const element = document.createElement('a')
+  element.setAttribute('href', `data:text/plain;charset=utf-8,${encodeURIComponent(text)}`)
+  element.setAttribute('download', filename)
+
+  element.style.display = 'none'
+  document.body.appendChild(element)
+
+  element.click()
+
+  document.body.removeChild(element)
+}
+
+const progressEnd = computed(() =>
+  [JobStatus.FAILED, JobStatus.COMPLETED].includes(progress.value[progress.value.length - 1]?.status),
+)
+
+useEventListener(() => logRef.value?.$el?.firstElementChild, 'scroll', onUserScroll, {
+  passive: true,
+})
+
+defineExpose({
+  pushProgress,
+})
+
+onMounted(() => {
+  scrollToBottom()
+})
+</script>
+
+<template>
+  <a-card
+    ref="logRef"
+    :body-style="{
+      'overflow': 'auto',
+      'width': '100%',
+      'height': '100%',
+      'backgroundColor': '#101015',
+      'borderRadius': '0.5rem',
+      'padding': '16px !important',
+      'scrollbar-color': 'var(--scrollbar-thumb) var(--scrollbar-track)',
+      'scrollbar-width': 'thin',
+      '--scrollbar-thumb': '#E7E7E9',
+      '--scrollbar-track': 'transparent',
+    }"
+  >
+    <div v-for="({ msg, status }, i) in progress" :key="i">
+      <div v-if="status === JobStatus.FAILED" class="flex items-center">
+        <component :is="iconMap.closeCircle" class="text-red-500" />
+
+        <span class="text-red-500 ml-2">{{ msg }}</span>
+      </div>
+
+      <div v-else class="flex items-center">
+        <MdiCurrencyUsd class="text-green-500" />
+
+        <span class="text-green-500 ml-2">{{ msg }}</span>
+      </div>
+    </div>
+
+    <div v-if="!progressEnd" class="flex items-center">
+      <component :is="iconMap.loading" class="text-green-500 animate-spin" />
+      <span class="text-green-500 ml-2">Loading...</span>
+    </div>
+
+    <a-button
+      v-if="progressEnd"
+      class="!absolute z-1 right-2 bottom-2 opacity-75 hover:opacity-100 !rounded-md !w-8 !h-8"
+      size="small"
+      @click="downloadLogs('logs.txt')"
+    >
+      <nc-tooltip>
+        <template #title>Download Logs</template>
+        <component :is="iconMap.download" />
+      </nc-tooltip>
+    </a-button>
+  </a-card>
+</template>
+
+<style lang="scss" scoped>
+.nc-progress-panel {
+  @apply p-6 flex-1 flex justify-center;
+}
+</style>
+
+<style lang="scss">
+.nc-modal-create-source {
+  .nc-modal {
+    @apply !p-0;
+    height: min(calc(100vh - 100px), 1024px);
+    max-height: min(calc(100vh - 100px), 1024px) !important;
+  }
+}
+
+.nc-dropdown-ext-db-type {
+  @apply !z-1000;
+}
+</style>

packages/nocodb/src/modules/jobs/jobs/meta-sync/meta-sync.processor.ts

@@ -3,12 +3,16 @@ import { Injectable } from '@nestjs/common';
 import type { Job } from 'bull';
 import type { NcContext, NcRequest } from '~/interface/config';
 import { MetaDiffsService } from '~/services/meta-diffs.service';
+import { JobsLogService } from '~/modules/jobs/jobs/jobs-log.service';
 
 @Injectable()
 export class MetaSyncProcessor {
   private readonly debugLog = debug('nc:jobs:meta-sync');
 
-  constructor(private readonly metaDiffsService: MetaDiffsService) {}
+  constructor(
+    private readonly metaDiffsService: MetaDiffsService,
+    private readonly jobsLogService: JobsLogService,
+  ) {}
 
   async job(job: Job) {
     this.debugLog(`job started for ${job.id}`);
@@ -23,15 +27,22 @@ export class MetaSyncProcessor {
     const context = info.context;
     const baseId = context.base_id;
 
+    const logBasic = (log) => {
+      this.jobsLogService.sendLog(job, { message: log });
+      this.debugLog(log);
+    };
+
     if (info.sourceId === 'all') {
       await this.metaDiffsService.metaDiffSync(context, {
         baseId: baseId,
+        logger: logBasic,
         req: info.req,
       });
     } else {
       await this.metaDiffsService.baseMetaDiffSync(context, {
         baseId: baseId,
         sourceId: info.sourceId,
+        logger: logBasic,
         req: info.req,
       });
     }

packages/nocodb/src/services/meta-diffs.service.ts

@@ -670,11 +670,13 @@ export class MetaDiffsService {
       base,
       source,
       throwOnFail = false,
+      logger,
       user,
     }: {
       base: Base;
       source: Source;
       throwOnFail?: boolean;
+      logger?: (message: string) => void;
       user: UserType;
     },
   ) {
@@ -685,6 +687,8 @@ export class MetaDiffsService {
 
     const virtualColumnInsert: Array<() => Promise<void>> = [];
 
+    logger?.(`Getting meta diff for ${source.alias}`);
+
     // @ts-ignore
     const sqlClient = await NcConnectionMgrv2.getSqlClient(source);
     const changes = await this.getMetaDiff(context, sqlClient, base, source);
@@ -702,7 +706,15 @@ export class MetaDiffsService {
         );
       });
 
+      if (detectedChanges.length === 0) {
+        logger?.(`No changes detected for ${table_name}`);
+        continue;
+      }
+
+      logger?.(`Applying changes for ${table_name}`);
+
       for (const change of detectedChanges) {
+        logger?.(`Applying change: ${change.msg}`);
         switch (change.type) {
           case MetaDiffType.TABLE_NEW:
             {
@@ -909,21 +921,38 @@ export class MetaDiffsService {
             break;
         }
       }
+      logger?.(`Changes applied for ${table_name}`);
     }
 
+    logger?.(`Processing virtual column changes`);
+
     await NcHelp.executeOperations(virtualColumnInsert, source.type);
 
+    logger?.(`Virtual column changes applied`);
+
+    logger?.(`Processing many to many relation changes`);
+
     // populate m2m relations
     await this.extractAndGenerateManyToManyRelations(
       context,
       await source.getModels(context),
     );
+
+    logger?.(`Many to many relation changes applied`);
   }
 
-  async metaDiffSync(context: NcContext, param: { baseId: string; req: any }) {
+  async metaDiffSync(
+    context: NcContext,
+    param: { baseId: string; logger?: (message: string) => void; req: any },
+  ) {
     const base = await Base.getWithInfo(context, param.baseId);
     for (const source of base.sources) {
-      await this.syncBaseMeta(context, { base, source, user: param.req.user });
+      await this.syncBaseMeta(context, {
+        base,
+        source,
+        logger: param.logger,
+        user: param.req.user,
+      });
     }
 
     this.appHooksService.emit(AppEvents.META_DIFF_SYNC, {
@@ -940,6 +969,7 @@ export class MetaDiffsService {
     param: {
       baseId: string;
       sourceId: string;
+      logger?: (message: string) => void;
       req: any;
     },
   ) {
@@ -950,6 +980,7 @@ export class MetaDiffsService {
       base,
       source,
       throwOnFail: true,
+      logger: param.logger,
       user: param.req.user,
     });
 

tests/playwright/pages/Dashboard/ProjectView/Metadata.ts

@@ -33,8 +33,9 @@ export class MetaDataPage extends BasePage {
   async sync() {
     await this.get().locator(`button:has-text("Sync Now")`).click();
     await this.verifyToast({ message: 'Table metadata recreated successfully' });
-    await this.get().locator(`.animate-spin`).waitFor({ state: 'visible' });
-    await this.get().locator(`.animate-spin`).waitFor({ state: 'detached', timeout: 10000 });
+    // wait for clickability of the sync button
+    await this.get().locator(`.sync-completed`).waitFor({ state: 'visible' });
+    await this.get().locator(`.sync-completed`).click();
   }
 
   async verifyRow({ index, model, state }: { index: number; model: string; state: string }) {