detect fields in context menu

app/controllers/templates_detect_fields_controller.rb

@@ -11,11 +11,14 @@ class TemplatesDetectFieldsController < ApplicationController
     sse = SSE.new(response.stream)
 
     documents = @template.schema_documents.preload(:blob)
+    documents = documents.where(uuid: params[:attachment_uuid]) if params[:attachment_uuid].present?
+
+    page_number = params[:page].present? ? params[:page].to_i : nil
 
     documents.each do |document|
       io = StringIO.new(document.download)
 
-      Templates::DetectFields.call(io, attachment: document) do |(attachment_uuid, page, fields)|
+      Templates::DetectFields.call(io, attachment: document, page_number:) do |(attachment_uuid, page, fields)|
         sse.write({ attachment_uuid:, page:, fields: })
       end
     end

app/javascript/template_builder/builder.vue

@@ -373,6 +373,7 @@
                 :draw-field-type="drawFieldType"
                 :editable="editable"
                 :base-url="baseUrl"
+                :with-fields-detection="withFieldsDetection"
                 @draw="[onDraw($event), withSelectedFieldType ? '' : drawFieldType = '', showDrawField = false]"
                 @drop-field="onDropfield"
                 @remove-area="removeArea"
@@ -381,6 +382,7 @@
                 @copy-selected-areas="copySelectedAreas"
                 @delete-selected-areas="deleteSelectedAreas"
                 @align-selected-areas="alignSelectedAreas"
+                @autodetect-fields="detectFieldsForPage"
               />
               <DocumentControls
                 v-if="isBreakpointLg && editable"
@@ -523,6 +525,43 @@
         @select="startFieldDraw($event)"
       />
     </div>
+    <Transition
+      enter-active-class="transition-all duration-300 ease-out"
+      enter-from-class="translate-y-4 opacity-0"
+      enter-to-class="translate-y-0 opacity-100"
+      leave-active-class="transition-all duration-300 ease-in"
+      leave-from-class="translate-y-0 opacity-100"
+      leave-to-class="translate-y-4 opacity-0"
+    >
+      <div
+        v-if="isDetectingPageFields || detectingFieldsAddedCount !== null"
+        class="sticky bottom-0 z-50"
+      >
+        <div class="absolute left-0 right-0 h-0 overflow-visible bottom-16 z-50 flex justify-center">
+          <div
+            class="rounded-full bg-base-content h-12 flex items-center justify-center space-x-1.5 uppercase font-semibold text-white text-sm cursor-default"
+            style="min-width: 180px"
+          >
+            <template v-if="detectingFieldsAddedCount !== null">
+              <span>{{ (detectingFieldsAddedCount === 1 ? t('field_added') : t('fields_added')).replace('{count}', detectingFieldsAddedCount) }}</span>
+            </template>
+            <template v-else>
+              <IconInnerShadowTop
+                v-if="!detectingAnalyzingProgress"
+                width="20"
+                class="animate-spin"
+              />
+              <span v-if="detectingAnalyzingProgress">
+                {{ Math.round(detectingAnalyzingProgress * 100) }}% {{ t('analyzing_') }}
+              </span>
+              <span v-else>
+                {{ t('processing_') }}
+              </span>
+            </template>
+          </div>
+        </div>
+      </div>
+    </Transition>
     <div
       id="docuseal_modal_container"
       class="modal-container"
@@ -855,6 +894,9 @@ export default {
       isDownloading: false,
       isLoadingBlankPage: false,
       isSaving: false,
+      isDetectingPageFields: false,
+      detectingAnalyzingProgress: null,
+      detectingFieldsAddedCount: null,
       selectedSubmitter: null,
       showDrawField: false,
       pendingFieldAttachmentUuids: [],
@@ -2444,6 +2486,194 @@ export default {
         }
       })
     },
+    detectFieldsForPage ({ page, attachmentUuid }) {
+      this.isDetectingPageFields = true
+      this.detectingAnalyzingProgress = null
+      this.detectingFieldsAddedCount = null
+
+      let totalFieldsAdded = 0
+      const hadFieldsBeforeDetection = this.template.fields.length > 0
+
+      const calculateIoU = (area1, area2) => {
+        const x1 = Math.max(area1.x, area2.x)
+        const y1 = Math.max(area1.y, area2.y)
+        const x2 = Math.min(area1.x + area1.w, area2.x + area2.w)
+        const y2 = Math.min(area1.y + area1.h, area2.y + area2.h)
+
+        const intersectionArea = Math.max(0, x2 - x1) * Math.max(0, y2 - y1)
+        const area1Size = area1.w * area1.h
+        const area2Size = area2.w * area2.h
+        const unionArea = area1Size + area2Size - intersectionArea
+
+        return unionArea > 0 ? intersectionArea / unionArea : 0
+      }
+
+      const hasOverlappingField = (newArea) => {
+        const pageAreas = this.fieldAreasIndex[newArea.attachment_uuid]?.[newArea.page] || []
+
+        return pageAreas.some(({ area: existingArea }) => {
+          return calculateIoU(existingArea, newArea) >= 0.1
+        })
+      }
+
+      const filterNonOverlappingFields = (detectedFields) => {
+        return detectedFields.filter((field) => {
+          return (field.areas || []).every((area) => !hasOverlappingField(area))
+        })
+      }
+
+      this.baseFetch(`/templates/${this.template.id}/detect_fields`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ attachment_uuid: attachmentUuid, page })
+      }).then(async (response) => {
+        const reader = response.body.getReader()
+        const decoder = new TextDecoder('utf-8')
+        let buffer = ''
+        const fields = []
+
+        while (true) {
+          const { value, done } = await reader.read()
+
+          buffer += decoder.decode(value, { stream: true })
+
+          const lines = buffer.split('\n\n')
+
+          buffer = lines.pop()
+
+          for (const line of lines) {
+            if (line.startsWith('data: ')) {
+              const jsonStr = line.replace(/^data: /, '')
+              const data = JSON.parse(jsonStr)
+
+              if (data.error) {
+                const errorFields = filterNonOverlappingFields(data.fields || fields)
+
+                if (errorFields.length) {
+                  errorFields.forEach((f) => {
+                    if (!f.submitter_uuid) {
+                      f.submitter_uuid = this.template.submitters[0].uuid
+                    }
+                    this.insertField(f)
+                  })
+
+                  totalFieldsAdded += errorFields.length
+
+                  this.save()
+                } else if (!(data.fields || fields).length) {
+                  alert(data.error)
+                }
+
+                break
+              } else if (data.analyzing) {
+                this.detectingAnalyzingProgress = data.progress
+              } else if (data.completed) {
+                if (data.submitters) {
+                  if (!hadFieldsBeforeDetection) {
+                    this.template.submitters = data.submitters
+                    this.selectedSubmitter = this.template.submitters[0]
+
+                    const finalFields = data.fields || fields
+
+                    finalFields.forEach((f) => {
+                      if (!f.submitter_uuid) {
+                        f.submitter_uuid = this.template.submitters[0].uuid
+                      }
+                    })
+
+                    const nonOverlappingFields = filterNonOverlappingFields(finalFields)
+
+                    nonOverlappingFields.forEach((f) => this.insertField(f))
+                    totalFieldsAdded += nonOverlappingFields.length
+
+                    if (nonOverlappingFields.length) {
+                      this.save()
+                    }
+                  } else {
+                    const existingSubmitters = this.template.submitters
+                    const submitterUuidMap = {}
+
+                    data.submitters.forEach((newSubmitter) => {
+                      const existingMatch = existingSubmitters.find(
+                        (s) => s.name.toLowerCase() === newSubmitter.name.toLowerCase()
+                      )
+
+                      if (existingMatch) {
+                        submitterUuidMap[newSubmitter.uuid] = existingMatch.uuid
+                      } else {
+                        submitterUuidMap[newSubmitter.uuid] = newSubmitter.uuid
+
+                        if (!existingSubmitters.find((s) => s.uuid === newSubmitter.uuid)) {
+                          this.template.submitters.push(newSubmitter)
+                        }
+                      }
+                    })
+
+                    const finalFields = data.fields || fields
+
+                    finalFields.forEach((f) => {
+                      if (f.submitter_uuid && submitterUuidMap[f.submitter_uuid]) {
+                        f.submitter_uuid = submitterUuidMap[f.submitter_uuid]
+                      } else if (!f.submitter_uuid) {
+                        f.submitter_uuid = this.template.submitters[0].uuid
+                      }
+                    })
+
+                    const nonOverlappingFields = filterNonOverlappingFields(finalFields)
+
+                    nonOverlappingFields.forEach((f) => this.insertField(f))
+                    totalFieldsAdded += nonOverlappingFields.length
+
+                    if (nonOverlappingFields.length) {
+                      this.save()
+                    }
+                  }
+                } else {
+                  const finalFields = data.fields || fields
+
+                  finalFields.forEach((f) => {
+                    if (!f.submitter_uuid) {
+                      f.submitter_uuid = this.template.submitters[0].uuid
+                    }
+                  })
+
+                  const nonOverlappingFields = filterNonOverlappingFields(finalFields)
+
+                  nonOverlappingFields.forEach((f) => this.insertField(f))
+                  totalFieldsAdded += nonOverlappingFields.length
+
+                  if (nonOverlappingFields.length) {
+                    this.save()
+                  }
+                }
+
+                break
+              } else if (data.fields) {
+                data.fields.forEach((f) => {
+                  if (!f.submitter_uuid) {
+                    f.submitter_uuid = this.template.submitters[0].uuid
+                  }
+                })
+
+                fields.push(...data.fields)
+              }
+            }
+          }
+
+          if (done) break
+        }
+      }).catch(error => {
+        console.error('Error in streaming message: ', error)
+      }).finally(() => {
+        this.isDetectingPageFields = false
+        this.detectingAnalyzingProgress = null
+        this.detectingFieldsAddedCount = totalFieldsAdded
+
+        setTimeout(() => {
+          this.detectingFieldsAddedCount = null
+        }, 1000)
+      })
+    },
     save ({ force } = { force: false }) {
       this.pendingFieldAttachmentUuids = []
 

app/javascript/template_builder/context_menu.vue

@@ -185,6 +185,18 @@
         </span>
         <span class="text-xs text-base-content/60 ml-4">Tab</span>
       </button>
+      <hr
+        v-if="showAutodetectFields"
+        class="my-1 border-base-300"
+      >
+      <button
+        v-if="showAutodetectFields"
+        class="w-full px-2 py-1 rounded-md hover:bg-base-100 flex items-center space-x-2 text-sm"
+        @click.stop="$emit('autodetect-fields')"
+      >
+        <IconSparkles class="w-4 h-4" />
+        <span>{{ t('autodetect_fields') }}</span>
+      </button>
     </div>
     <Teleport
       v-if="isShowFormulaModal"
@@ -239,7 +251,7 @@
 </template>
 
 <script>
-import { IconCopy, IconClipboard, IconTrashX, IconTypography, IconInfoCircle, IconRouteAltLeft, IconMathFunction, IconClick, IconNewSection, IconLayoutAlignLeft, IconLayoutAlignRight, IconLayoutAlignTop, IconLayoutAlignBottom } from '@tabler/icons-vue'
+import { IconCopy, IconClipboard, IconTrashX, IconTypography, IconInfoCircle, IconRouteAltLeft, IconMathFunction, IconClick, IconNewSection, IconLayoutAlignLeft, IconLayoutAlignRight, IconLayoutAlignTop, IconLayoutAlignBottom, IconSparkles } from '@tabler/icons-vue'
 import FormulaModal from './formula_modal'
 import FontModal from './font_modal'
 import ConditionsModal from './conditions_modal'
@@ -263,6 +275,7 @@ export default {
     IconLayoutAlignRight,
     IconLayoutAlignTop,
     IconLayoutAlignBottom,
+    IconSparkles,
     FormulaModal,
     FontModal,
     ConditionsModal,
@@ -294,9 +307,13 @@ export default {
     template: {
       type: Object,
       default: null
+    },
+    withFieldsDetection: {
+      type: Boolean,
+      default: false
     }
   },
-  emits: ['copy', 'paste', 'delete', 'close', 'align'],
+  emits: ['copy', 'paste', 'delete', 'close', 'align', 'autodetect-fields'],
   data () {
     return {
       isShowFormulaModal: false,
@@ -392,6 +409,9 @@ export default {
     },
     showSelectFields () {
       return !this.contextMenu.area && !this.isMultiSelection
+    },
+    showAutodetectFields () {
+      return this.withFieldsDetection && this.editable && !this.contextMenu.area && !this.isMultiSelection
     }
   },
   mounted () {

app/javascript/template_builder/document.vue

@@ -23,6 +23,7 @@
       :total-pages="sortedPreviewImages.length"
       :image="image"
       :attachment-uuid="document.uuid"
+      :with-fields-detection="withFieldsDetection"
       @drop-field="$emit('drop-field', { ...$event, attachment_uuid: document.uuid })"
       @remove-area="$emit('remove-area', $event)"
       @copy-field="$emit('copy-field', $event)"
@@ -30,6 +31,7 @@
       @copy-selected-areas="$emit('copy-selected-areas')"
       @delete-selected-areas="$emit('delete-selected-areas')"
       @align-selected-areas="$emit('align-selected-areas', $event)"
+      @autodetect-fields="$emit('autodetect-fields', $event)"
       @scroll-to="scrollToArea"
       @draw="$emit('draw', { area: {...$event.area, attachment_uuid: document.uuid }, isTooSmall: $event.isTooSmall })"
     />
@@ -122,9 +124,14 @@ export default {
       type: Boolean,
       required: false,
       default: false
+    },
+    withFieldsDetection: {
+      type: Boolean,
+      required: false,
+      default: false
     }
   },
-  emits: ['draw', 'drop-field', 'remove-area', 'paste-field', 'copy-field', 'copy-selected-areas', 'delete-selected-areas', 'align-selected-areas'],
+  emits: ['draw', 'drop-field', 'remove-area', 'paste-field', 'copy-field', 'copy-selected-areas', 'delete-selected-areas', 'align-selected-areas', 'autodetect-fields'],
   data () {
     return {
       pageRefs: []

app/javascript/template_builder/i18n.js

@@ -194,7 +194,9 @@ const en = {
   align_right: 'Align Right',
   align_top: 'Align Top',
   align_bottom: 'Align Bottom',
-  fields_selected: '{count} Fields Selected'
+  fields_selected: '{count} Fields Selected',
+  field_added: '{count} Field Added',
+  fields_added: '{count} Fields Added'
 }
 
 const es = {
@@ -393,7 +395,9 @@ const es = {
   align_right: 'Alinear a la derecha',
   align_top: 'Alinear arriba',
   align_bottom: 'Alinear abajo',
-  fields_selected: '{count} Campos Seleccionados'
+  fields_selected: '{count} Campos Seleccionados',
+  field_added: '{count} Campo Añadido',
+  fields_added: '{count} Campos Añadidos'
 }
 
 const it = {
@@ -592,7 +596,9 @@ const it = {
   align_right: 'Allinea a destra',
   align_top: 'Allinea in alto',
   align_bottom: 'Allinea in basso',
-  fields_selected: '{count} Campi Selezionati'
+  fields_selected: '{count} Campi Selezionati',
+  field_added: '{count} Campo Aggiunto',
+  fields_added: '{count} Campi Aggiunti'
 }
 
 const pt = {
@@ -791,7 +797,9 @@ const pt = {
   align_right: 'Alinhar à direita',
   align_top: 'Alinhar ao topo',
   align_bottom: 'Alinhar à parte inferior',
-  fields_selected: '{count} Campos Selecionados'
+  fields_selected: '{count} Campos Selecionados',
+  field_added: '{count} Campo Adicionado',
+  fields_added: '{count} Campos Adicionados'
 }
 
 const fr = {
@@ -990,7 +998,9 @@ const fr = {
   align_right: 'Aligner à droite',
   align_top: 'Aligner en haut',
   align_bottom: 'Aligner en bas',
-  fields_selected: '{count} Champs Sélectionnés'
+  fields_selected: '{count} Champs Sélectionnés',
+  field_added: '{count} Champ Ajouté',
+  fields_added: '{count} Champs Ajoutés'
 }
 
 const de = {
@@ -1189,7 +1199,9 @@ const de = {
   align_right: 'Rechts ausrichten',
   align_top: 'Oben ausrichten',
   align_bottom: 'Unten ausrichten',
-  fields_selected: '{count} Felder Ausgewählt'
+  fields_selected: '{count} Felder Ausgewählt',
+  field_added: '{count} Feld Hinzugefügt',
+  fields_added: '{count} Felder Hinzugefügt'
 }
 
 const nl = {
@@ -1388,7 +1400,9 @@ const nl = {
   align_right: 'Rechts uitlijnen',
   align_top: 'Boven uitlijnen',
   align_bottom: 'Onder uitlijnen',
-  fields_selected: '{count} Velden Geselecteerd'
+  fields_selected: '{count} Velden Geselecteerd',
+  field_added: '{count} Veld Toegevoegd',
+  fields_added: '{count} Velden Toegevoegd'
 }
 
 export { en, es, it, pt, fr, de, nl }

app/javascript/template_builder/page.vue

@@ -72,9 +72,11 @@
         :context-menu="contextMenu"
         :field="contextMenu.field"
         :editable="editable"
+        :with-fields-detection="withFieldsDetection"
         @copy="handleCopy"
         @delete="handleDelete"
         @paste="handlePaste"
+        @autodetect-fields="handleAutodetectFields"
         @close="closeContextMenu"
       />
       <ContextMenu
@@ -207,9 +209,14 @@ export default {
       type: String,
       required: false,
       default: ''
+    },
+    withFieldsDetection: {
+      type: Boolean,
+      required: false,
+      default: false
     }
   },
-  emits: ['draw', 'drop-field', 'remove-area', 'copy-field', 'paste-field', 'scroll-to', 'copy-selected-areas', 'delete-selected-areas', 'align-selected-areas'],
+  emits: ['draw', 'drop-field', 'remove-area', 'copy-field', 'paste-field', 'scroll-to', 'copy-selected-areas', 'delete-selected-areas', 'align-selected-areas', 'autodetect-fields'],
   data () {
     return {
       areaRefs: [],
@@ -414,6 +421,14 @@ export default {
 
       this.closeContextMenu()
     },
+    handleAutodetectFields () {
+      this.$emit('autodetect-fields', {
+        page: this.number,
+        attachmentUuid: this.attachmentUuid
+      })
+
+      this.closeContextMenu()
+    },
     setAreaRefs (el) {
       if (el) {
         this.areaRefs.push(el)

lib/templates/detect_fields.rb

@@ -60,21 +60,23 @@ module Templates
 
     # rubocop:disable Metrics, Style
     def call(io, attachment: nil, confidence: 0.3, temperature: 1, inference: Templates::ImageToFields, nms: 0.1,
-             nmm: 0.5, split_page: false, aspect_ratio: true, padding: 20, regexp_type: true, &)
+             nmm: 0.5, split_page: false, aspect_ratio: true, padding: 20, regexp_type: true, page_number: nil, &)
       fields, head_node =
         if attachment&.image?
           process_image_attachment(io, attachment:, confidence:, nms:, nmm:, split_page:, inference:,
-                                       temperature:, aspect_ratio:, padding:, &)
+                                       temperature:, aspect_ratio:, padding:, page_number:, &)
         else
           process_pdf_attachment(io, attachment:, confidence:, nms:, nmm:, split_page:, inference:,
-                                     temperature:, aspect_ratio:, regexp_type:, padding:, &)
+                                     temperature:, aspect_ratio:, regexp_type:, padding:, page_number:, &)
         end
 
       [fields, head_node]
     end
 
     def process_image_attachment(io, attachment:, confidence:, nms:, nmm:, temperature:, inference:,
-                                 split_page: false, aspect_ratio: false, padding: nil)
+                                 split_page: false, aspect_ratio: false, padding: nil, page_number: nil)
+      return [[], nil] if page_number && page_number != 0
+
       image = Vips::Image.new_from_buffer(io.read, '')
 
       fields = inference.call(image, confidence:, nms:, nmm:, split_page:,
@@ -105,14 +107,19 @@ module Templates
     end
 
     def process_pdf_attachment(io, attachment:, confidence:, nms:, nmm:, temperature:, inference:,
-                               split_page: false, aspect_ratio: false, padding: nil, regexp_type: false)
+                               split_page: false, aspect_ratio: false, padding: nil, regexp_type: false,
+                               page_number: nil)
       doc = Pdfium::Document.open_bytes(io.read)
 
       head_node = PageNode.new(elem: ''.b, page: 0, attachment_uuid: attachment&.uuid)
       tail_node = head_node
 
-      fields = doc.page_count.times.flat_map do |page_number|
+      page_range = page_number ? [page_number] : (0...doc.page_count)
-        page = doc.get_page(page_number)
+
+      fields = page_range.flat_map do |current_page_number|
+        next [] if current_page_number >= doc.page_count
+
+        page = doc.get_page(current_page_number)
 
         size_key = page.width > page.height ? :width : :height
         size = padding ? inference.resolution * 1.5 : inference.resolution
@@ -149,13 +156,13 @@ module Templates
             areas: [{
               x: field.x, y: field.y,
               w: field.w, h: field.h,
-              page: page_number,
+              page: current_page_number,
               attachment_uuid: attachment&.uuid
             }]
           }
         end
 
-        yield [attachment&.uuid, page_number, fields] if block_given?
+        yield [attachment&.uuid, current_page_number, fields] if block_given?
 
         fields
       ensure