optimize for large files

lib/submissions/ensure_result_generated.rb

@@ -4,7 +4,7 @@ module Submissions
   module EnsureResultGenerated
     WAIT_FOR_RETRY = 2.seconds
     CHECK_EVENT_INTERVAL = 1.second
-    CHECK_COMPLETE_TIMEOUT = 20.seconds
+    CHECK_COMPLETE_TIMEOUT = 90.seconds
 
     WaitForCompleteTimeout = Class.new(StandardError)
 

lib/templates/clone_attachments.rb

@@ -13,13 +13,15 @@ module Templates
           record: template
         )
 
-        document.preview_images_attachments.each do |preview_image|
-          ActiveStorage::Attachment.create!(
-            uuid: preview_image.uuid,
-            blob_id: preview_image.blob_id,
-            name: 'preview_images',
-            record: new_document
-          )
+        ApplicationRecord.no_touching do
+          document.preview_images_attachments.each do |preview_image|
+            ActiveStorage::Attachment.create!(
+              uuid: preview_image.uuid,
+              blob_id: preview_image.blob_id,
+              name: 'preview_images',
+              record: new_document
+            )
+          end
         end
       end
     end

lib/templates/create_attachments.rb

@@ -3,6 +3,7 @@
 module Templates
   module CreateAttachments
     PDF_CONTENT_TYPE = 'application/pdf'
+    ANNOTATIONS_SIZE_LIMIT = 6.megabytes
     InvalidFileType = Class.new(StandardError)
 
     module_function
@@ -18,7 +19,10 @@ module Templates
         document = template.documents.create!(blob:)
 
         if blob.content_type == PDF_CONTENT_TYPE && blob.metadata['pdf'].nil?
-          blob.metadata['pdf'] = { 'annotations' => Templates::BuildAnnotations.call(document_data) }
+          annotations =
+            document_data.size > ANNOTATIONS_SIZE_LIMIT ? [] : Templates::BuildAnnotations.call(document_data)
+
+          blob.metadata['pdf'] = { 'annotations' => annotations }
           blob.metadata['sha256'] = Base64.urlsafe_encode64(Digest::SHA256.digest(document_data))
         end
 
@@ -37,9 +41,10 @@ module Templates
         data = file.read
 
         if file.content_type == PDF_CONTENT_TYPE
+          annotations = data.size > ANNOTATIONS_SIZE_LIMIT ? [] : Templates::BuildAnnotations.call(data)
           metadata = { 'identified' => true, 'analyzed' => true,
                        'sha256' => Base64.urlsafe_encode64(Digest::SHA256.digest(data)),
-                       'pdf' => { 'annotations' => Templates::BuildAnnotations.call(data) } }
+                       'pdf' => { 'annotations' => annotations } }
         end
 
         ActiveStorage::Blob.create_and_upload!(

lib/templates/process_document.rb

@@ -9,6 +9,12 @@ module Templates
     PDF_CONTENT_TYPE = 'application/pdf'
     Q = 35
     MAX_WIDTH = 1400
+    MAX_NUMBER_OF_PAGES_PROCESSED =
+      if Docuseal.multitenant?
+        70
+      else
+        40
+      end
 
     module_function
 
@@ -42,14 +48,38 @@ module Templates
 
     def generate_pdf_preview_images(attachment, data)
       ActiveStorage::Attachment.where(name: ATTACHMENT_NAME, record: attachment).destroy_all
-      number_of_pages = HexaPDF::Document.new(io: StringIO.new(data)).pages.size - 1
+      number_of_pages = HexaPDF::Document.new(io: StringIO.new(data)).pages.size
 
-      (0..number_of_pages).each do |page_number|
+      (attachment.metadata['pdf'] ||= {})[:number_of_pages] = number_of_pages
+
+      attachment.save!
+
+      (0..[number_of_pages - 1, MAX_NUMBER_OF_PAGES_PROCESSED].min).each do |page_number|
         page = Vips::Image.new_from_buffer(data, '', dpi: DPI, page: page_number)
         page = page.resize(MAX_WIDTH / page.width.to_f)
 
         io = StringIO.new(page.write_to_buffer(FORMAT, Q: Q, interlace: true))
 
+        ApplicationRecord.no_touching do
+          ActiveStorage::Attachment.create!(
+            blob: ActiveStorage::Blob.create_and_upload!(
+              io:, filename: "#{page_number}#{FORMAT}",
+              metadata: { analyzed: true, identified: true, width: page.width, height: page.height }
+            ),
+            name: ATTACHMENT_NAME,
+            record: attachment
+          )
+        end
+      end
+    end
+
+    def generate_pdf_preview_from_file(attachment, file_path, page_number)
+      page = Vips::Image.new_from_file(file_path, dpi: DPI, page: page_number)
+      page = page.resize(MAX_WIDTH / page.width.to_f)
+
+      io = StringIO.new(page.write_to_buffer(FORMAT, Q: Q, interlace: true))
+
+      ApplicationRecord.no_touching do
         ActiveStorage::Attachment.create!(
           blob: ActiveStorage::Blob.create_and_upload!(
             io:, filename: "#{page_number}#{FORMAT}",
@@ -59,6 +89,8 @@ module Templates
           record: attachment
         )
       end
+
+      io
     end
   end
 end