Merge from docusealco/wip

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Install Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 4.0.1
+          ruby-version: 4.0.5
       - name: Cache gems
         uses: actions/cache@v4
         with:
@@ -37,7 +37,7 @@ jobs:
       - name: Install Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 4.0.1
+          ruby-version: 4.0.5
       - name: Cache gems
         uses: actions/cache@v4
         with:
@@ -89,7 +89,7 @@ jobs:
       - name: Install Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 4.0.1
+          ruby-version: 4.0.5
       - name: Cache gems
         uses: actions/cache@v4
         with:
@@ -132,7 +132,7 @@ jobs:
       - name: Install Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 4.0.1
+          ruby-version: 4.0.5
       - name: Set up Node
         uses: actions/setup-node@v1
         with:
@@ -163,7 +163,7 @@ jobs:
           yarn install
           sudo apt-get update
           sudo apt-get install -y libvips
-          wget -O pdfium-linux.tgz "https://github.com/docusealco/pdfium-binaries/releases/latest/download/pdfium-linux-$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/').tgz"
+          wget -O pdfium-linux.tgz "https://github.com/bblanchon/pdfium-binaries/releases/latest/download/pdfium-linux-$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/').tgz"
           sudo tar -xzf pdfium-linux.tgz --strip-components=1 -C /usr/lib lib/libpdfium.so
           rm -f pdfium-linux.tgz
       - name: Run

Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:4.0.1-alpine AS download
+FROM ruby:4.0.5-alpine AS download
 
 WORKDIR /fonts
 
@@ -13,7 +13,7 @@ RUN apk --no-cache add wget && \
     mkdir -p /pdfium-linux && \
     tar -xzf pdfium-linux.tgz -C /pdfium-linux
 
-FROM ruby:4.0.1-alpine AS webpack
+FROM ruby:4.0.5-alpine AS webpack
 
 ENV RAILS_ENV=production
 ENV NODE_ENV=production
@@ -40,7 +40,7 @@ COPY ./app/views ./app/views
 
 RUN echo "gem 'shakapacker'" > Gemfile && ./bin/shakapacker
 
-FROM ruby:4.0.1-alpine AS app
+FROM ruby:4.0.5-alpine AS app
 
 ENV RAILS_ENV=production
 ENV BUNDLE_WITHOUT="development:test"
@@ -48,7 +48,7 @@ ENV OPENSSL_CONF=/etc/openssl_legacy.cnf
 
 WORKDIR /app
 
-RUN apk add --no-cache libpq vips redis vips-heif onnxruntime
+RUN apk add --no-cache libpq vips redis onnxruntime
 
 RUN addgroup -g 2000 docuseal && adduser -u 2000 -G docuseal -s /bin/sh -D -h /home/docuseal docuseal
 
@@ -94,6 +94,7 @@ WORKDIR /data/docuseal
 ENV HOME=/home/docuseal
 ENV WORKDIR=/data/docuseal
 ENV VIPS_MAX_COORD=17000
+ENV VIPS_BLOCK_UNTRUSTED=1
 
 EXPOSE 3000
 CMD ["/app/bin/bundle", "exec", "puma", "-C", "/app/config/puma.rb", "--dir", "/app"]

Gemfile

@@ -2,7 +2,7 @@
 
 source 'https://rubygems.org'
 
-ruby '4.0.1'
+ruby '4.0.5'
 
 gem 'addressable'
 gem 'arabic-letter-connector', require: false

Gemfile.lock

@@ -195,7 +195,7 @@ GEM
       railties (>= 6.1.0)
     faker (3.6.1)
       i18n (>= 1.8.11, < 2)
-    faraday (2.14.1)
+    faraday (2.14.2)
       faraday-net_http (>= 2.0, < 3.5)
       json
       logger
@@ -275,7 +275,7 @@ GEM
       reline (>= 0.4.2)
     jmespath (1.6.2)
     json (2.19.5)
-    jwt (3.1.2)
+    jwt (3.2.0)
       base64
     language_server-protocol (3.17.0.5)
     launchy (3.1.1)
@@ -662,7 +662,7 @@ DEPENDENCIES
   webmock
 
 RUBY VERSION
-  ruby 4.0.1
+  ruby 4.0.5
 
 BUNDLED WITH
   4.0.3

app/controllers/api/active_storage_blobs_proxy_controller.rb

@@ -9,7 +9,9 @@ module Api
 
     before_action :set_cors_headers
     before_action :set_noindex_headers
+    before_action :set_security_headers
 
+    # rubocop:disable Metrics
     def show
       blob_uuid, purp, exp = ApplicationRecord.signed_id_verifier.verified(params[:signed_uuid])
 
@@ -21,6 +23,12 @@ module Api
 
       blob = ActiveStorage::Blob.find_by!(uuid: blob_uuid)
 
+      if Submitters::DANGEROUS_EXTENSIONS.include?(blob.filename.extension.to_s.downcase)
+        Rollbar.error('Dangerous extension') if defined?(Rollbar)
+
+        return head :unprocessable_content
+      end
+
       attachment = blob.attachments.take
 
       @record = attachment.record
@@ -45,6 +53,7 @@ module Api
         end
       end
     end
+    # rubocop:enable Metrics
 
     private
 

app/controllers/api/active_storage_blobs_proxy_legacy_controller.rb

@@ -9,6 +9,7 @@ module Api
 
     before_action :set_cors_headers
     before_action :set_noindex_headers
+    before_action :set_security_headers
 
     # rubocop:disable Metrics
     def show
@@ -18,6 +19,12 @@ module Api
 
       return head :not_found unless blob
 
+      if Submitters::DANGEROUS_EXTENSIONS.include?(blob.filename.extension.to_s.downcase)
+        Rollbar.error('Dangerous extension') if defined?(Rollbar)
+
+        return head :unprocessable_content
+      end
+
       is_permitted = blob.attachments.any? do |a|
         (current_user && a.record.account.id == current_user.account_id) ||
           a.record.account.account_configs.any? { |e| e.key == 'legacy_blob_proxy' } ||

app/controllers/api/api_base_controller.rb

@@ -102,6 +102,10 @@ module Api
       headers['X-Robots-Tag'] = 'noindex'
     end
 
+    def set_security_headers
+      response.headers['X-Content-Type-Options'] = 'nosniff'
+    end
+
     def set_cors_headers
       headers['Access-Control-Allow-Origin'] = '*'
       headers['Access-Control-Allow-Methods'] = 'POST, GET, PUT, PATCH, DELETE, OPTIONS'

app/controllers/api/attachments_controller.rb

@@ -16,8 +16,10 @@ module Api
         return render json: { error: I18n.t('form_has_been_archived') }, status: :unprocessable_content
       end
 
+      file = params[:file]
+
       if params[:type].in?(%w[initials signature])
-        image = Vips::Image.new_from_file(params[:file].path)
+        image = ImageUtils.load_vips(file.read, content_type: file.content_type)
 
         if ImageUtils.blank?(image)
           Rollbar.error("Empty signature: #{@submitter.id}") if defined?(Rollbar)
@@ -33,7 +35,7 @@ module Api
         end
       end
 
-      attachment = Submitters.create_attachment!(@submitter, params)
+      attachment = Submitters.create_attachment!(@submitter, file)
 
       if params[:remember_signature] == 'true' && @submitter.email.present?
         cookies.encrypted[:signature_uuids] = build_new_cookie_signatures_json(@submitter, attachment)

app/controllers/user_initials_controller.rb

@@ -11,6 +11,12 @@ class UserInitialsController < ApplicationController
 
     return redirect_to settings_profile_index_path, notice: I18n.t('unable_to_save_initials') if file.blank?
 
+    extension = File.extname(file.original_filename).delete_prefix('.').downcase
+
+    if Submitters::DANGEROUS_EXTENSIONS.include?(extension)
+      raise Submitters::MaliciousFileExtension, "File type '.#{extension}' is not allowed."
+    end
+
     blob = ActiveStorage::Blob.create_and_upload!(io: file.open,
                                                   filename: file.original_filename,
                                                   content_type: file.content_type)

app/controllers/user_signatures_controller.rb

@@ -11,6 +11,12 @@ class UserSignaturesController < ApplicationController
 
     return redirect_to settings_profile_index_path, notice: I18n.t('unable_to_save_signature') if file.blank?
 
+    extension = File.extname(file.original_filename).delete_prefix('.').downcase
+
+    if Submitters::DANGEROUS_EXTENSIONS.include?(extension)
+      raise Submitters::MaliciousFileExtension, "File type '.#{extension}' is not allowed."
+    end
+
     blob = ActiveStorage::Blob.create_and_upload!(io: file.open,
                                                   filename: file.original_filename,
                                                   content_type: file.content_type)

app/javascript/application.js

@@ -21,6 +21,7 @@ import SubmittersAutocomplete from './elements/submitter_autocomplete'
 import FolderAutocomplete from './elements/folder_autocomplete'
 import SignatureForm from './elements/signature_form'
 import SubmitForm from './elements/submit_form'
+import ConvertUpload from './elements/convert_upload'
 import PromptPassword from './elements/prompt_password'
 import EmailsTextarea from './elements/emails_textarea'
 import ToggleSubmit from './elements/toggle_submit'
@@ -111,6 +112,7 @@ safeRegisterElement('submitters-autocomplete', SubmittersAutocomplete)
 safeRegisterElement('folder-autocomplete', FolderAutocomplete)
 safeRegisterElement('signature-form', SignatureForm)
 safeRegisterElement('submit-form', SubmitForm)
+safeRegisterElement('convert-upload', ConvertUpload)
 safeRegisterElement('prompt-password', PromptPassword)
 safeRegisterElement('emails-textarea', EmailsTextarea)
 safeRegisterElement('toggle-cookies', ToggleCookies)

app/javascript/elements/convert_upload.js

@@ -0,0 +1,73 @@
+export function convertImage (sourceFile, targetType, quality) {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader()
+
+    reader.onload = function (event) {
+      const img = new Image()
+
+      img.onload = function () {
+        const canvas = document.createElement('canvas')
+        const ctx = canvas.getContext('2d')
+
+        canvas.width = img.width
+        canvas.height = img.height
+        ctx.drawImage(img, 0, 0)
+        canvas.toBlob(function (blob) {
+          const ext = targetType === 'image/jpeg' ? '.jpg' : '.png'
+          const newFile = new File([blob], sourceFile.name.replace(/\.\w+$/, ext), { type: targetType })
+          resolve(newFile)
+        }, targetType, quality)
+      }
+
+      img.onerror = () => reject(new Error(`browser cannot decode ${sourceFile.type || sourceFile.name}`))
+
+      img.src = event.target.result
+    }
+    reader.onerror = reject
+    reader.readAsDataURL(sourceFile)
+  })
+}
+
+export async function convertImagesInInput (input) {
+  if (!input.files || input.files.length === 0) return
+
+  const dt = new DataTransfer()
+  let didConvert = false
+
+  for (const file of Array.from(input.files)) {
+    let converted = file
+
+    try {
+      if (['image/bmp', 'image/vnd.microsoft.icon', 'image/svg+xml', 'image/gif'].includes(file.type)) {
+        converted = await convertImage(file, 'image/png')
+        didConvert = true
+      } else if (['image/heic', 'image/heif', 'image/heic-sequence', 'image/heif-sequence', 'image/avif', 'image/avif-sequence', 'image/webp'].includes(file.type)) {
+        converted = await convertImage(file, 'image/jpeg', 0.9)
+        didConvert = true
+      }
+    } catch (e) {
+      alert(e.message)
+    }
+
+    dt.items.add(converted)
+  }
+
+  if (didConvert) {
+    input.files = dt.files
+  }
+}
+
+export default class extends HTMLElement {
+  connectedCallback () {
+    const input = this.querySelector('input[type="file"]')
+    const form = input.form
+
+    input.addEventListener('change', async () => {
+      await convertImagesInInput(input)
+
+      form.querySelector('[type="submit"]')?.setAttribute('disabled', true)
+
+      form.requestSubmit()
+    })
+  }
+}

app/javascript/elements/dashboard_dropzone.js

@@ -1,4 +1,5 @@
 import { target, targets, targetable } from '@github/catalyst/lib/targetable'
+import { convertImagesInInput } from './convert_upload'
 
 const loadingIconHtml = `<svg xmlns="http://www.w3.org/2000/svg" class="animate-spin" width="44" height="44" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
   <path stroke="none" d="M0 0h24v24H0z" fill="none" />
@@ -150,12 +151,16 @@ export default targetable(class extends HTMLElement {
     if (!this.isLoading) this.hideDraghover()
   }
 
-  uploadFiles (files, url) {
+  async uploadFiles (files, url) {
     this.isLoading = true
 
     this.form.action = url
 
-    this.form.querySelector('[type="file"]').files = files
+    const input = this.form.querySelector('[type="file"]')
+
+    input.files = files
+
+    await convertImagesInInput(input)
 
     this.form.querySelector('[type="submit"]').click()
   }

app/javascript/elements/file_dropzone.js

@@ -1,5 +1,6 @@
 import { actionable } from '@github/catalyst/lib/actionable'
 import { target, targetable } from '@github/catalyst/lib/targetable'
+import { convertImagesInInput } from './convert_upload'
 
 export default actionable(targetable(class extends HTMLElement {
   static [target.static] = [
@@ -38,17 +39,21 @@ export default actionable(targetable(class extends HTMLElement {
     this.classList.add('border-base-300', 'hover:bg-base-200/30')
   }
 
-  onDrop (e) {
+  async onDrop (e) {
     e.preventDefault()
 
     this.input.files = e.dataTransfer.files
 
-    this.uploadFiles(e.dataTransfer.files)
+    await convertImagesInInput(this.input)
+
+    this.uploadFiles(this.input.files)
   }
 
-  onSelectFiles (e) {
+  async onSelectFiles (e) {
     e.preventDefault()
 
+    await convertImagesInInput(this.input)
+
     this.uploadFiles(this.input.files)
   }
 

app/javascript/submission_form/dropzone.vue

@@ -51,6 +51,36 @@
 <script>
 import { IconCloudUpload, IconInnerShadowTop } from '@tabler/icons-vue'
 
+function convertImage (sourceFile, targetType, quality) {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader()
+
+    reader.onload = function (event) {
+      const img = new Image()
+
+      img.onload = function () {
+        const canvas = document.createElement('canvas')
+        const ctx = canvas.getContext('2d')
+
+        canvas.width = img.width
+        canvas.height = img.height
+        ctx.drawImage(img, 0, 0)
+        canvas.toBlob(function (blob) {
+          const ext = targetType === 'image/jpeg' ? '.jpg' : '.png'
+          const newFile = new File([blob], sourceFile.name.replace(/\.\w+$/, ext), { type: targetType })
+          resolve(newFile)
+        }, targetType, quality)
+      }
+
+      img.onerror = () => reject(new Error(`browser cannot decode ${sourceFile.type || sourceFile.name}`))
+
+      img.src = event.target.result
+    }
+    reader.onerror = reject
+    reader.readAsDataURL(sourceFile)
+  })
+}
+
 export default {
   name: 'FileDropzone',
   components: {
@@ -153,8 +183,14 @@ export default {
               }
             })
           } else {
-            if (file.type === 'image/bmp' || file.type === 'image/vnd.microsoft.icon') {
+            try {
-              file = await this.convertBmpToPng(file)
+              if (['image/bmp', 'image/vnd.microsoft.icon', 'image/svg+xml', 'image/gif'].includes(file.type)) {
+                file = await convertImage(file, 'image/png')
+              } else if (['image/heic', 'image/heif', 'image/heic-sequence', 'image/heif-sequence', 'image/avif', 'image/avif-sequence', 'image/webp'].includes(file.type)) {
+                file = await convertImage(file, 'image/jpeg', 0.9)
+              }
+            } catch (e) {
+              alert(e.message)
             }
 
             formData.append('file', file)
@@ -181,32 +217,6 @@ export default {
       }).finally(() => {
         this.isLoading = false
       })
-    },
-    convertBmpToPng (bmpFile) {
-      return new Promise((resolve, reject) => {
-        const reader = new FileReader()
-
-        reader.onload = function (event) {
-          const img = new Image()
-
-          img.onload = function () {
-            const canvas = document.createElement('canvas')
-            const ctx = canvas.getContext('2d')
-
-            canvas.width = img.width
-            canvas.height = img.height
-            ctx.drawImage(img, 0, 0)
-            canvas.toBlob(function (blob) {
-              const newFile = new File([blob], bmpFile.name.replace(/\.\w+$/, '.png'), { type: 'image/png' })
-              resolve(newFile)
-            }, 'image/png')
-          }
-
-          img.src = event.target.result
-        }
-        reader.onerror = reject
-        reader.readAsDataURL(bmpFile)
-      })
     }
   }
 }

app/javascript/template_builder/dynamic_editor.js

@@ -94,6 +94,8 @@ img.ProseMirror-separator {
 }
 dynamic-variable {
   background-color: #fef3c7;
+  word-break: break-all;
+  overflow-wrap: anywhere;
 }`)
 
 function collectDomAttrs (dom) {
@@ -136,11 +138,12 @@ function collectSpanDomAttrs (dom) {
   return result
 }
 
-function createBlockNode (name, tag, content) {
+function createBlockNode (name, tag, content, extra = {}) {
   return Node.create({
     name,
     group: 'block',
     content: content || 'block+',
+    ...extra,
     addAttributes () {
       return {
         htmlAttrs: { default: {} }
@@ -194,9 +197,9 @@ const CustomHeading = Node.create({
 })
 
 const SectionNode = createBlockNode('section', 'section')
-const ArticleNode = createBlockNode('article', 'article')
+const ArticleNode = createBlockNode('article', 'article', null, { isolating: true })
-const HeaderNode = createBlockNode('header', 'header')
+const HeaderNode = createBlockNode('header', 'header', null, { isolating: true })
-const FooterNode = createBlockNode('footer', 'footer')
+const FooterNode = createBlockNode('footer', 'footer', null, { isolating: true })
 const DivNode = createBlockNode('div', 'div')
 const BlockquoteNode = createBlockNode('blockquote', 'blockquote')
 const PreNode = createBlockNode('pre', 'pre')

app/javascript/template_builder/upload.vue

@@ -168,6 +168,65 @@
 <script>
 import { IconUpload, IconInnerShadowTop, IconChevronDown, IconBrandGoogleDrive } from '@tabler/icons-vue'
 
+function convertImage (sourceFile, targetType, quality) {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader()
+
+    reader.onload = function (event) {
+      const img = new Image()
+
+      img.onload = function () {
+        const canvas = document.createElement('canvas')
+        const ctx = canvas.getContext('2d')
+
+        canvas.width = img.width
+        canvas.height = img.height
+        ctx.drawImage(img, 0, 0)
+        canvas.toBlob(function (blob) {
+          const ext = targetType === 'image/jpeg' ? '.jpg' : '.png'
+          const newFile = new File([blob], sourceFile.name.replace(/\.\w+$/, ext), { type: targetType })
+          resolve(newFile)
+        }, targetType, quality)
+      }
+
+      img.onerror = () => reject(new Error(`browser cannot decode ${sourceFile.type || sourceFile.name}`))
+
+      img.src = event.target.result
+    }
+    reader.onerror = reject
+    reader.readAsDataURL(sourceFile)
+  })
+}
+
+async function convertImagesInInput (input) {
+  if (!input.files || input.files.length === 0) return
+
+  const dt = new DataTransfer()
+  let didConvert = false
+
+  for (const file of Array.from(input.files)) {
+    let converted = file
+
+    try {
+      if (['image/bmp', 'image/vnd.microsoft.icon', 'image/svg+xml', 'image/gif'].includes(file.type)) {
+        converted = await convertImage(file, 'image/png')
+        didConvert = true
+      } else if (['image/heic', 'image/heif', 'image/heic-sequence', 'image/heif-sequence', 'image/avif', 'image/avif-sequence', 'image/webp'].includes(file.type)) {
+        converted = await convertImage(file, 'image/jpeg', 0.9)
+        didConvert = true
+      }
+    } catch (e) {
+      alert(e.message)
+    }
+
+    dt.items.add(converted)
+  }
+
+  if (didConvert) {
+    input.files = dt.files
+  }
+}
+
 export default {
   name: 'DocumentsUpload',
   components: {
@@ -282,6 +341,10 @@ export default {
     async upload ({ path } = {}) {
       this.isLoading = true
 
+      if (this.$refs.input) {
+        await convertImagesInInput(this.$refs.input)
+      }
+
       return this.baseFetch(path || this.uploadUrl, {
         method: 'POST',
         headers: { Accept: 'application/json' },

app/jobs/reindex_search_entry_job.rb

@@ -3,8 +3,6 @@
 class ReindexSearchEntryJob
   include Sidekiq::Job
 
-  InvalidFormat = Class.new(StandardError)
-
   def perform(params = {})
     entry = SearchEntry.find_or_initialize_by(params.slice('record_type', 'record_id'))
 

app/views/templates/_upload_button.html.erb

@@ -15,8 +15,8 @@
     </span>
   </label>
   <input type="hidden" name="form_id" value="<%= form_id %>">
-  <submit-form data-on="change" data-disable="true">
+  <convert-upload>
     <input id="upload_template" name="files[]" class="hidden" type="file" accept="image/*, application/pdf, application/zip, application/json<%= ", #{Templates::CreateAttachments::DOCUMENT_EXTENSIONS.join(', ')}" if Docuseal.advanced_formats? %>" multiple>
-  </submit-form>
+  </convert-upload>
   <input hidden name="folder_name" value="<%= local_assigns[:folder_name] %>">
 <% end %>

config/application.rb

@@ -25,6 +25,14 @@ module DocuSeal
 
     config.active_storage.draw_routes = ENV['MULTITENANT'] != 'true'
 
+    config.active_storage.content_types_to_serve_as_binary += %w[
+      application/javascript
+      text/javascript
+      application/ecmascript
+      text/ecmascript
+      application/wasm
+    ]
+
     config.i18n.available_locales = %i[en en-US en-GB es-ES fr-FR pt-PT de-DE it-IT nl-NL
                                        es it de fr nl pl uk cs pt he ar ko ja]
     config.i18n.fallbacks = [:en]

config/initializers/marcel.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+priority = %w[application/pdf image/jpeg image/png]
+
+indexes = Marcel::MAGIC.each_with_index.with_object({}) do |((type, _), i), acc|
+  acc[type] = i if priority.include?(type)
+
+  break acc if acc.size == priority.size
+end
+
+pdf_index, jpg_index, png_index = indexes.values_at(*priority)
+
+Marcel::MAGIC[0], Marcel::MAGIC[pdf_index] = Marcel::MAGIC[pdf_index], Marcel::MAGIC[0]
+Marcel::MAGIC[1], Marcel::MAGIC[jpg_index] = Marcel::MAGIC[jpg_index], Marcel::MAGIC[1]
+Marcel::MAGIC[2], Marcel::MAGIC[png_index] = Marcel::MAGIC[png_index], Marcel::MAGIC[2]

lib/image_utils.rb

@@ -1,8 +1,25 @@
 # frozen_string_literal: true
 
 module ImageUtils
+  ICO_REGEXP = %r{\Aimage/(?:x-icon|vnd\.microsoft\.icon)\z}
+  BMP_REGEXP = %r{\Aimage/(?:bmp|x-bmp|x-ms-bmp)\z}
+
   module_function
 
+  def load_vips(data, content_type: nil, autorot: false)
+    content_type ||= Marcel::MimeType.for(data)
+
+    if ICO_REGEXP.match?(content_type)
+      LoadIco.call(data)
+    elsif BMP_REGEXP.match?(content_type)
+      LoadBmp.call(data)
+    else
+      image = Vips::Image.new_from_buffer(data, '')
+
+      autorot ? image.autorot : image
+    end
+  end
+
   def blank?(image)
     stats = image.stats
 

lib/submissions/generate_audit_trail.rb

@@ -363,7 +363,7 @@ module Submissions
 
               image =
                 begin
-                  Submissions::GenerateResultAttachments.load_vips_image(attachment).autorot
+                  ImageUtils.load_vips(attachment.download, content_type: attachment.content_type, autorot: true)
                 rescue Vips::Error
                   next unless attachment.content_type.starts_with?('image/')
                   next if attachment.byte_size.zero?
@@ -379,7 +379,7 @@ module Submissions
                 if field['type'] == 'image' && !resized_image.has_alpha?
                   StringIO.new(resized_image.colourspace(:srgb).write_to_buffer('.jpg', strip: true))
                 else
-                  StringIO.new(resized_image.write_to_buffer('.png'))
+                  StringIO.new(resized_image.write_to_buffer('.png', strip: true))
                 end
 
               width = field['type'] == 'initials' ? 50 : 200

lib/submissions/generate_result_attachments.rb

@@ -11,9 +11,6 @@ module Submissions
                   'Helvetica'
                 end
 
-    ICO_REGEXP = %r{\Aimage/(?:x-icon|vnd\.microsoft\.icon)\z}
-    BMP_REGEXP = %r{\Aimage/(?:bmp|x-bmp|x-ms-bmp)\z}
-
     FONT_BOLD_NAME = if File.exist?(FONT_BOLD_PATH)
                        FONT_BOLD_PATH
                      else
@@ -313,7 +310,10 @@ module Submissions
 
             image =
               begin
-                load_vips_image(attachment, attachments_data_cache).autorot
+                attachments_data_cache[attachment.uuid] ||= attachment.download
+
+                ImageUtils.load_vips(attachments_data_cache[attachment.uuid],
+                                     content_type: attachment.content_type, autorot: true)
               rescue Vips::Error
                 next unless attachment.content_type.starts_with?('image/')
                 next if attachment.byte_size.zero?
@@ -358,7 +358,8 @@ module Submissions
               image_x = area_x + ((half_width - image_width) / 2.0)
               image_y = height - area_y - image_height
 
-              io = StringIO.new(image.resize([scale * 4, 1].select(&:positive?).min).write_to_buffer('.png'))
+              io =
+                StringIO.new(image.resize([scale * 4, 1].select(&:positive?).min).write_to_buffer('.png', strip: true))
 
               canvas.image(io, at: [image_x, image_y], width: image_width, height: image_height)
 
@@ -425,7 +426,8 @@ module Submissions
 
               scale = [area_w / image.width, image_height / image.height].min
 
-              io = StringIO.new(image.resize([scale * 4, 1].select(&:positive?).min).write_to_buffer('.png'))
+              io =
+                StringIO.new(image.resize([scale * 4, 1].select(&:positive?).min).write_to_buffer('.png', strip: true))
 
               layouter.fit([text], area_w, base_font_size / 0.65)
                       .draw(canvas, area_x + TEXT_LEFT_MARGIN,
@@ -451,7 +453,10 @@ module Submissions
 
             image =
               begin
-                load_vips_image(attachment, attachments_data_cache).autorot
+                attachments_data_cache[attachment.uuid] ||= attachment.download
+
+                ImageUtils.load_vips(attachments_data_cache[attachment.uuid],
+                                     content_type: attachment.content_type, autorot: true)
               rescue Vips::Error
                 next unless attachment.content_type.starts_with?('image/')
                 next if attachment.byte_size.zero?
@@ -468,7 +473,7 @@ module Submissions
               if field_type == 'image' && !resized_image.has_alpha?
                 StringIO.new(resized_image.colourspace(:srgb).write_to_buffer('.jpg', strip: true))
               else
-                StringIO.new(resized_image.write_to_buffer('.png'))
+                StringIO.new(resized_image.write_to_buffer('.png', strip: true))
               end
 
             canvas.image(
@@ -1021,20 +1026,6 @@ module Submissions
       []
     end
 
-    def load_vips_image(attachment, cache = {})
-      cache[attachment.uuid] ||= attachment.download
-
-      data = cache[attachment.uuid]
-
-      if ICO_REGEXP.match?(attachment.content_type)
-        LoadIco.call(data)
-      elsif BMP_REGEXP.match?(attachment.content_type)
-        LoadBmp.call(data)
-      else
-        Vips::Image.new_from_buffer(data, '')
-      end
-    end
-
     def r
       Rails.application.routes.url_helpers
     end

lib/submitters.rb

@@ -122,27 +122,20 @@ module Submitters
     end
   end
 
-  def create_attachment!(submitter, params)
+  def create_attachment!(submitter, file)
-    blob =
+    raise ParamsError, 'file param is missing' if file.blank?
-      if (file = params[:file])
-        extension = File.extname(file.original_filename).delete_prefix('.').downcase
 
-        if DANGEROUS_EXTENSIONS.include?(extension)
+    extension = File.extname(file.original_filename).delete_prefix('.').downcase
-          raise MaliciousFileExtension, "File type '.#{extension}' is not allowed."
-        end
 
-        ActiveStorage::Blob.create_and_upload!(io: file.open,
+    if DANGEROUS_EXTENSIONS.include?(extension)
-                                               filename: file.original_filename,
+      raise MaliciousFileExtension, "File type '.#{extension}' is not allowed."
-                                               content_type: file.content_type)
+    end
-      else
-        raise ParamsError, 'file param is missing'
-      end
 
-    ActiveStorage::Attachment.create!(
+    blob = ActiveStorage::Blob.create_and_upload!(io: file.tap(&:rewind).open,
-      blob:,
+                                                  filename: file.original_filename,
-      name: 'attachments',
+                                                  content_type: file.content_type)
-      record: submitter
+
-    )
+    ActiveStorage::Attachment.create!(blob:, name: 'attachments', record: submitter)
   end
 
   def normalize_preferences(account, user, params)

lib/submitters/create_stamp_attachment.rb

@@ -25,7 +25,7 @@ module Submitters
     def build_attachment(submitter, with_logo: true)
       image = generate_stamp_image(submitter, with_logo:)
 
-      image_data = image.write_to_buffer('.png')
+      image_data = image.write_to_buffer('.png', strip: true)
 
       checksum = Digest::MD5.base64digest(image_data)
 
@@ -40,7 +40,7 @@ module Submitters
     def generate_stamp_image(submitter, with_logo: true)
       logo =
         if with_logo
-          Vips::Image.new_from_buffer(load_logo(submitter).read, '')
+          ImageUtils.load_vips(load_logo(submitter).read)
         else
           Vips::Image.new_from_buffer(TRANSPARENT_PIXEL, '').resize(WIDTH)
         end

lib/submitters/normalize_values.rb

@@ -212,8 +212,8 @@ module Submitters
         elsif type.in?(%w[signature initials]) && value.length < 60
           find_or_create_blob_from_text(account, value, type)
         elsif (data = Base64.decode64(value.sub(BASE64_PREFIX_REGEXP, ''))) &&
-              Marcel::MimeType.for(data).exclude?('octet-stream')
+              (mime_type = Marcel::MimeType.for(data)).exclude?('octet-stream')
-          find_or_create_blob_from_base64(account, data, type)
+          find_or_create_blob_from_base64(account, data, type, mime_type:)
         elsif type == 'image' && (value.starts_with?('<html>') || value.starts_with?('<!DOCTYPE'))
           raise InvalidDefaultValue, "Invalid #{type} value" unless purpose == :api
 
@@ -236,15 +236,27 @@ module Submitters
       raise InvalidDefaultValue, "HTML content is not allowed: #{value.first(200)}..."
     end
 
-    def find_or_create_blob_from_base64(account, data, type)
+    def find_or_create_blob_from_base64(account, data, type, mime_type: nil)
       checksum = Digest::MD5.base64digest(data)
 
       blob = find_blob_by_checksum(checksum, account)
 
-      blob || ActiveStorage::Blob.create_and_upload!(
+      return blob if blob
-        io: StringIO.new(data),
+
-        filename: "#{type}.png"
+      mime_type ||= Marcel::MimeType.for(data)
-      )
+
+      detected_extensions = Marcel::TYPE_EXTS[mime_type].to_a.map(&:downcase)
+
+      if detected_extensions.any? { |e| Submitters::DANGEROUS_EXTENSIONS.include?(e) }
+        raise InvalidDefaultValue, "File type '.#{detected_extensions.first}' is not allowed."
+      end
+
+      extension = detected_extensions.first
+      extension = 'png' if extension.blank? && type.in?(%w[signature initials stamp image])
+
+      filename = extension.present? ? "#{type}.#{extension}" : type
+
+      ActiveStorage::Blob.create_and_upload!(io: StringIO.new(data), filename:)
     end
 
     def find_or_create_blob_from_text(account, text, type)
@@ -261,6 +273,13 @@ module Submitters
     end
 
     def find_or_create_blob_from_url(account, url)
+      filename = Addressable::URI.parse(url).path.split('/').last.to_s
+      extension = File.extname(filename).delete_prefix('.').downcase
+
+      if Submitters::DANGEROUS_EXTENSIONS.include?(extension)
+        raise InvalidDefaultValue, "File type '.#{extension}' is not allowed."
+      end
+
       cache_key = [account.id, url].join(':')
       checksum = CHECKSUM_CACHE_STORE.fetch(cache_key)
 
@@ -276,10 +295,7 @@ module Submitters
 
       blob = find_blob_by_checksum(checksum, account)
 
-      blob || ActiveStorage::Blob.create_and_upload!(
+      blob || ActiveStorage::Blob.create_and_upload!(io: StringIO.new(data), filename:)
-        io: StringIO.new(data),
-        filename: Addressable::URI.parse(url).path.split('/').last
-      )
     end
 
     def find_blob_by_checksum(checksum, account)

lib/templates/detect_fields.rb

@@ -77,7 +77,7 @@ module Templates
                                  split_page: false, aspect_ratio: false, padding: nil, page_number: nil)
       return [[], nil] if page_number && page_number != 0
 
-      image = Vips::Image.new_from_buffer(io.read, '')
+      image = ImageUtils.load_vips(io.read, content_type: attachment.content_type)
 
       fields = inference.call(image, confidence:, nms:, nmm:, split_page:,
                                      temperature:, aspect_ratio:, padding:)

lib/templates/process_document.rb

@@ -7,7 +7,6 @@ module Templates
     PREVIEW_FORMAT = '.jpg'
     ATTACHMENT_NAME = 'preview_images'
 
-    BMP_REGEXP = %r{\Aimage/(?:bmp|x-bmp|x-ms-bmp)\z}
     PDF_CONTENT_TYPE = 'application/pdf'
     CONCURRENCY = 2
     Q = 95
@@ -59,19 +58,13 @@ module Templates
     def generate_preview_image(attachment, data)
       ActiveStorage::Attachment.where(name: ATTACHMENT_NAME, record: attachment).destroy_all
 
-      image =
+      image = ImageUtils.load_vips(data, content_type: attachment.content_type, autorot: true)
-        if BMP_REGEXP.match?(attachment.content_type)
+      image = image.resize(MAX_WIDTH / image.width.to_f)
-          LoadBmp.call(data)
-        else
-          Vips::Image.new_from_buffer(data, '')
-        end
-
-      image = image.autorot.resize(MAX_WIDTH / image.width.to_f)
 
       bitdepth = 2**image.stats.to_a[1..3].pluck(2).uniq.size
 
       io = StringIO.new(image.write_to_buffer(FORMAT, compression: 6, filter: 0, bitdepth:,
-                                                      palette: true, Q: Q, dither: 0))
+                                                      palette: true, Q: Q, dither: 0, strip: true))
 
       ActiveStorage::Attachment.create!(
         blob: ActiveStorage::Blob.create_and_upload!(
@@ -110,7 +103,15 @@ module Templates
 
       promises =
         range.map do |page_number|
-          Concurrent::Promise.execute(executor: pool) { build_and_upload_blob(doc, page_number) }
+          doc_page = doc.get_page(page_number)
+
+          bytes, width, height = doc_page.render_to_bitmap(width: MAX_WIDTH)
+
+          image = Vips::Image.new_from_memory(bytes, width, height, 4, :uchar)
+
+          Concurrent::Promise.execute(executor: pool) { build_and_upload_blob(image, page_number) }
+        ensure
+          doc_page&.close
         end
 
       Concurrent::Promise.zip(*promises).value!.each do |blob|
@@ -129,39 +130,27 @@ module Templates
       pool&.kill
     end
 
-    def build_and_upload_blob(doc, page_number, format = FORMAT)
+    def build_and_upload_blob(image, page_number, format = FORMAT)
-      doc_page = doc.get_page(page_number)
+      image = image.copy(interpretation: :srgb)
-
-      data, width, height = doc_page.render_to_bitmap(width: MAX_WIDTH)
-
-      page = Vips::Image.new_from_memory(data, width, height, 4, :uchar)
-
-      page = page.copy(interpretation: :srgb)
 
       data =
         if format == FORMAT
-          bitdepth = 2**page.stats.to_a[1..3].pluck(2).uniq.size
+          bitdepth = 2**image.stats.to_a[1..3].pluck(2).uniq.size
 
-          page.write_to_buffer(format, compression: 6, filter: 0, bitdepth:,
+          image.write_to_buffer(format, compression: 6, filter: 0, bitdepth:,
-                                       palette: true, Q: Q, dither: 0)
+                                        palette: true, Q: Q, dither: 0)
         else
-          page.write_to_buffer(format, interlace: true, Q: JPEG_Q)
+          image.write_to_buffer(format, interlace: true, Q: JPEG_Q)
         end
 
       blob = ActiveStorage::Blob.new(
         filename: "#{page_number}#{format}",
-        metadata: { analyzed: true, identified: true, width: page.width, height: page.height }
+        metadata: { analyzed: true, identified: true, width: image.width, height: image.height }
       )
 
       blob.upload(StringIO.new(data))
 
       blob
-    rescue Vips::Error, Pdfium::PdfiumError => e
-      Rollbar.warning(e) if defined?(Rollbar)
-
-      nil
-    ensure
-      doc_page&.close
     end
 
     def maybe_flatten_form(data, pdf)
@@ -206,7 +195,15 @@ module Templates
     def generate_pdf_preview_from_file(attachment, file_path, page_number)
       doc = Pdfium::Document.open_file(file_path)
 
-      blob = build_and_upload_blob(doc, page_number, PREVIEW_FORMAT)
+      doc_page = doc.get_page(page_number)
+
+      bytes, width, height = doc_page.render_to_bitmap(width: MAX_WIDTH)
+
+      doc_page.close
+
+      image = Vips::Image.new_from_memory(bytes, width, height, 4, :uchar)
+
+      blob = build_and_upload_blob(image, page_number, PREVIEW_FORMAT)
 
       ApplicationRecord.no_touching do
         ActiveStorage::Attachment.create!(