add lock events

lib/submissions/ensure_audit_generated.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Submissions
+  module EnsureAuditGenerated
+    WAIT_FOR_RETRY = 2.seconds
+    CHECK_EVENT_INTERVAL = 1.second
+    CHECK_COMPLETE_TIMEOUT = 90.seconds
+    KEY_PREFIX = 'audit_trail'
+
+    WaitForCompleteTimeout = Class.new(StandardError)
+    NotCompletedYet = Class.new(StandardError)
+
+    module_function
+
+    def call(submission)
+      return nil unless submission
+
+      raise NotCompletedYet unless submission.submitters.all?(&:completed_at?)
+
+      key = [KEY_PREFIX, submission.id].join(':')
+
+      if ApplicationRecord.uncached { LockEvent.exists?(key:, event_name: :complete) }
+        return submission.audit_trail_attachment
+      end
+
+      events = ApplicationRecord.uncached { LockEvent.where(key:).order(:id).to_a }
+
+      if events.present? && events.last.event_name.in?(%w[start retry])
+        wait_for_complete_or_fail(submission)
+      else
+        LockEvent.create!(key:, event_name: events.present? ? :retry : :start)
+
+        result = Submissions::GenerateAuditTrail.call(submission)
+
+        LockEvent.create!(key:, event_name: :complete)
+
+        result
+      end
+    rescue ActiveRecord::RecordNotUnique
+      sleep WAIT_FOR_RETRY
+
+      retry
+    rescue StandardError => e
+      Rollbar.error(e) if defined?(Rollbar)
+      Rails.logger.error(e)
+
+      LockEvent.create!(key:, event_name: :fail)
+
+      raise
+    end
+
+    def wait_for_complete_or_fail(submission)
+      total_wait_time = 0
+
+      loop do
+        sleep CHECK_EVENT_INTERVAL
+        total_wait_time += CHECK_EVENT_INTERVAL
+
+        last_event =
+          ApplicationRecord.uncached do
+            LockEvent.where(key: [KEY_PREFIX, submission.id].join(':')).order(:id).last
+          end
+
+        if last_event.event_name.in?(%w[complete fail])
+          break ApplicationRecord.uncached do
+            ActiveStorage::Attachment.find_by(record: submission, name: 'audit_trail')
+          end
+        end
+
+        raise WaitForCompleteTimeout if total_wait_time > CHECK_COMPLETE_TIMEOUT
+      end
+    end
+  end
+end

lib/submissions/ensure_combined_generated.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Submissions
+  module EnsureCombinedGenerated
+    WAIT_FOR_RETRY = 2.seconds
+    CHECK_EVENT_INTERVAL = 1.second
+    CHECK_COMPLETE_TIMEOUT = 90.seconds
+    KEY_PREFIX = 'combined_document'
+
+    WaitForCompleteTimeout = Class.new(StandardError)
+    NotCompletedYet = Class.new(StandardError)
+
+    module_function
+
+    def call(submitter)
+      return nil unless submitter
+
+      raise NotCompletedYet unless submitter.completed_at?
+
+      key = [KEY_PREFIX, submitter.id].join(':')
+
+      if ApplicationRecord.uncached { LockEvent.exists?(key:, event_name: :complete) }
+        return submitter.submission.combined_document_attachment
+      end
+
+      events = ApplicationRecord.uncached { LockEvent.where(key:).order(:id).to_a }
+
+      if events.present? && events.last.event_name.in?(%w[start retry])
+        wait_for_complete_or_fail(submitter)
+      else
+        LockEvent.create!(key:, event_name: events.present? ? :retry : :start)
+
+        result = Submissions::GenerateCombinedAttachment.call(submitter)
+
+        LockEvent.create!(key:, event_name: :complete)
+
+        result
+      end
+    rescue ActiveRecord::RecordNotUnique
+      sleep WAIT_FOR_RETRY
+
+      retry
+    rescue StandardError => e
+      Rollbar.error(e) if defined?(Rollbar)
+      Rails.logger.error(e)
+
+      LockEvent.create!(key:, event_name: :fail)
+
+      raise
+    end
+
+    def wait_for_complete_or_fail(submitter)
+      total_wait_time = 0
+
+      loop do
+        sleep CHECK_EVENT_INTERVAL
+        total_wait_time += CHECK_EVENT_INTERVAL
+
+        last_event =
+          ApplicationRecord.uncached do
+            LockEvent.where(key: [KEY_PREFIX, submitter.id].join(':')).order(:id).last
+          end
+
+        if last_event.event_name.in?(%w[complete fail])
+          break ApplicationRecord.uncached do
+            ActiveStorage::Attachment.find_by(record: submitter.submission, name: 'combined_document')
+          end
+        end
+
+        raise WaitForCompleteTimeout if total_wait_time > CHECK_COMPLETE_TIMEOUT
+      end
+    end
+  end
+end

lib/submissions/ensure_result_generated.rb

@@ -16,21 +16,20 @@ module Submissions
 
       raise NotCompletedYet unless submitter.completed_at?
 
-      return submitter.documents if ApplicationRecord.uncached { submitter.document_generation_events.complete.exists? }
+      key = ['result_attachments', submitter.id].join(':')
 
-      events =
-        ApplicationRecord.uncached do
-          DocumentGenerationEvent.where(submitter:).order(:created_at).to_a
-        end
+      return submitter.documents if ApplicationRecord.uncached { LockEvent.exists?(key:, event_name: :complete) }
+
+      events = ApplicationRecord.uncached { LockEvent.where(key:).order(:id).to_a }
 
       if events.present? && events.last.event_name.in?(%w[start retry])
         wait_for_complete_or_fail(submitter)
       else
-        submitter.document_generation_events.create!(event_name: events.present? ? :retry : :start)
+        LockEvent.create!(key:, event_name: events.present? ? :retry : :start)
 
         documents = GenerateResultAttachments.call(submitter)
 
-        submitter.document_generation_events.create!(event_name: :complete)
+        LockEvent.create!(key:, event_name: :complete)
 
         documents
       end
@@ -42,7 +41,7 @@ module Submissions
       Rollbar.error(e) if defined?(Rollbar)
       Rails.logger.error(e)
 
-      submitter.document_generation_events.create!(event_name: :fail)
+      LockEvent.create!(key:, event_name: :fail)
 
       raise
     end
@@ -56,7 +55,7 @@ module Submissions
 
         last_event =
           ApplicationRecord.uncached do
-            DocumentGenerationEvent.where(submitter:).order(:created_at).last
+            LockEvent.where(key: ['result_attachments', submitter.id].join(':')).order(:id).last
           end
 
         break submitter.documents.reload if last_event.event_name.in?(%w[complete fail])

lib/submissions/serialize_for_api.rb

@@ -74,7 +74,7 @@ module Submissions
       if !attachment && params[:include].to_s.include?('combined_document_url')
         submitter = submitters.max_by(&:completed_at)
 
-        attachment = Submissions::GenerateCombinedAttachment.call(submitter)
+        attachment = Submissions::EnsureCombinedGenerated.call(submitter)
       end
 
       ActiveStorage::Blob.proxy_url(attachment.blob, expires_at:) if attachment