adjust migration

app/jobs/process_submitter_completion_job.rb

@@ -6,8 +6,6 @@ class ProcessSubmitterCompletionJob
   def perform(params = {})
     submitter = Submitter.find(params['submitter_id'])
 
-    create_completed_submitter!(submitter)
-
     is_all_completed = !submitter.submission.submitters.exists?(completed_at: nil)
 
     if !is_all_completed && submitter.submission.submitters_order_preserved?
@@ -26,31 +24,36 @@ class ProcessSubmitterCompletionJob
       enqueue_completed_emails(submitter)
     end
 
-    create_completed_documents!(submitter)
+    create_completed_submitter!(submitter)
 
     enqueue_completed_webhooks(submitter, is_all_completed:)
   end
 
   def create_completed_submitter!(submitter)
+    completed_submitter = CompletedSubmitter.find_or_initialize_by(submitter_id: submitter.id)
+
+    return completed_submitter if completed_submitter.persisted?
+
     submission = submitter.submission
-    sms_count = submitter.submission_events.where(event_type: %w[send_sms send_2fa_sms]).count
-    completed_submitter = CompletedSubmitter.where(submitter_id: submitter.id).first_or_initialize
+
     completed_submitter.assign_attributes(
       submission_id: submitter.submission_id,
       account_id: submission.account_id,
       template_id: submission.template_id,
       source: submission.source,
-      sms_count:,
+      sms_count: submitter.submission_events.where(event_type: %w[send_sms send_2fa_sms]).count,
       completed_at: submitter.completed_at
     )
 
-    completed_submitter.save!
-  end
+    submitter.documents.each do |attachment|
+      next if attachment.metadata['sha256'].blank?
 
-  def create_completed_documents!(submitter)
-    submitter.documents.map { |s| s.metadata['sha256'] }.compact_blank.each do |sha256|
-      CompletedDocument.where(submitter_id: submitter.id, sha256:).first_or_create!
+      completed_submitter.completed_documents << CompletedDocument.new(sha256: attachment.metadata['sha256'])
     end
+
+    completed_submitter.save!
+
+    completed_submitter
   end
 
   def enqueue_completed_webhooks(submitter, is_all_completed: false)

app/models/completed_document.rb

@@ -17,4 +17,6 @@
 #
 class CompletedDocument < ApplicationRecord
   belongs_to :submitter
+
+  has_one :completed_submitter, primary_key: :submitter_id, inverse_of: :completed_documents, dependent: :destroy
 end

app/models/completed_submitter.rb

@@ -17,11 +17,17 @@
 #
 # Indexes
 #
-#  index_completed_submitters_on_account_id  (account_id)
+#  index_completed_submitters_on_account_id    (account_id)
+#  index_completed_submitters_on_submitter_id  (submitter_id)
 #
 class CompletedSubmitter < ApplicationRecord
   belongs_to :submitter
   belongs_to :submission
   belongs_to :account
   belongs_to :template
+
+  has_many :completed_documents, dependent: :destroy,
+                                 primary_key: :submitter_id,
+                                 foreign_key: :submitter_id,
+                                 inverse_of: :submitter
 end

db/migrate/20241018115034_create_completed_submitters_and_documents.rb

@@ -3,7 +3,7 @@
 class CreateCompletedSubmittersAndDocuments < ActiveRecord::Migration[7.2]
   def change
     create_table :completed_submitters do |t|
-      t.bigint :submitter_id, null: false
+      t.bigint :submitter_id, null: false, index: true
       t.bigint :submission_id, null: false
       t.bigint :account_id, null: false, index: true
       t.bigint :template_id, null: false

db/migrate/20241022125135_populate_completed_submitters_and_documents.rb

@@ -7,7 +7,8 @@ class PopulateCompletedSubmittersAndDocuments < ActiveRecord::Migration[7.2]
     self.table_name = 'submitters'
 
     belongs_to :submission, class_name: 'MigrationSubmission'
-    has_many :submission_events, class_name: 'MigrationSubmissionEvent', foreign_key: :submitter_id
+    has_many :submission_sms_events, -> { where(event_type: %w[send_sms send_2fa_sms]) },
+             class_name: 'MigrationSubmissionEvent', foreign_key: :submitter_id
   end
 
   class MigrationSubmission < ApplicationRecord
@@ -27,18 +28,26 @@ class PopulateCompletedSubmittersAndDocuments < ActiveRecord::Migration[7.2]
   end
 
   def up
-    completed_submitters = MigrationSubmitter.where.not(completed_at: nil)
+    submitters = MigrationSubmitter.where.not(completed_at: nil)
+                                   .preload(:submission, :submission_sms_events)
+
+    count = submitters.count
+
+    puts "Updating the database - it might take ~#{(count / 1000 * 3) + 1} seconds" if count > 2000
+
+    submitters.find_each do |submitter|
+      completed_submitter = MigrationCompletedSubmitter.find_or_initialize_by(submitter_id: submitter.id)
+
+      next if completed_submitter.persisted?
 
-    completed_submitters.order(created_at: :asc).preload(:submission).find_each do |submitter|
       submission = submitter.submission
-      sms_count = submitter.submission_events.where(event_type: %w[send_sms send_2fa_sms]).count
-      completed_submitter = MigrationCompletedSubmitter.where(submitter_id: submitter.id).first_or_initialize
+
       completed_submitter.assign_attributes(
         submission_id: submitter.submission_id,
         account_id: submission.account_id,
         template_id: submission.template_id,
         source: submission.source,
-        sms_count:,
+        sms_count: submitter.submission_sms_events.size,
         completed_at: submitter.completed_at,
         created_at: submitter.completed_at,
         updated_at: submitter.completed_at
@@ -47,21 +56,18 @@ class PopulateCompletedSubmittersAndDocuments < ActiveRecord::Migration[7.2]
       completed_submitter.save!
     end
 
-    ActiveStorage::Attachment.where(record_id: completed_submitters.select(:id),
-                                    record_type: 'Submitter',
-                                    name: 'documents')
-                             .order(created_at: :asc)
-                             .find_each do |attachment|
+    attachments = ActiveStorage::Attachment.where(record_type: 'Submitter', name: 'documents').preload(:blob)
+
+    attachments.find_each do |attachment|
       sha256 = attachment.metadata['sha256']
-      submitter_id = attachment.record_id
 
       next if sha256.blank?
 
-      completed_document = MigrationCompletedDocument.where(submitter_id:, sha256:).first_or_initialize
-      completed_document.assign_attributes(
-        created_at: attachment.created_at,
-        updated_at: attachment.created_at
-      )
+      completed_document = MigrationCompletedDocument.find_or_initialize_by(submitter_id: attachment.record_id, sha256:)
+
+      next if completed_document.persisted?
+
+      completed_document.assign_attributes(created_at: attachment.created_at, updated_at: attachment.created_at)
 
       completed_document.save!
     end

db/schema.rb

@@ -109,6 +109,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_22_125135) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["account_id"], name: "index_completed_submitters_on_account_id"
+    t.index ["submitter_id"], name: "index_completed_submitters_on_submitter_id"
   end
 
   create_table "document_generation_events", force: :cascade do |t|