use sidekiq

.gitignore

@@ -36,3 +36,4 @@ yarn-debug.log*
 /attachments
 /docuseal
 /ee
+dump.rdb

.rubocop.yml

@@ -76,5 +76,8 @@ Rails/SkipsModelValidations:
 Rails/ApplicationController:
   Enabled: false
 
+Rails/Output:
+  Enabled: false
+
 Capybara/ClickLinkOrButtonStyle:
   Enabled: false

Dockerfile

@@ -41,7 +41,7 @@ ENV OPENSSL_CONF=/app/openssl_legacy.cnf
 
 WORKDIR /app
 
-RUN apk add --no-cache sqlite-dev libpq-dev mariadb-dev vips-dev vips-poppler poppler-utils vips-heif gcompat ttf-freefont && mkdir /fonts && rm /usr/share/fonts/freefont/FreeSans.otf
+RUN apk add --no-cache sqlite-dev libpq-dev mariadb-dev vips-dev vips-poppler poppler-utils redis vips-heif gcompat ttf-freefont && mkdir /fonts && rm /usr/share/fonts/freefont/FreeSans.otf
 
 RUN echo $'.include = /etc/ssl/openssl.cnf\n\
 \n\

Gemfile

@@ -27,7 +27,7 @@ gem 'pagy'
 gem 'pg', require: false
 gem 'premailer-rails'
 gem 'pretender'
-gem 'puma'
+gem 'puma', require: false
 gem 'rack'
 gem 'rails'
 gem 'rails_autolink'
@@ -37,7 +37,7 @@ gem 'rqrcode'
 gem 'ruby-vips'
 gem 'rubyXL'
 gem 'shakapacker'
-gem 'sidekiq', require: ENV.key?('REDIS_URL')
+gem 'sidekiq'
 gem 'sqlite3', require: false, force_ruby_platform: true
 gem 'strip_attributes'
 gem 'turbo-rails'

app/controllers/api/submissions_controller.rb

@@ -64,14 +64,14 @@ module Api
       submissions = create_submissions(@template, params)
 
       submissions.each do |submission|
-        SendSubmissionCreatedWebhookRequestJob.perform_later({ 'submission_id' => submission.id })
+        SendSubmissionCreatedWebhookRequestJob.perform_async({ 'submission_id' => submission.id })
       end
 
       Submissions.send_signature_requests(submissions)
 
       submissions.each do |submission|
         if submission.submitters.all?(&:completed_at?) && submission.submitters.last
-          ProcessSubmitterCompletionJob.perform_later({ 'submitter_id' => submission.submitters.last.id })
+          ProcessSubmitterCompletionJob.perform_async({ 'submitter_id' => submission.submitters.last.id })
         end
       end
 
@@ -94,7 +94,7 @@ module Api
       else
         @submission.update!(archived_at: Time.current)
 
-        SendSubmissionArchivedWebhookRequestJob.perform_later('submission_id' => @submission.id)
+        SendSubmissionArchivedWebhookRequestJob.perform_async('submission_id' => @submission.id)
       end
 
       render json: @submission.as_json(only: %i[id archived_at])

app/controllers/api/submitter_form_views_controller.rb

@@ -13,7 +13,7 @@ module Api
 
       SubmissionEvents.create_with_tracking_data(submitter, 'view_form', request)
 
-      SendFormViewedWebhookRequestJob.perform_later({ 'submitter_id' => submitter.id })
+      SendFormViewedWebhookRequestJob.perform_async({ 'submitter_id' => submitter.id })
 
       render json: {}
     end

app/controllers/api/submitters_controller.rb

@@ -66,7 +66,7 @@ module Api
       end
 
       if @submitter.completed_at?
-        ProcessSubmitterCompletionJob.perform_later({ 'submitter_id' => @submitter.id })
+        ProcessSubmitterCompletionJob.perform_async({ 'submitter_id' => @submitter.id })
       elsif normalized_params[:send_email] || normalized_params[:send_sms]
         Submitters.send_signature_requests([@submitter])
       end

app/controllers/api/templates_clone_controller.rb

@@ -20,7 +20,7 @@ module Api
 
       Templates::CloneAttachments.call(template: cloned_template, original_template: @template)
 
-      SendTemplateCreatedWebhookRequestJob.perform_later('template_id' => cloned_template.id)
+      SendTemplateCreatedWebhookRequestJob.perform_async('template_id' => cloned_template.id)
 
       render json: Templates::SerializeForApi.call(cloned_template)
     end

app/controllers/api/templates_controller.rb

@@ -65,7 +65,7 @@ module Api
 
       @template.update!(template_params)
 
-      SendTemplateUpdatedWebhookRequestJob.perform_later('template_id' => @template.id)
+      SendTemplateUpdatedWebhookRequestJob.perform_async('template_id' => @template.id)
 
       render json: @template.as_json(only: %i[id updated_at])
     end

app/controllers/start_form_controller.rb

@@ -37,7 +37,7 @@ class StartFormController < ApplicationController
 
       if @submitter.save
         if is_new_record
-          SendSubmissionCreatedWebhookRequestJob.perform_later({ 'submission_id' => @submitter.submission.id })
+          SendSubmissionCreatedWebhookRequestJob.perform_async({ 'submission_id' => @submitter.submission.id })
         end
 
         redirect_to submit_form_path(@submitter.slug)

app/controllers/submissions_controller.rb

@@ -45,7 +45,7 @@ class SubmissionsController < ApplicationController
       end
 
     submissions.each do |submission|
-      SendSubmissionCreatedWebhookRequestJob.perform_later({ 'submission_id' => submission.id })
+      SendSubmissionCreatedWebhookRequestJob.perform_async({ 'submission_id' => submission.id })
     end
 
     Submissions.send_signature_requests(submissions)
@@ -56,7 +56,7 @@ class SubmissionsController < ApplicationController
   def destroy
     @submission.update!(archived_at: Time.current)
 
-    SendSubmissionArchivedWebhookRequestJob.perform_later('submission_id' => @submission.id)
+    SendSubmissionArchivedWebhookRequestJob.perform_async('submission_id' => @submission.id)
 
     redirect_back(fallback_location: template_path(@submission.template), notice: 'Submission has been archived')
   end

app/controllers/submitters_send_email_controller.rb

@@ -13,7 +13,7 @@ class SubmittersSendEmailController < ApplicationController
                            alert: 'Email has been sent already.')
     end
 
-    SendSubmitterInvitationEmailJob.perform_later('submitter_id' => @submitter.id)
+    SendSubmitterInvitationEmailJob.perform_async('submitter_id' => @submitter.id)
 
     @submitter.sent_at ||= Time.current
     @submitter.save!

app/controllers/templates_controller.rb

@@ -61,7 +61,7 @@ class TemplatesController < ApplicationController
     if @template.save
       Templates::CloneAttachments.call(template: @template, original_template: @base_template) if @base_template
 
-      SendTemplateUpdatedWebhookRequestJob.perform_later('template_id' => @template.id)
+      SendTemplateUpdatedWebhookRequestJob.perform_async('template_id' => @template.id)
 
       maybe_redirect_to_template(@template)
     else
@@ -72,7 +72,7 @@ class TemplatesController < ApplicationController
   def update
     @template.update!(template_params)
 
-    SendTemplateUpdatedWebhookRequestJob.perform_later('template_id' => @template.id)
+    SendTemplateUpdatedWebhookRequestJob.perform_async('template_id' => @template.id)
 
     head :ok
   end

app/controllers/templates_uploads_controller.rb

@@ -18,7 +18,7 @@ class TemplatesUploadsController < ApplicationController
 
     @template.update!(schema:)
 
-    SendTemplateCreatedWebhookRequestJob.perform_later('template_id' => @template.id)
+    SendTemplateCreatedWebhookRequestJob.perform_async('template_id' => @template.id)
 
     redirect_to edit_template_path(@template)
   rescue Templates::CreateAttachments::PdfEncrypted

app/controllers/webhook_settings_controller.rb

@@ -17,7 +17,7 @@ class WebhookSettingsController < ApplicationController
   def update
     submitter = current_account.submitters.where.not(completed_at: nil).order(:id).last
 
-    SendFormCompletedWebhookRequestJob.perform_later({ 'submitter_id' => submitter.id })
+    SendFormCompletedWebhookRequestJob.perform_async({ 'submitter_id' => submitter.id })
 
     redirect_back(fallback_location: settings_webhooks_path, notice: 'Webhook request has been sent.')
   end

app/jobs/process_submitter_completion_job.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-class ProcessSubmitterCompletionJob < ApplicationJob
+class ProcessSubmitterCompletionJob
+  include Sidekiq::Job
+
   def perform(params = {})
     submitter = Submitter.find(params['submitter_id'])
 
@@ -20,7 +22,7 @@ class ProcessSubmitterCompletionJob < ApplicationJob
 
     return if Accounts.load_webhook_url(submitter.account).blank?
 
-    SendFormCompletedWebhookRequestJob.perform_later({ 'submitter_id' => submitter.id })
+    SendFormCompletedWebhookRequestJob.perform_async({ 'submitter_id' => submitter.id })
   end
 
   def enqueue_completed_emails(submitter)

app/jobs/send_form_completed_webhook_request_job.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-class SendFormCompletedWebhookRequestJob < ApplicationJob
+class SendFormCompletedWebhookRequestJob
-  queue_as :webhooks
+  include Sidekiq::Job
+
+  sidekiq_options queue: :webhooks
 
   USER_AGENT = 'DocuSeal.co Webhook'
 
@@ -38,8 +40,7 @@ class SendFormCompletedWebhookRequestJob < ApplicationJob
 
     if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS &&
        (!Docuseal.multitenant? || submitter.account.account_configs.exists?(key: :plan))
-      SendFormCompletedWebhookRequestJob.set(wait: (2**attempt).minutes)
+      SendFormCompletedWebhookRequestJob.perform_in((2**attempt).minutes, {
-                                        .perform_later({
                                                       'submitter_id' => submitter.id,
                                                       'attempt' => attempt + 1,
                                                       'last_status' => resp&.status.to_i

app/jobs/send_form_started_webhook_request_job.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-class SendFormStartedWebhookRequestJob < ApplicationJob
+class SendFormStartedWebhookRequestJob
-  queue_as :webhooks
+  include Sidekiq::Job
+
+  sidekiq_options queue: :webhooks
 
   USER_AGENT = 'DocuSeal.co Webhook'
 
@@ -36,8 +38,7 @@ class SendFormStartedWebhookRequestJob < ApplicationJob
 
     if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS &&
        (!Docuseal.multitenant? || submitter.account.account_configs.exists?(key: :plan))
-      SendFormStartedWebhookRequestJob.set(wait: (2**attempt).minutes)
+      SendFormStartedWebhookRequestJob.perform_in((2**attempt).minutes, {
-                                      .perform_later({
                                                     'submitter_id' => submitter.id,
                                                     'attempt' => attempt + 1,
                                                     'last_status' => resp&.status.to_i

app/jobs/send_form_viewed_webhook_request_job.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-class SendFormViewedWebhookRequestJob < ApplicationJob
+class SendFormViewedWebhookRequestJob
-  queue_as :webhooks
+  include Sidekiq::Job
+
+  sidekiq_options queue: :webhooks
 
   USER_AGENT = 'DocuSeal.co Webhook'
 
@@ -36,8 +38,7 @@ class SendFormViewedWebhookRequestJob < ApplicationJob
 
     if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS &&
        (!Docuseal.multitenant? || submitter.account.account_configs.exists?(key: :plan))
-      SendFormViewedWebhookRequestJob.set(wait: (2**attempt).minutes)
+      SendFormViewedWebhookRequestJob.perform_in((2**attempt).minutes, {
-                                     .perform_later({
                                                    'submitter_id' => submitter.id,
                                                    'attempt' => attempt + 1,
                                                    'last_status' => resp&.status.to_i

app/jobs/send_submission_archived_webhook_request_job.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-class SendSubmissionArchivedWebhookRequestJob < ApplicationJob
+class SendSubmissionArchivedWebhookRequestJob
-  queue_as :webhooks
+  include Sidekiq::Job
+
+  sidekiq_options queue: :webhooks
 
   USER_AGENT = 'DocuSeal.co Webhook'
 
@@ -34,8 +36,7 @@ class SendSubmissionArchivedWebhookRequestJob < ApplicationJob
 
     if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS &&
        (!Docuseal.multitenant? || submission.account.account_configs.exists?(key: :plan))
-      SendSubmissionArchivedWebhookRequestJob.set(wait: (2**attempt).minutes)
+      SendSubmissionArchivedWebhookRequestJob.perform_in((2**attempt).minutes, {
-                                             .perform_later({
                                                            'submission_id' => submission.id,
                                                            'attempt' => attempt + 1,
                                                            'last_status' => resp&.status.to_i

app/jobs/send_submission_created_webhook_request_job.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-class SendSubmissionCreatedWebhookRequestJob < ApplicationJob
+class SendSubmissionCreatedWebhookRequestJob
-  queue_as :webhooks
+  include Sidekiq::Job
+
+  sidekiq_options queue: :webhooks
 
   USER_AGENT = 'DocuSeal.co Webhook'
 
@@ -34,8 +36,7 @@ class SendSubmissionCreatedWebhookRequestJob < ApplicationJob
 
     if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS &&
        (!Docuseal.multitenant? || submission.account.account_configs.exists?(key: :plan))
-      SendSubmissionCreatedWebhookRequestJob.set(wait: (2**attempt).minutes)
+      SendSubmissionCreatedWebhookRequestJob.perform_in((2**attempt).minutes, {
-                                            .perform_later({
                                                           'submission_id' => submission.id,
                                                           'attempt' => attempt + 1,
                                                           'last_status' => resp&.status.to_i

app/jobs/send_submitter_invitation_email_job.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-class SendSubmitterInvitationEmailJob < ApplicationJob
+class SendSubmitterInvitationEmailJob
+  include Sidekiq::Job
+
   def perform(params = {})
     submitter = Submitter.find(params['submitter_id'])
 

app/jobs/send_template_created_webhook_request_job.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-class SendTemplateCreatedWebhookRequestJob < ApplicationJob
+class SendTemplateCreatedWebhookRequestJob
-  queue_as :webhooks
+  include Sidekiq::Job
+
+  sidekiq_options queue: :webhooks
 
   USER_AGENT = 'DocuSeal.co Webhook'
 
@@ -34,8 +36,7 @@ class SendTemplateCreatedWebhookRequestJob < ApplicationJob
 
     if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS &&
        (!Docuseal.multitenant? || template.account.account_configs.exists?(key: :plan))
-      SendTemplateCreatedWebhookRequestJob.set(wait: (2**attempt).minutes)
+      SendTemplateCreatedWebhookRequestJob.perform_in((2**attempt).minutes, {
-                                          .perform_later({
                                                         'template_id' => template.id,
                                                         'attempt' => attempt + 1,
                                                         'last_status' => resp&.status.to_i

app/jobs/send_template_updated_webhook_request_job.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-class SendTemplateUpdatedWebhookRequestJob < ApplicationJob
+class SendTemplateUpdatedWebhookRequestJob
-  queue_as :webhooks
+  include Sidekiq::Job
+
+  sidekiq_options queue: :webhooks
 
   USER_AGENT = 'DocuSeal.co Webhook'
 
@@ -34,8 +36,7 @@ class SendTemplateUpdatedWebhookRequestJob < ApplicationJob
 
     if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS &&
        (!Docuseal.multitenant? || template.account.account_configs.exists?(key: :plan))
-      SendTemplateUpdatedWebhookRequestJob.set(wait: (2**attempt).minutes)
+      SendTemplateUpdatedWebhookRequestJob.perform_in((2**attempt).minutes, {
-                                          .perform_later({
                                                         'template_id' => template.id,
                                                         'attempt' => attempt + 1,
                                                         'last_status' => resp&.status.to_i

app/models/user.rb

@@ -81,6 +81,12 @@ class User < ApplicationRecord
     true
   end
 
+  def sidekiq?
+    return true if Rails.env.development?
+
+    role == 'admin'
+  end
+
   def self.sign_in_after_reset_password
     if PasswordsController::Current.user.present?
       !PasswordsController::Current.user.otp_required_for_login

config/application.rb

@@ -19,7 +19,7 @@ module DocuSeal
   class Application < Rails::Application
     config.load_defaults 7.1
 
-    config.autoload_lib(ignore: %w[assets tasks])
+    config.autoload_lib(ignore: %w[assets tasks puma])
 
     config.active_storage.routes_prefix = ''
 

config/boot.rb

@@ -2,41 +2,7 @@
 
 ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
 
-if ENV['RAILS_ENV'] == 'production' && ENV['SECRET_KEY_BASE'].to_s.empty?
+require_relative 'dotenv'
-  require 'dotenv'
-  require 'securerandom'
-
-  dotenv_path = "#{ENV.fetch('WORKDIR', '.')}/docuseal.env"
-
-  unless File.exist?(dotenv_path)
-    default_env = <<~TEXT
-      DATABASE_URL= # keep empty to use sqlite or specify postgresql database URL
-      SECRET_KEY_BASE=#{SecureRandom.hex(64)}
-    TEXT
-
-    File.write(dotenv_path, default_env)
-  end
-
-  database_url = ENV.fetch('DATABASE_URL', nil)
-
-  Dotenv.load(dotenv_path)
-
-  ENV['DATABASE_URL'] = ENV['DATABASE_URL'].to_s.empty? ? database_url : ENV.fetch('DATABASE_URL', nil)
-end
-
-if ENV['DATABASE_URL'].to_s.split('@').last.to_s.split('/').first.to_s.include?('_')
-  require 'addressable'
-
-  url = Addressable::URI.parse(ENV.fetch('DATABASE_URL', ''))
-
-  ENV['DATABASE_HOST'] = url.host
-  ENV['DATABASE_PORT'] = (url.port || 5432).to_s
-  ENV['DATABASE_USER'] = url.user
-  ENV['DATABASE_PASSWORD'] = url.password
-  ENV['DATABASE_NAME'] = url.path.to_s.delete_prefix('/')
-
-  ENV.delete('DATABASE_URL')
-end
 
 require 'bundler/setup' # Set up gems listed in the Gemfile.
 require 'bootsnap/setup' # Speed up boot time by caching expensive operations.

config/database.yml

@@ -1,7 +1,7 @@
 default: &default
   adapter: postgresql
   encoding: unicode
-  pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15) %>
+  pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15).to_i + ENV.fetch('SIDEKIQ_THREADS', 5).to_i %>
 
 development:
   <<: *default
@@ -22,12 +22,14 @@ production:
 <% elsif ENV['DATABASE_URL'].to_s.empty? %>
   adapter: sqlite3
   database: <%= ENV['WORKDIR'] || '.' %>/db.sqlite3
+  pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15).to_i + ENV.fetch('SIDEKIQ_THREADS', 5).to_i %>
+  timeout: 5000
 <% elsif ENV['DATABASE_URL'].match?(/\Apostgres/) %>
   <<: *default
   url: <%= ENV['DATABASE_URL'] %>
 <% elsif ENV['DATABASE_URL'].match?(/\Amysql/) %>
   adapter: mysql2
   encoding: utf8mb4
-  pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15) %>
+  pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15).to_i + ENV.fetch('SIDEKIQ_THREADS', 5).to_i %>
   url: <%= ENV['DATABASE_URL'] %>
 <% end %>

config/dotenv.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+if ENV['RAILS_ENV'] == 'production' && ENV['SECRET_KEY_BASE'].to_s.empty?
+  require 'dotenv'
+  require 'securerandom'
+
+  dotenv_path = "#{ENV.fetch('WORKDIR', '.')}/docuseal.env"
+
+  unless File.exist?(dotenv_path)
+    default_env = <<~TEXT
+      DATABASE_URL= # keep empty to use sqlite or specify postgresql database URL
+      SECRET_KEY_BASE=#{SecureRandom.hex(64)}
+    TEXT
+
+    File.write(dotenv_path, default_env)
+  end
+
+  database_url = ENV.fetch('DATABASE_URL', nil)
+
+  Dotenv.load(dotenv_path)
+
+  ENV['DATABASE_URL'] = ENV['DATABASE_URL'].to_s.empty? ? database_url : ENV.fetch('DATABASE_URL', nil)
+end
+
+if ENV['DATABASE_URL'].to_s.split('@').last.to_s.split('/').first.to_s.include?('_')
+  require 'addressable'
+
+  url = Addressable::URI.parse(ENV.fetch('DATABASE_URL', ''))
+
+  ENV['DATABASE_HOST'] = url.host
+  ENV['DATABASE_PORT'] = (url.port || 5432).to_s
+  ENV['DATABASE_USER'] = url.user
+  ENV['DATABASE_PASSWORD'] = url.password
+  ENV['DATABASE_NAME'] = url.path.to_s.delete_prefix('/')
+
+  ENV.delete('DATABASE_URL')
+end
+
+if ENV['REDIS_URL'].to_s.empty?
+  require 'digest'
+
+  redis_password = Digest::SHA1.hexdigest("redis#{ENV.fetch('SECRET_KEY_BASE', '')}")
+
+  ENV['REDIS_URL'] = "redis://default:#{redis_password}@0.0.0.0:6379/0"
+  ENV['LOCAL_REDIS_URL'] = ENV.fetch('REDIS_URL', nil)
+end

config/environments/development.rb

@@ -47,7 +47,7 @@ Rails.application.configure do
     config.cache_store = :null_store
   end
 
-  config.active_job.queue_adapter = :sidekiq if defined?(Sidekiq)
+  config.active_job.queue_adapter = :sidekiq
 
   # Store uploaded files on the local file system (see config/storage.yml for options).
   config.active_storage.service = :disk

config/environments/production.rb

@@ -27,7 +27,7 @@ Rails.application.configure do
   config.action_controller.perform_caching = true
   config.active_record.sqlite3_production_warning = false
 
-  config.active_job.queue_adapter = :sidekiq if defined?(Sidekiq)
+  config.active_job.queue_adapter = :sidekiq
 
   # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"]
   # or in config/master.key. This key is used to decrypt credentials (and other encrypted files).

config/environments/test.rb

@@ -44,7 +44,7 @@ Rails.application.configure do
   # The :test delivery method accumulates sent emails in the
   # ActionMailer::Base.deliveries array.
   config.action_mailer.delivery_method = :test
-  config.active_job.queue_adapter = :test
+  config.active_job.queue_adapter = :sidekiq
 
   # Print deprecation notices to the stderr.
   config.active_support.deprecation = :stderr

config/initializers/sidekiq.rb

@@ -1,16 +1,14 @@
 # frozen_string_literal: true
 
-if defined?(Sidekiq)
+require 'sidekiq/web' if defined?(Puma)
-  require 'sidekiq/web'
 
+if !ENV['SIDEKIQ_BASIC_AUTH_PASSWORD'].to_s.empty? && defined?(Sidekiq::Web)
   Sidekiq::Web.use(Rack::Auth::Basic) do |_, password|
-    next true if Rails.env.development?
-
     ActiveSupport::SecurityUtils.secure_compare(
       Digest::SHA256.hexdigest(password),
       Digest::SHA256.hexdigest(ENV.fetch('SIDEKIQ_BASIC_AUTH_PASSWORD'))
     )
   end
-
-  Sidekiq.strict_args!
 end
+
+Sidekiq.strict_args!

config/puma.rb

@@ -5,7 +5,9 @@
 # Any libraries that use thread pools should be configured to match
 # the maximum value specified for Puma. Default is set to 5 threads for minimum
 # and maximum; this matches the default thread size of Active Record.
-#
+
+require_relative 'dotenv'
+
 max_threads_count = ENV.fetch('RAILS_MAX_THREADS', 15)
 min_threads_count = ENV.fetch('RAILS_MIN_THREADS') { max_threads_count }
 threads min_threads_count, max_threads_count
@@ -41,5 +43,10 @@ workers ENV.fetch('WEB_CONCURRENCY', 0)
 #
 # preload_app!
 
-# Allow puma to be restarted by `bin/rails restart` command.
+if ENV['MULTITENANT'] != 'true' || ENV['DEMO'] == 'true'
-plugin :tmp_restart
+  require_relative '../lib/puma/plugin/redis_server'
+  require_relative '../lib/puma/plugin/sidekiq_embed'
+
+  plugin :sidekiq_embed
+  plugin :redis_server
+end

config/routes.rb

@@ -2,7 +2,12 @@
 
 Rails.application.routes.draw do
   mount LetterOpenerWeb::Engine, at: '/letter_opener' if Rails.env.development?
-  mount Sidekiq::Web => '/sidekiq' if defined?(Sidekiq)
+
+  if !Docuseal.multitenant? && defined?(Sidekiq::Web)
+    authenticated :user, ->(u) { u.sidekiq? } do
+      mount Sidekiq::Web => '/jobs'
+    end
+  end
 
   root 'dashboard#index'
 

lib/puma/plugin/redis_server.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'puma/plugin'
+
+# rubocop:disable Metrics
+Puma::Plugin.create do
+  def start(launcher)
+    return if ENV['LOCAL_REDIS_URL'].to_s.empty?
+
+    @puma_pid = $PROCESS_ID
+
+    launcher.events.on_booted do
+      @redis_server_pid = fork_redis
+    end
+
+    in_background { monitor_redis }
+
+    at_exit do
+      stop_redis_server if Process.pid == @puma_pid
+    end
+
+    launcher.events.on_stopped { stop_redis_server }
+    launcher.events.on_restart { stop_redis_server }
+  end
+
+  private
+
+  def monitor_redis
+    loop do
+      if redis_dead?
+        Process.kill(:INT, @puma_pid)
+
+        break
+      end
+
+      sleep 5
+    end
+  end
+
+  def redis_dead?
+    return false unless @redis_server_pid
+
+    Process.waitpid(@redis_server_pid, Process::WNOHANG)
+
+    false
+  rescue Errno::ECHILD, Errno::ESRCH
+    true
+  end
+
+  def fork_redis
+    fork do
+      Process.setsid
+
+      Dir.chdir(ENV.fetch('WORKDIR', nil)) unless ENV['WORKDIR'].to_s.empty?
+
+      exec('redis-server', '--requirepass', Digest::SHA1.hexdigest("redis#{ENV.fetch('SECRET_KEY_BASE', '')}"),
+           out: '/dev/null')
+    end
+  end
+
+  def stop_redis_server
+    if @redis_server_pid
+      Process.kill(:INT, @redis_server_pid)
+      Process.wait(@redis_server_pid)
+    end
+  rescue Errno::ECHILD, Errno::ESRCH
+    nil
+  end
+end
+# rubocop:enable Metrics

lib/puma/plugin/sidekiq_embed.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'puma/plugin'
+
+# rubocop:disable Metrics
+Puma::Plugin.create do
+  def config(cfg)
+    return if cfg.instance_variable_get(:@options)[:workers] <= 0
+
+    cfg.on_worker_boot { start_sidekiq! }
+
+    cfg.on_worker_shutdown { @sidekiq&.stop }
+    cfg.on_refork { @sidekiq&.stop }
+  end
+
+  def start(launcher)
+    launcher.events.on_booted do
+      next if Puma.stats_hash[:workers].to_i != 0
+
+      start_sidekiq!
+    end
+
+    launcher.events.on_stopped { Thread.new { @sidekiq&.stop }.join }
+    launcher.events.on_restart { Thread.new { @sidekiq&.stop }.join }
+  end
+
+  def fire_event(config, event)
+    arr = config[:lifecycle_events][event]
+
+    arr.each(&:call)
+
+    arr.clear
+  end
+
+  def start_sidekiq!
+    Thread.new do
+      wait_for_redis!
+
+      configs = Sidekiq.configure_embed do |config|
+        config.logger.level = Logger::INFO
+        sidekiq_config = YAML.load_file('config/sidekiq.yml')
+        config.queues = sidekiq_config['queues']
+        config.concurrency = ENV.fetch('SIDEKIQ_THREADS', 5).to_i
+        config.merge!(sidekiq_config)
+        config[:max_retries] = 13
+
+        ActiveSupport.run_load_hooks(:sidekiq_config, config)
+      end.instance_variable_get(:@config)
+
+      @sidekiq = Sidekiq::Launcher.new(configs, embedded: true)
+
+      @sidekiq.run
+
+      fire_event(configs, :startup)
+    end
+  end
+
+  def wait_for_redis!
+    attempt = 0
+
+    loop do
+      attempt += 1
+
+      sleep (attempt - 1) / 10.0
+
+      RedisClient.new(url: ENV.fetch('REDIS_URL', nil)).call('GET', '1')
+
+      break
+    rescue RedisClient::CannotConnectError
+      raise('Unable to connect to redis') if attempt > 10
+    end
+  end
+end
+# rubocop:enable Metrics

lib/submitters.rb

@@ -73,7 +73,7 @@ module Submitters
       next if submitter.email.blank?
       next if submitter.preferences['send_email'] == false
 
-      SendSubmitterInvitationEmailJob.perform_later('submitter_id' => submitter.id)
+      SendSubmitterInvitationEmailJob.perform_async('submitter_id' => submitter.id)
     end
   end
 end

lib/submitters/submit_values.rb

@@ -14,14 +14,14 @@ module Submitters
       unless submitter.submission_events.exists?(event_type: 'start_form')
         SubmissionEvents.create_with_tracking_data(submitter, 'start_form', request)
 
-        SendFormStartedWebhookRequestJob.perform_later({ 'submitter_id' => submitter.id })
+        SendFormStartedWebhookRequestJob.perform_async({ 'submitter_id' => submitter.id })
       end
 
       update_submitter!(submitter, params, request)
 
       submitter.submission.save!
 
-      ProcessSubmitterCompletionJob.perform_later({ 'submitter_id' => submitter.id }) if submitter.completed_at?
+      ProcessSubmitterCompletionJob.perform_async({ 'submitter_id' => submitter.id }) if submitter.completed_at?
 
       submitter
     end

spec/rails_helper.rb

@@ -9,6 +9,9 @@ require 'rspec/rails'
 require 'capybara/cuprite'
 require 'capybara/rspec'
 require 'webmock/rspec'
+require 'sidekiq/testing'
+
+Sidekiq::Testing.fake!
 
 WebMock.disable_net_connect!(allow_localhost: true)
 
@@ -56,4 +59,12 @@ RSpec.configure do |config|
       driven_by :headless_cuprite
     end
   end
+
+  config.before do
+    Sidekiq::Worker.clear_all
+  end
+
+  config.before do |example|
+    Sidekiq::Testing.inline! if example.metadata[:sidekiq] == :inline
+  end
 end

spec/system/submit_form_spec.rb

@@ -81,7 +81,7 @@ RSpec.describe 'Submit Form' do
 
       expect do
         click_on 'submit'
-      end.to change(enqueued_jobs, :size).by(2)
+      end.to change(enqueued_jobs, :size).by(1)
     end
   end
 end