fulltext search

2025-06-05 13:16:55 +03:00
parent 9d1860f038
commit 33811945be
36 changed files with 682 additions and 25 deletions
@@ -73,6 +73,14 @@ module Docuseal
    @default_pkcs ||= GenerateCertificate.load_pkcs(Docuseal::CERTS)
  end

+  def fulltext_search?(_user)
+    return false unless SearchEntry.table_exists?
+    return true if Docuseal.multitenant?
+    return true if Rails.env.local?
+
+    false
+  end
+
  def enable_pwa?
    true
  end
@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+module PhoneCodes
+  ALL = [
+    '+1',
+    '+93',
+    '+358',
+    '+355',
+    '+213',
+    '+1684',
+    '+376',
+    '+244',
+    '+1264',
+    '+1268',
+    '+54',
+    '+374',
+    '+297',
+    '+61',
+    '+43',
+    '+994',
+    '+1242',
+    '+973',
+    '+880',
+    '+1246',
+    '+32',
+    '+501',
+    '+229',
+    '+1441',
+    '+975',
+    '+591',
+    '+387',
+    '+267',
+    '+55',
+    '+246',
+    '+673',
+    '+359',
+    '+226',
+    '+257',
+    '+855',
+    '+237',
+    '+1',
+    '+238',
+    '+1345',
+    '+235',
+    '+56',
+    '+86',
+    '+61',
+    '+61',
+    '+57',
+    '+269',
+    '+243',
+    '+682',
+    '+506',
+    '+225',
+    '+385',
+    '+357',
+    '+420',
+    '+45',
+    '+253',
+    '+1767',
+    '+1849',
+    '+593',
+    '+20',
+    '+503',
+    '+240',
+    '+291',
+    '+372',
+    '+251',
+    '+500',
+    '+298',
+    '+679',
+    '+358',
+    '+33',
+    '+594',
+    '+689',
+    '+241',
+    '+220',
+    '+995',
+    '+49',
+    '+233',
+    '+350',
+    '+30',
+    '+299',
+    '+1473',
+    '+590',
+    '+1671',
+    '+502',
+    '+224',
+    '+245',
+    '+592',
+    '+509',
+    '+504',
+    '+852',
+    '+36',
+    '+354',
+    '+91',
+    '+62',
+    '+964',
+    '+353',
+    '+44',
+    '+972',
+    '+39',
+    '+1876',
+    '+81',
+    '+44',
+    '+962',
+    '+7',
+    '+254',
+    '+686',
+    '+82',
+    '+965',
+    '+996',
+    '+856',
+    '+371',
+    '+961',
+    '+266',
+    '+231',
+    '+423',
+    '+370',
+    '+352',
+    '+853',
+    '+389',
+    '+261',
+    '+265',
+    '+60',
+    '+960',
+    '+223',
+    '+356',
+    '+692',
+    '+596',
+    '+222',
+    '+230',
+    '+262',
+    '+52',
+    '+691',
+    '+373',
+    '+377',
+    '+976',
+    '+382',
+    '+1664',
+    '+212',
+    '+258',
+    '+264',
+    '+674',
+    '+977',
+    '+31',
+    '+687',
+    '+64',
+    '+227',
+    '+234',
+    '+683',
+    '+672',
+    '+1670',
+    '+47',
+    '+968',
+    '+92',
+    '+680',
+    '+507',
+    '+675',
+    '+595',
+    '+51',
+    '+63',
+    '+872',
+    '+48',
+    '+351',
+    '+1939',
+    '+974',
+    '+40',
+    '+250',
+    '+262',
+    '+590',
+    '+290',
+    '+1869',
+    '+1758',
+    '+590',
+    '+508',
+    '+1784',
+    '+685',
+    '+378',
+    '+239',
+    '+966',
+    '+221',
+    '+381',
+    '+248',
+    '+232',
+    '+65',
+    '+421',
+    '+386',
+    '+677',
+    '+27',
+    '+34',
+    '+94',
+    '+597',
+    '+47',
+    '+268',
+    '+46',
+    '+41',
+    '+886',
+    '+992',
+    '+255',
+    '+66',
+    '+670',
+    '+228',
+    '+690',
+    '+676',
+    '+1868',
+    '+216',
+    '+90',
+    '+993',
+    '+1649',
+    '+688',
+    '+256',
+    '+380',
+    '+971',
+    '+44',
+    '+598',
+    '+998',
+    '+678',
+    '+84',
+    '+1284',
+    '+1340',
+    '+681',
+    '+967',
+    '+260'
+  ].freeze
+
+  REGEXP = /\A#{Regexp.union(ALL).source}/i
+end
@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module SearchEntries
+  TRANSLITERATIONS =
+    I18n::Backend::Transliterator::HashTransliterator::DEFAULT_APPROXIMATIONS.reject { |_, v| v.length > 1 }
+
+  MAX_VALUE_LENGTH = 100
+
+  UUID_REGEXP = /\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i
+
+  FIELD_SEARCH_QUERY_SQL = <<~SQL.squish
+    tsvector @@ (quote_literal(coalesce((ts_lexize('english_stem', :keyword))[1], :keyword)) || ':*' || :weight)::tsquery
+  SQL
+
+  module_function
+
+  def reindex_all
+    Submitter.find_each { |submitter| index_submitter(submitter) }
+    Submission.find_each { |submission| index_submission(submission) }
+    Template.find_each { |template| index_template(template) }
+  end
+
+  def enqueue_reindex(records)
+    return unless SearchEntry.table_exists?
+
+    args = Array.wrap(records).map { |e| [{ 'record_type' => e.class.name, 'record_id' => e.id }] }
+
+    ReindexSearchEntryJob.perform_bulk(args)
+  end
+
+  def reindex_record(record)
+    case record
+    when Submitter
+      index_submitter(record)
+    when Template
+      index_template(record)
+    when Submission
+      index_submission(record)
+
+      record.submitters.each do |submitter|
+        index_submitter(submitter)
+      end
+    else
+      raise ArgumentError, 'Invalid Record'
+    end
+  end
+
+  def build_tsquery(keyword)
+    if keyword.match?(/\d/) && !keyword.match?(/\p{L}/)
+      number = keyword.gsub(/\D/, '')
+
+      ["tsvector @@ ((quote_literal(?) || ':*')::tsquery || (quote_literal(?) || ':*')::tsquery || plainto_tsquery(?))",
+       number, number.length > 1 ? number.delete_prefix('0') : number, keyword]
+    elsif keyword.match?(/[^\p{L}\d&@._\-+]/) || keyword.match?(/\A['"].*['"]\z/)
+      ['tsvector @@ plainto_tsquery(?)', TextUtils.transliterate(keyword.downcase)]
+    else
+      [
+        "tsvector @@ (quote_literal(coalesce((ts_lexize('english_stem', :keyword))[1], :keyword)) || ':*')::tsquery",
+        { keyword: TextUtils.transliterate(keyword.downcase).squish }
+      ]
+    end
+  end
+
+  def build_weights_tsquery(terms, weight)
+    last_query = Arel.sql(<<~SQL.squish)
+      (quote_literal(coalesce((ts_lexize('english_stem', :term#{terms.size - 1}))[1], :term#{terms.size - 1})) ||  ':*' || :weight)::tsquery
+    SQL
+
+    query = terms[..-2].reduce(last_query) do |acc, term|
+      index = terms.index(term)
+
+      arel = Arel.sql(<<~SQL.squish)
+        (quote_literal(coalesce((ts_lexize('english_stem', :term#{index}))[1], :term#{index})) ||  ':' || :weight)::tsquery
+      SQL
+
+      Arel::Nodes::InfixOperation.new('&&', arel, acc)
+    end
+
+    ["tsvector @@ (#{query.to_sql})", terms.index_by.with_index { |_, index| :"term#{index}" }.merge(weight:)]
+  end
+
+  def index_submitter(submitter)
+    return if submitter.email.blank? && submitter.phone.blank? && submitter.name.blank?
+
+    sql = SearchEntry.sanitize_sql_array(
+      [
+        "SELECT setweight(to_tsvector(?), 'A') || setweight(to_tsvector(?), 'B') ||
+                setweight(to_tsvector(?), 'C') || setweight(to_tsvector(?), 'D')".squish,
+        [submitter.email.to_s, submitter.email.to_s.split('@').last].join(' ').downcase,
+        [submitter.phone.to_s.gsub(/\D/, ''),
+         submitter.phone.to_s.gsub(PhoneCodes::REGEXP, '').gsub(/\D/, '')].uniq.join(' '),
+        TextUtils.transliterate(submitter.name.to_s.downcase),
+        build_submitter_values_string(submitter)
+      ]
+    )
+
+    entry = submitter.search_entry || submitter.build_search_entry
+
+    entry.account_id = submitter.account_id
+    entry.tsvector = SearchEntry.connection.select_value(sql)
+
+    return if entry.tsvector.blank?
+
+    entry.save!
+
+    entry
+  rescue ActiveRecord::RecordNotUnique
+    submitter.reload
+
+    retry
+  end
+
+  def build_submitter_values_string(submitter)
+    values =
+      submitter.values.values.flatten.filter_map do |v|
+        next if !v.is_a?(String) || v.length > MAX_VALUE_LENGTH || UUID_REGEXP.match?(v)
+
+        TextUtils.transliterate(v)
+      end
+
+    values.uniq.join(' ')
+  end
+
+  def index_template(template)
+    sql = SearchEntry.sanitize_sql_array(
+      ['SELECT to_tsvector(?)', TextUtils.transliterate(template.name.to_s.downcase)]
+    )
+
+    entry = template.search_entry || template.build_search_entry
+
+    entry.account_id = template.account_id
+    entry.tsvector = SearchEntry.connection.select_value(sql)
+
+    return if entry.tsvector.blank?
+
+    entry.save!
+
+    entry
+  rescue ActiveRecord::RecordNotUnique
+    template.reload
+
+    retry
+  end
+
+  def index_submission(submission)
+    return if submission.name.blank?
+
+    sql = SearchEntry.sanitize_sql_array(
+      ['SELECT to_tsvector(?)', TextUtils.transliterate(submission.name.to_s.downcase)]
+    )
+
+    entry = submission.search_entry || submission.build_search_entry
+
+    entry.account_id = submission.account_id
+    entry.tsvector = SearchEntry.connection.select_value(sql)
+
+    return if entry.tsvector.blank?
+
+    entry.save!
+
+    entry
+  rescue ActiveRecord::RecordNotUnique
+    submission.reload
+
+    retry
+  end
+end
@@ -7,7 +7,15 @@ module Submissions

  module_function

-  def search(submissions, keyword, search_values: false, search_template: false)
+  def search(current_user, submissions, keyword, search_values: false, search_template: false)
+    if Docuseal.fulltext_search?(current_user)
+      fulltext_search(current_user, submissions, keyword, search_template:)
+    else
+      plain_search(submissions, keyword, search_values:, search_template:)
+    end
+  end
+
+  def plain_search(submissions, keyword, search_values: false, search_template: false)
    return submissions if keyword.blank?

    term = "%#{keyword.downcase}%"
@@ -29,6 +37,36 @@ module Submissions
    submissions.joins(:submitters).where(arel).group(:id)
  end

+  def fulltext_search(current_user, submissions, keyword, search_template: false)
+    return submissions if keyword.blank?
+
+    arel = SearchEntry.where(record_type: 'Submission')
+                      .where(account_id: current_user.account_id)
+                      .where(*SearchEntries.build_tsquery(keyword))
+                      .select(:record_id).arel
+
+    if search_template
+      arel = Arel::Nodes::Union.new(
+        arel,
+        Submission.where(
+          template_id: SearchEntry.where(record_type: 'Template')
+                                  .where(account_id: current_user.account_id)
+                                  .where(*SearchEntries.build_tsquery(keyword))
+                                  .select(:record_id)
+        ).select(:id).arel
+      )
+    end
+
+    arel = Arel::Nodes::Union.new(
+      arel, Submitter.joins(:search_entry)
+                     .where(search_entry: { account_id: current_user.account_id })
+                     .where(*SearchEntries.build_tsquery(keyword))
+                     .select(:submission_id).arel
+    )
+
+    submissions.where(Submission.arel_table[:id].in(arel))
+  end
+
  def update_template_fields!(submission)
    submission.template_fields = submission.template.fields
    submission.template_schema = submission.template.schema
@@ -4,9 +4,71 @@ module Submitters
  TRUE_VALUES = ['1', 'true', true].freeze
  PRELOAD_ALL_PAGES_AMOUNT = 200

+  FIELD_NAME_WEIGHTS = {
+    'email' => 'A',
+    'phone' => 'B',
+    'name' => 'C',
+    'values' => 'D'
+  }.freeze
+
  module_function

-  def search(submitters, keyword)
+  def search(current_user, submitters, keyword)
+    if Docuseal.fulltext_search?(current_user)
+      fulltext_search(current_user, submitters, keyword)
+    else
+      plain_search(submitters, keyword)
+    end
+  end
+
+  def fulltext_search(current_user, submitters, keyword)
+    return submitters if keyword.blank?
+
+    submitters.where(
+      id: SearchEntry.where(record_type: 'Submitter')
+                     .where(account_id: current_user.account_id)
+                     .where(*SearchEntries.build_tsquery(keyword))
+                     .select(:record_id)
+    )
+  end
+
+  def fulltext_search_field(current_user, submitters, keyword, field_name)
+    return submitters if keyword.blank?
+
+    weight = FIELD_NAME_WEIGHTS[field_name]
+
+    return submitters if weight.blank?
+
+    query =
+      if keyword.match?(/\d/) && !keyword.match?(/\p{L}/)
+        number = keyword.gsub(/\D/, '')
+
+        ["tsvector @@ ((quote_literal(?) || ':*#{weight}')::tsquery || (quote_literal(?) || ':*#{weight}')::tsquery)",
+         number, number.length > 1 ? number.delete_prefix('0') : number]
+      elsif keyword.match?(/[^\p{L}\d&@._\-+]/)
+        terms = TextUtils.transliterate(keyword.downcase).split(/\b/).map(&:squish).compact_blank.uniq
+
+        if terms.size > 1
+          SearchEntries.build_weights_tsquery(terms, weight)
+        else
+          [
+            SearchEntries::FIELD_SEARCH_QUERY_SQL,
+            { keyword: TextUtils.transliterate(keyword.downcase).squish, weight: }
+          ]
+        end
+      else
+        [SearchEntries::FIELD_SEARCH_QUERY_SQL, { keyword: TextUtils.transliterate(keyword.downcase).squish, weight: }]
+      end
+
+    submitters.where(
+      id: SearchEntry.where(record_type: 'Submitter')
+                     .where(account_id: current_user.account_id)
+                     .where(*query)
+                     .select(:record_id)
+    )
+  end
+
+  def plain_search(submitters, keyword)
    return submitters if keyword.blank?

    term = "%#{keyword.downcase}%"
@@ -48,6 +48,8 @@ module Submitters
        submitter.save!
      end

+      SearchEntries.enqueue_reindex(submitter) if submitter.completed_at?
+
      submitter
    end

@@ -37,12 +37,31 @@ module Templates
    hash
  end

-  def search(templates, keyword)
+  def search(current_user, templates, keyword)
+    if Docuseal.fulltext_search?(current_user)
+      fulltext_search(current_user, templates, keyword)
+    else
+      plain_search(templates, keyword)
+    end
+  end
+
+  def plain_search(templates, keyword)
    return templates if keyword.blank?

    templates.where(Template.arel_table[:name].lower.matches("%#{keyword.downcase}%"))
  end

+  def fulltext_search(current_user, templates, keyword)
+    return templates if keyword.blank?
+
+    templates.where(
+      id: SearchEntry.where(record_type: 'Template')
+                     .where(account_id: current_user.account_id)
+                     .where(*SearchEntries.build_tsquery(keyword))
+                     .select(:record_id)
+    )
+  end
+
  def filter_undefined_submitters(template_submitters)
    template_submitters.to_a.select do |item|
      item['invite_by_uuid'].blank? && item['optional_invite_by_uuid'].blank? &&
@@ -5,6 +5,11 @@ module TextUtils
  MASK_REGEXP = /[^\s\-_\[\]\(\)\+\?\.\,]/
  MASK_SYMBOL = 'X'

+  TRANSLITERATIONS =
+    I18n::Backend::Transliterator::HashTransliterator::DEFAULT_APPROXIMATIONS.reject { |_, v| v.length > 1 }
+
+  TRANSLITERATION_REGEXP = Regexp.union(TRANSLITERATIONS.keys)
+
  module_function

  def rtl?(text)
@@ -15,6 +20,10 @@ module TextUtils
    false
  end

+  def transliterate(text)
+    text.to_s.gsub(TRANSLITERATION_REGEXP) { |e| TRANSLITERATIONS[e] }
+  end
+
  def mask_value(text, unmask_size = 0)
    if unmask_size.is_a?(Numeric) && !unmask_size.zero? && unmask_size.abs < text.length
      if unmask_size.negative?