From 25184666a51bc21d4cd6fb010ec769a27a496480 Mon Sep 17 00:00:00 2001 From: rgarcia Date: Wed, 11 Nov 2015 22:27:38 +0100 Subject: [PATCH 1/9] uses tsvector column to search --- app/models/proposal.rb | 16 +++++++++++++++- ...0151111202657_adds_tsvector_update_trigger.rb | 13 +++++++++++++ db/schema.rb | 4 +++- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20151111202657_adds_tsvector_update_trigger.rb diff --git a/app/models/proposal.rb b/app/models/proposal.rb index dd2560ba1..60ef20420 100644 --- a/app/models/proposal.rb +++ b/app/models/proposal.rb @@ -28,6 +28,7 @@ class Proposal < ActiveRecord::Base validates :terms_of_service, acceptance: { allow_nil: false }, on: :create before_validation :set_responsible_name + before_validation :calculate_tsvector before_save :calculate_hot_score, :calculate_confidence_score @@ -50,13 +51,26 @@ class Proposal < ActiveRecord::Base tags: :name }, using: { - tsearch: { dictionary: "spanish" }, + tsearch: { dictionary: "spanish", tsvector_column: 'tsv' }, trigram: { threshold: 0.1 }, }, ranked_by: '(:tsearch + proposals.cached_votes_up)', order_within_rank: "proposals.created_at DESC" } + #remember to add tags.name + def searchable_fields + %w(title question summary description) + end + + def searchable_fields_to_sql + searchable_fields.collect { |field| "coalesce(#{field},'')" }.join(" || ' ' || ") + end + + def calculate_tsvector + ActiveRecord::Base.connection.execute("UPDATE proposals SET tsv = (to_tsvector('spanish', #{searchable_fields_to_sql})) WHERE id = #{self.id}") + end + def description super.try :html_safe end diff --git a/db/migrate/20151111202657_adds_tsvector_update_trigger.rb b/db/migrate/20151111202657_adds_tsvector_update_trigger.rb new file mode 100644 index 000000000..b837f7fc6 --- /dev/null +++ b/db/migrate/20151111202657_adds_tsvector_update_trigger.rb @@ -0,0 +1,13 @@ +class AddsTsvectorUpdateTrigger < ActiveRecord::Migration + + def up + add_column :proposals, :tsv, :tsvector + add_index :products, :tsv, using: "gin" + end + + def down + remove_index :products, :tsv + remove_column :proposals, :tsv + end + +end diff --git a/db/schema.rb b/db/schema.rb index 95bd4a36d..4a4df2497 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20151103194329) do +ActiveRecord::Schema.define(version: 20151111202657) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -215,6 +215,7 @@ ActiveRecord::Schema.define(version: 20151103194329) do t.text "summary" t.string "video_url" t.integer "physical_votes", default: 0 + t.tsvector "tsv" end add_index "proposals", ["author_id", "hidden_at"], name: "index_proposals_on_author_id_and_hidden_at", using: :btree @@ -227,6 +228,7 @@ ActiveRecord::Schema.define(version: 20151103194329) do add_index "proposals", ["question"], name: "index_proposals_on_question", using: :btree add_index "proposals", ["summary"], name: "index_proposals_on_summary", using: :btree add_index "proposals", ["title"], name: "index_proposals_on_title", using: :btree + add_index "proposals", ["tsv"], name: "index_proposals_on_tsv", using: :gin create_table "settings", force: :cascade do |t| t.string "key" From fa83e196caf74ed874717d9a576a6f807a180a33 Mon Sep 17 00:00:00 2001 From: rgarcia Date: Tue, 17 Nov 2015 23:20:18 +0100 Subject: [PATCH 2/9] adds tags to tsvector --- app/models/proposal.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/models/proposal.rb b/app/models/proposal.rb index 60ef20420..84718a5d2 100644 --- a/app/models/proposal.rb +++ b/app/models/proposal.rb @@ -58,17 +58,21 @@ class Proposal < ActiveRecord::Base order_within_rank: "proposals.created_at DESC" } - #remember to add tags.name def searchable_fields - %w(title question summary description) + %w(title question summary description) << tags_to_sql end - def searchable_fields_to_sql + def tags_to_sql + "\'#{tag_list.join(' ')}\'" + end + + def searchable_content searchable_fields.collect { |field| "coalesce(#{field},'')" }.join(" || ' ' || ") end def calculate_tsvector - ActiveRecord::Base.connection.execute("UPDATE proposals SET tsv = (to_tsvector('spanish', #{searchable_fields_to_sql})) WHERE id = #{self.id}") + ActiveRecord::Base.connection.execute(" + UPDATE proposals SET tsv = (to_tsvector('spanish', #{searchable_content})) WHERE id = #{self.id}") end def description From 1de91658c7d6b3776c9cbe5d03bfe5aab3d2c594 Mon Sep 17 00:00:00 2001 From: rgarcia Date: Wed, 18 Nov 2015 10:19:02 +0100 Subject: [PATCH 3/9] extracts cached tsvector to concern --- app/models/concerns/search_cache.rb | 25 +++++++++++++++++++++++++ app/models/proposal.rb | 17 ++--------------- 2 files changed, 27 insertions(+), 15 deletions(-) create mode 100644 app/models/concerns/search_cache.rb diff --git a/app/models/concerns/search_cache.rb b/app/models/concerns/search_cache.rb new file mode 100644 index 000000000..04e3869f8 --- /dev/null +++ b/app/models/concerns/search_cache.rb @@ -0,0 +1,25 @@ +module SearchCache + extend ActiveSupport::Concern + + included do + after_save :calculate_tsvector + end + + def calculate_tsvector + ActiveRecord::Base.connection.execute(" + UPDATE proposals SET tsv = (to_tsvector('spanish', #{fields_to_sql})) WHERE id = #{self.id}") + end + + def fields_to_sql + fields.collect { |field| "coalesce(#{field},'')" }.join(" || ' ' || ") + end + + def fields + searchable_fields << tags_to_sql + end + + def tags_to_sql + "\'#{tag_list.join(' ')}\'" + end + +end \ No newline at end of file diff --git a/app/models/proposal.rb b/app/models/proposal.rb index 84718a5d2..21f379309 100644 --- a/app/models/proposal.rb +++ b/app/models/proposal.rb @@ -5,6 +5,7 @@ class Proposal < ActiveRecord::Base include Measurable include Sanitizable include PgSearch + include SearchCache apply_simple_captcha acts_as_votable @@ -28,7 +29,6 @@ class Proposal < ActiveRecord::Base validates :terms_of_service, acceptance: { allow_nil: false }, on: :create before_validation :set_responsible_name - before_validation :calculate_tsvector before_save :calculate_hot_score, :calculate_confidence_score @@ -59,20 +59,7 @@ class Proposal < ActiveRecord::Base } def searchable_fields - %w(title question summary description) << tags_to_sql - end - - def tags_to_sql - "\'#{tag_list.join(' ')}\'" - end - - def searchable_content - searchable_fields.collect { |field| "coalesce(#{field},'')" }.join(" || ' ' || ") - end - - def calculate_tsvector - ActiveRecord::Base.connection.execute(" - UPDATE proposals SET tsv = (to_tsvector('spanish', #{searchable_content})) WHERE id = #{self.id}") + %w(title question summary description) end def description From c7bf31c588cb7cb02d991bd68d38546546b3d486 Mon Sep 17 00:00:00 2001 From: kikito Date: Wed, 18 Nov 2015 17:00:07 +0100 Subject: [PATCH 4/9] Make search cache order aware & accept values instead of column names --- app/models/concerns/search_cache.rb | 25 +++++++++++++------------ app/models/proposal.rb | 11 +++++++++-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/app/models/concerns/search_cache.rb b/app/models/concerns/search_cache.rb index 04e3869f8..52134aefd 100644 --- a/app/models/concerns/search_cache.rb +++ b/app/models/concerns/search_cache.rb @@ -7,19 +7,20 @@ module SearchCache def calculate_tsvector ActiveRecord::Base.connection.execute(" - UPDATE proposals SET tsv = (to_tsvector('spanish', #{fields_to_sql})) WHERE id = #{self.id}") + UPDATE proposals SET tsv = (#{searchable_values_sql}) WHERE id = #{self.id}") end - def fields_to_sql - fields.collect { |field| "coalesce(#{field},'')" }.join(" || ' ' || ") + private + + def searchable_values_sql + cx = ActiveRecord::Base.connection + arr = [] + searchable_values.each do |val, weight| + if val.present? + arr << "setweight(to_tsvector('spanish', coalesce(#{cx.quote(val)}, '')), #{cx.quote(weight)})" + end + end + arr.join(" || ") end - def fields - searchable_fields << tags_to_sql - end - - def tags_to_sql - "\'#{tag_list.join(' ')}\'" - end - -end \ No newline at end of file +end diff --git a/app/models/proposal.rb b/app/models/proposal.rb index 21f379309..349dd1451 100644 --- a/app/models/proposal.rb +++ b/app/models/proposal.rb @@ -58,8 +58,15 @@ class Proposal < ActiveRecord::Base order_within_rank: "proposals.created_at DESC" } - def searchable_fields - %w(title question summary description) + def searchable_values + values = { + title => 'A', + question => 'B', + summary => 'C', + description => 'D' + } + tag_list.each{ |tag| values[tag] = 'D' } + values end def description From 8188b29e10119366a8fff9edc681984754371772 Mon Sep 17 00:00:00 2001 From: rgarcia Date: Wed, 18 Nov 2015 17:29:09 +0100 Subject: [PATCH 5/9] adds consistent indentation to hash --- app/models/proposal.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/proposal.rb b/app/models/proposal.rb index 349dd1451..1f3518144 100644 --- a/app/models/proposal.rb +++ b/app/models/proposal.rb @@ -60,9 +60,9 @@ class Proposal < ActiveRecord::Base def searchable_values values = { - title => 'A', - question => 'B', - summary => 'C', + title => 'A', + question => 'B', + summary => 'C', description => 'D' } tag_list.each{ |tag| values[tag] = 'D' } From f63500b0c806a22fa8af42186c91b7efb66ad8f7 Mon Sep 17 00:00:00 2001 From: rgarcia Date: Wed, 18 Nov 2015 17:29:40 +0100 Subject: [PATCH 6/9] breaks down method into smaller ones --- app/models/concerns/search_cache.rb | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/models/concerns/search_cache.rb b/app/models/concerns/search_cache.rb index 52134aefd..1d4d5a039 100644 --- a/app/models/concerns/search_cache.rb +++ b/app/models/concerns/search_cache.rb @@ -13,14 +13,15 @@ module SearchCache private def searchable_values_sql - cx = ActiveRecord::Base.connection - arr = [] - searchable_values.each do |val, weight| - if val.present? - arr << "setweight(to_tsvector('spanish', coalesce(#{cx.quote(val)}, '')), #{cx.quote(weight)})" - end - end - arr.join(" || ") + searchable_values.collect { |value, weight| set_tsvector(value, weight) }.join(" || ") end -end + def set_tsvector(value, weight) + "setweight(to_tsvector('spanish', coalesce(#{quote(value)}, '')), #{quote(weight)})" + end + + def quote(value) + ActiveRecord::Base.connection.quote(value) + end + +end \ No newline at end of file From 4be9f7b0655269b91f75abe09a18039927fe73dd Mon Sep 17 00:00:00 2001 From: rgarcia Date: Wed, 18 Nov 2015 17:53:30 +0100 Subject: [PATCH 7/9] renaming rake tasks to touch debates and proposals --- lib/tasks/debates.rake | 2 +- lib/tasks/proposals.rake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/debates.rake b/lib/tasks/debates.rake index ce37e2743..57902dc8f 100644 --- a/lib/tasks/debates.rake +++ b/lib/tasks/debates.rake @@ -1,6 +1,6 @@ namespace :debates do desc "Updates all debates by recalculating their hot_score" - task hot_score: :environment do + task touch: :environment do Debate.find_in_batches do |debates| debates.each(&:save) end diff --git a/lib/tasks/proposals.rake b/lib/tasks/proposals.rake index c2bf8742e..ddba8f913 100644 --- a/lib/tasks/proposals.rake +++ b/lib/tasks/proposals.rake @@ -1,6 +1,6 @@ namespace :proposals do desc "Updates all proposals by recalculating their hot_score" - task hot_score: :environment do + task touch: :environment do Proposal.find_in_batches do |proposals| proposals.each(&:save) end From a34369be4335935854c49a163cfc1980a866bc73 Mon Sep 17 00:00:00 2001 From: rgarcia Date: Wed, 18 Nov 2015 18:03:32 +0100 Subject: [PATCH 8/9] typo --- db/migrate/20151111202657_adds_tsvector_update_trigger.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrate/20151111202657_adds_tsvector_update_trigger.rb b/db/migrate/20151111202657_adds_tsvector_update_trigger.rb index b837f7fc6..ed60a259d 100644 --- a/db/migrate/20151111202657_adds_tsvector_update_trigger.rb +++ b/db/migrate/20151111202657_adds_tsvector_update_trigger.rb @@ -2,11 +2,11 @@ class AddsTsvectorUpdateTrigger < ActiveRecord::Migration def up add_column :proposals, :tsv, :tsvector - add_index :products, :tsv, using: "gin" + add_index :proposals, :tsv, using: "gin" end def down - remove_index :products, :tsv + remove_index :proposals, :tsv remove_column :proposals, :tsv end From 09b0f799b937d54fcf9a835e38ea35091ea5dc0e Mon Sep 17 00:00:00 2001 From: rgarcia Date: Wed, 18 Nov 2015 18:03:39 +0100 Subject: [PATCH 9/9] updates schema --- db/schema.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index 4a4df2497..448fa1a8e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -95,10 +95,10 @@ ActiveRecord::Schema.define(version: 20151111202657) do t.string "visit_id" t.datetime "hidden_at" t.integer "flags_count", default: 0 + t.datetime "ignored_flag_at" t.integer "cached_votes_total", default: 0 t.integer "cached_votes_up", default: 0 t.integer "cached_votes_down", default: 0 - t.datetime "ignored_flag_at" t.integer "comments_count", default: 0 t.datetime "confirmed_hide_at" t.integer "cached_anonymous_votes_total", default: 0 @@ -114,7 +114,6 @@ ActiveRecord::Schema.define(version: 20151111202657) do add_index "debates", ["cached_votes_total"], name: "index_debates_on_cached_votes_total", using: :btree add_index "debates", ["cached_votes_up"], name: "index_debates_on_cached_votes_up", using: :btree add_index "debates", ["confidence_score"], name: "index_debates_on_confidence_score", using: :btree - add_index "debates", ["description"], name: "index_debates_on_description", using: :btree add_index "debates", ["hidden_at"], name: "index_debates_on_hidden_at", using: :btree add_index "debates", ["hot_score"], name: "index_debates_on_hot_score", using: :btree add_index "debates", ["title"], name: "index_debates_on_title", using: :btree @@ -222,7 +221,6 @@ ActiveRecord::Schema.define(version: 20151111202657) do add_index "proposals", ["author_id"], name: "index_proposals_on_author_id", using: :btree add_index "proposals", ["cached_votes_up"], name: "index_proposals_on_cached_votes_up", using: :btree add_index "proposals", ["confidence_score"], name: "index_proposals_on_confidence_score", using: :btree - add_index "proposals", ["description"], name: "index_proposals_on_description", using: :btree add_index "proposals", ["hidden_at"], name: "index_proposals_on_hidden_at", using: :btree add_index "proposals", ["hot_score"], name: "index_proposals_on_hot_score", using: :btree add_index "proposals", ["question"], name: "index_proposals_on_question", using: :btree