diff --git a/Gemfile b/Gemfile index 7a00911fe..392aa2bfd 100644 --- a/Gemfile +++ b/Gemfile @@ -49,6 +49,7 @@ gem 'daemons' gem 'devise-async' gem 'newrelic_rpm', '~> 3.14' gem 'whenever', require: false +gem 'pg_search' gem 'ahoy_matey', '~> 1.2.1' gem 'groupdate' # group temporary data diff --git a/Gemfile.lock b/Gemfile.lock index 4721ee9de..6ec99b1a7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -263,6 +263,10 @@ GEM paranoia (2.1.3) activerecord (~> 4.0) pg (0.18.3) + pg_search (1.0.5) + activerecord (>= 3.1) + activesupport (>= 3.1) + arel poltergeist (1.7.0) capybara (~> 2.1) cliver (~> 0.3.1) @@ -455,6 +459,7 @@ DEPENDENCIES omniauth-twitter paranoia pg + pg_search poltergeist quiet_assets rails (= 4.2.4) diff --git a/app/models/debate.rb b/app/models/debate.rb index 0b64b7b1e..17560e246 100644 --- a/app/models/debate.rb +++ b/app/models/debate.rb @@ -5,6 +5,7 @@ class Debate < ActiveRecord::Base include Conflictable include Measurable include Sanitizable + include PgSearch apply_simple_captcha acts_as_votable @@ -36,6 +37,22 @@ class Debate < ActiveRecord::Base # Ahoy setup visitable # Ahoy will automatically assign visit_id on create + pg_search_scope :pg_search, { + against: { + title: 'A', + description: 'B' + }, + associated_against: { + tags: :name + }, + using: { + tsearch: { dictionary: "spanish" }, + trigram: { threshold: 0.1 }, + }, + ranked_by: '(:tsearch + debates.cached_votes_up)', + order_within_rank: "debates.created_at DESC" + } + def description super.try :html_safe end @@ -102,12 +119,7 @@ class Debate < ActiveRecord::Base end def self.search(terms) - return none unless terms.present? - - debate_ids = where("debates.title ILIKE ? OR debates.description ILIKE ?", - "%#{terms}%", "%#{terms}%").pluck(:id) - tag_ids = tagged_with(terms, wild: true, any: true).pluck(:id) - where(id: [debate_ids, tag_ids].flatten.compact) + self.pg_search(terms) end def after_hide diff --git a/app/models/proposal.rb b/app/models/proposal.rb index 7d5a5bb17..d56b1fa90 100644 --- a/app/models/proposal.rb +++ b/app/models/proposal.rb @@ -4,6 +4,7 @@ class Proposal < ActiveRecord::Base include Conflictable include Measurable include Sanitizable + include PgSearch apply_simple_captcha acts_as_votable @@ -38,6 +39,24 @@ class Proposal < ActiveRecord::Base scope :sort_by_random, -> { order("RANDOM()") } scope :sort_by_flags, -> { order(flags_count: :desc, updated_at: :desc) } + pg_search_scope :pg_search, { + against: { + title: 'A', + question: 'B', + summary: 'C', + description: 'D' + }, + associated_against: { + tags: :name + }, + using: { + tsearch: { dictionary: "spanish" }, + trigram: { threshold: 0.1 }, + }, + ranked_by: '(:tsearch + proposals.cached_votes_up)', + order_within_rank: "proposals.created_at DESC" + } + def description super.try :html_safe end @@ -93,7 +112,7 @@ class Proposal < ActiveRecord::Base end def self.search(terms) - terms.present? ? where("title ILIKE ? OR description ILIKE ? OR question ILIKE ?", "%#{terms}%", "%#{terms}%", "%#{terms}%") : none + self.pg_search(terms) end def self.votes_needed_for_success diff --git a/db/migrate/20151028213830_add_unaccent_extension.rb b/db/migrate/20151028213830_add_unaccent_extension.rb new file mode 100644 index 000000000..f67f7d365 --- /dev/null +++ b/db/migrate/20151028213830_add_unaccent_extension.rb @@ -0,0 +1,5 @@ +class AddUnaccentExtension < ActiveRecord::Migration + def change + execute "create extension unaccent" + end +end diff --git a/db/migrate/20151028221647_add_pg_trgm_extension.rb b/db/migrate/20151028221647_add_pg_trgm_extension.rb new file mode 100644 index 000000000..f6fca0047 --- /dev/null +++ b/db/migrate/20151028221647_add_pg_trgm_extension.rb @@ -0,0 +1,5 @@ +class AddPgTrgmExtension < ActiveRecord::Migration + def change + execute "create extension pg_trgm" + end +end diff --git a/db/schema.rb b/db/schema.rb index 1a956a453..a5b2ec8c2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,10 +11,12 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20151021113348) do +ActiveRecord::Schema.define(version: 20151028221647) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + enable_extension "unaccent" + enable_extension "pg_trgm" create_table "activities", force: :cascade do |t| t.integer "user_id" diff --git a/spec/features/debates_spec.rb b/spec/features/debates_spec.rb index 737b050e7..e38c7edc7 100644 --- a/spec/features/debates_spec.rb +++ b/spec/features/debates_spec.rb @@ -504,10 +504,12 @@ feature 'Debates' do within("#debates") do expect(page).to have_css('.debate', count: 4) + expect(page).to have_content(debate2.title) expect(page).to have_content(debate4.title) expect(page).to have_content(debate5.title) expect(page).to have_content(debate6.title) + expect(page).to_not have_content(debate1.title) expect(page).to_not have_content(debate3.title) end diff --git a/spec/features/proposals_spec.rb b/spec/features/proposals_spec.rb index 5556ed4d8..ea2ef358c 100644 --- a/spec/features/proposals_spec.rb +++ b/spec/features/proposals_spec.rb @@ -574,9 +574,11 @@ feature 'Proposals' do within("#proposals") do expect(page).to have_css('.proposal', count: 3) + expect(page).to have_content(proposal2.title) expect(page).to have_content(proposal4.title) expect(page).to have_content(proposal5.title) + expect(page).to_not have_content(proposal1.title) expect(page).to_not have_content(proposal3.title) end diff --git a/spec/models/debate_spec.rb b/spec/models/debate_spec.rb index 159ebdbb1..b7f65fdc9 100644 --- a/spec/models/debate_spec.rb +++ b/spec/models/debate_spec.rb @@ -318,35 +318,6 @@ describe Debate do end - describe "self.search" do - it "find debates by title" do - debate1 = create(:debate, title: "Karpov vs Kasparov") - create(:debate, title: "Bird vs Magic") - search = Debate.search("Kasparov") - expect(search.size).to eq(1) - expect(search.first).to eq(debate1) - end - - it "find debates by description" do - debate1 = create(:debate, description: "...chess masters...") - create(:debate, description: "...basket masters...") - search = Debate.search("chess") - expect(search.size).to eq(1) - expect(search.first).to eq(debate1) - end - - it "find debates by title and description" do - create(:debate, title: "Karpov vs Kasparov", description: "...played like Gauss...") - create(:debate, title: "Euler vs Gauss", description: "...math masters...") - search = Debate.search("Gauss") - expect(search.size).to eq(2) - end - - it "returns no results if no search term provided" do - expect(Debate.search(" ").size).to eq(0) - end - end - describe "cache" do let(:debate) { create(:debate) } @@ -457,4 +428,169 @@ describe Debate do end + describe "search" do + + context "attributes" do + + it "searches by title" do + debate = create(:debate, title: 'save the world') + results = Debate.search('save the world') + expect(results).to eq([debate]) + end + + it "searches by description" do + debate = create(:debate, description: 'in order to save the world one must think about...') + results = Debate.search('one must think') + expect(results).to eq([debate]) + end + + end + + context "stemming" do + + it "searches word stems" do + debate = create(:debate, title: 'limpiar') + + results = Debate.search('limpiará') + expect(results).to eq([debate]) + + results = Debate.search('limpiémos') + expect(results).to eq([debate]) + + results = Debate.search('limpió') + expect(results).to eq([debate]) + end + + end + + context "accents" do + + it "searches with accents" do + debate = create(:debate, title: 'difusión') + + results = Debate.search('difusion') + expect(results).to eq([debate]) + + debate2 = create(:debate, title: 'estadisticas') + results = Debate.search('estadísticas') + expect(results).to eq([debate2]) + end + + end + + context "case" do + + it "searches case insensite" do + debate = create(:debate, title: 'SHOUT') + + results = Debate.search('shout') + expect(results).to eq([debate]) + + debate2 = create(:debate, title: "scream") + results = Debate.search("SCREAM") + expect(results).to eq([debate2]) + end + + end + + context "typos" do + + it "searches with typos" do + debate = create(:debate, title: 'difusión') + + results = Debate.search('difuon') + expect(results).to eq([debate]) + + debate2 = create(:debate, title: 'desarrollo') + results = Debate.search('desarolo') + expect(results).to eq([debate2]) + end + + end + + context "order" do + + it "orders by weight" do + debate_description = create(:debate, description: 'stop corruption') + debate_title = create(:debate, title: 'stop corruption') + + results = Debate.search('stop corruption') + + expect(results.first).to eq(debate_title) + expect(results.second).to eq(debate_description) + end + + it "orders by weight and then votes" do + title_some_votes = create(:debate, title: 'stop corruption', cached_votes_up: 5) + title_least_voted = create(:debate, title: 'stop corruption', cached_votes_up: 2) + title_most_voted = create(:debate, title: 'stop corruption', cached_votes_up: 10) + description_most_voted = create(:debate, description: 'stop corruption', cached_votes_up: 10) + + results = Debate.search('stop corruption') + + expect(results.first).to eq(title_most_voted) + expect(results.second).to eq(description_most_voted) + expect(results.third).to eq(title_some_votes) + expect(results.fourth).to eq(title_least_voted) + end + + it "orders by weight and then votes and then created_at" do + newest = create(:debate, title: 'stop corruption', cached_votes_up: 5, created_at: Time.now) + oldest = create(:debate, title: 'stop corruption', cached_votes_up: 5, created_at: 1.month.ago) + old = create(:debate, title: 'stop corruption', cached_votes_up: 5, created_at: 1.week.ago) + + results = Debate.search('stop corruption') + + expect(results.first).to eq(newest) + expect(results.second).to eq(old) + expect(results.third).to eq(oldest) + end + + end + + context "tags" do + + it "searches by tags" do + debate = create(:debate, tag_list: 'Latina') + + results = Debate.search('Latina') + expect(results.first).to eq(debate) + end + + end + + context "no results" do + + it "no words match" do + debate = create(:debate, title: 'save world') + + results = Debate.search('destroy planet') + expect(results).to eq([]) + end + + it "too many typos" do + debate = create(:debate, title: 'fantastic') + + results = Debate.search('frantac') + expect(results).to eq([]) + end + + it "too much stemming" do + debate = create(:debate, title: 'reloj') + + results = Debate.search('superrelojimetro') + expect(results).to eq([]) + end + + it "empty" do + debate = create(:debate, title: 'great') + + results = Debate.search('') + expect(results).to eq([]) + end + + end + + end + end diff --git a/spec/models/proposal_spec.rb b/spec/models/proposal_spec.rb index bb13f0769..8193bf701 100644 --- a/spec/models/proposal_spec.rb +++ b/spec/models/proposal_spec.rb @@ -355,4 +355,185 @@ describe Proposal do end end + describe "search" do + + context "attributes" do + + it "searches by title" do + proposal = create(:proposal, title: 'save the world') + results = Proposal.search('save the world') + expect(results).to eq([proposal]) + end + + it "searches by summary" do + proposal = create(:proposal, summary: 'basically...') + results = Proposal.search('basically') + expect(results).to eq([proposal]) + end + + it "searches by description" do + proposal = create(:proposal, description: 'in order to save the world one must think about...') + results = Proposal.search('one must think') + expect(results).to eq([proposal]) + end + + it "searches by question" do + proposal = create(:proposal, question: 'to be or not to be') + results = Proposal.search('to be or not to be') + expect(results).to eq([proposal]) + end + + end + + context "stemming" do + + it "searches word stems" do + proposal = create(:proposal, summary: 'limpiar') + + results = Proposal.search('limpiará') + expect(results).to eq([proposal]) + + results = Proposal.search('limpiémos') + expect(results).to eq([proposal]) + + results = Proposal.search('limpió') + expect(results).to eq([proposal]) + end + + end + + context "accents" do + + it "searches with accents" do + proposal = create(:proposal, summary: 'difusión') + + results = Proposal.search('difusion') + expect(results).to eq([proposal]) + + proposal2 = create(:proposal, summary: 'estadisticas') + results = Proposal.search('estadísticas') + expect(results).to eq([proposal2]) + end + + end + + context "case" do + + it "searches case insensite" do + proposal = create(:proposal, title: 'SHOUT') + + results = Proposal.search('shout') + expect(results).to eq([proposal]) + + proposal2 = create(:proposal, title: "scream") + results = Proposal.search("SCREAM") + expect(results).to eq([proposal2]) + end + + end + + context "typos" do + + it "searches with typos" do + proposal = create(:proposal, summary: 'difusión') + + results = Proposal.search('difuon') + expect(results).to eq([proposal]) + + proposal2 = create(:proposal, summary: 'desarrollo') + results = Proposal.search('desarolo') + expect(results).to eq([proposal2]) + end + + end + + context "order" do + + it "orders by weight" do + proposal_question = create(:proposal, question: 'stop corruption') + proposal_title = create(:proposal, title: 'stop corruption') + proposal_description = create(:proposal, description: 'stop corruption') + proposal_summary = create(:proposal, summary: 'stop corruption') + + results = Proposal.search('stop corruption') + + expect(results.first).to eq(proposal_title) + expect(results.second).to eq(proposal_question) + expect(results.third).to eq(proposal_summary) + expect(results.fourth).to eq(proposal_description) + end + + it "orders by weight and then votes" do + title_some_votes = create(:proposal, title: 'stop corruption', cached_votes_up: 5) + title_least_voted = create(:proposal, title: 'stop corruption', cached_votes_up: 2) + title_most_voted = create(:proposal, title: 'stop corruption', cached_votes_up: 10) + summary_most_voted = create(:proposal, summary: 'stop corruption', cached_votes_up: 10) + + results = Proposal.search('stop corruption') + + expect(results.first).to eq(title_most_voted) + expect(results.second).to eq(summary_most_voted) + expect(results.third).to eq(title_some_votes) + expect(results.fourth).to eq(title_least_voted) + end + + it "orders by weight and then votes and then created_at" do + newest = create(:proposal, title: 'stop corruption', cached_votes_up: 5, created_at: Time.now) + oldest = create(:proposal, title: 'stop corruption', cached_votes_up: 5, created_at: 1.month.ago) + old = create(:proposal, title: 'stop corruption', cached_votes_up: 5, created_at: 1.week.ago) + + results = Proposal.search('stop corruption') + + expect(results.first).to eq(newest) + expect(results.second).to eq(old) + expect(results.third).to eq(oldest) + end + + end + + context "tags" do + + it "searches by tags" do + proposal = create(:proposal, tag_list: 'Latina') + + results = Proposal.search('Latina') + expect(results.first).to eq(proposal) + end + + end + + context "no results" do + + it "no words match" do + proposal = create(:proposal, title: 'save world') + + results = Proposal.search('destroy planet') + expect(results).to eq([]) + end + + it "too many typos" do + proposal = create(:proposal, title: 'fantastic') + + results = Proposal.search('frantac') + expect(results).to eq([]) + end + + it "too much stemming" do + proposal = create(:proposal, title: 'reloj') + + results = Proposal.search('superrelojimetro') + expect(results).to eq([]) + end + + it "empty" do + proposal = create(:proposal, title: 'great') + + results = Proposal.search('') + expect(results).to eq([]) + end + + end + + end + end