From be886ba77a7a666226f5c3c7f086e03491f9e77e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?=
 <senenrodero@gmail.com>
Date: Thu, 7 Nov 2019 13:20:45 +0100
Subject: [PATCH 1/2] Add rake task to remove duplicated local census records

Also supress migration messages during spec execution to keep test log as clean
as possible.
---
 lib/tasks/local_census_records.rake         | 13 ++++++++
 spec/lib/tasks/local_census_records_spec.rb | 36 +++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 lib/tasks/local_census_records.rake
 create mode 100644 spec/lib/tasks/local_census_records_spec.rb

diff --git a/lib/tasks/local_census_records.rake b/lib/tasks/local_census_records.rake
new file mode 100644
index 000000000..64ad7d383
--- /dev/null
+++ b/lib/tasks/local_census_records.rake
@@ -0,0 +1,13 @@
+namespace :local_census_records do
+  desc "Remove duplicated records from database"
+  task remove_duplicates: :environment do
+    ids = LocalCensusRecord.group(:document_type, :document_number).pluck("MIN(id) as id")
+    duplicates = LocalCensusRecord.count - ids.size
+
+    if duplicates > 0
+      ApplicationLogger.new.info "Removing local census records duplicates"
+      LocalCensusRecord.where("id NOT IN (?)", ids).destroy_all
+      ApplicationLogger.new.info "Removed #{duplicates} records."
+    end
+  end
+end
diff --git a/spec/lib/tasks/local_census_records_spec.rb b/spec/lib/tasks/local_census_records_spec.rb
new file mode 100644
index 000000000..cb0e3c00c
--- /dev/null
+++ b/spec/lib/tasks/local_census_records_spec.rb
@@ -0,0 +1,36 @@
+require "rails_helper"
+require Rails.root.join("db", "migrate", "20190530082138_add_unique_index_to_local_census_records")
+
+describe "LocalCensusRecord tasks" do
+  let(:run_rake_task) do
+    Rake::Task["local_census_records:remove_duplicates"].reenable
+    Rake.application.invoke_task("local_census_records:remove_duplicates")
+  end
+
+  describe "#remove_duplicates" do
+    around do |example|
+      ActiveRecord::Migration.suppress_messages do
+        example.run
+      end
+    end
+
+    before { AddUniqueIndexToLocalCensusRecords.new.down }
+    after { AddUniqueIndexToLocalCensusRecords.new.up }
+
+    it "Remove all duplicates keeping older records" do
+      record1 = create(:local_census_record, document_type: "1", document_number: "#DOCUMENT_NUMBER")
+      record2 = create(:local_census_record, document_type: "2", document_number: "#DOCUMENT_NUMBER")
+      dup_record1 = build(:local_census_record, document_type: "1", document_number: "#DOCUMENT_NUMBER")
+      dup_record1.save!(validate: false)
+      dup_record2 = build(:local_census_record, document_type: "2", document_number: "#DOCUMENT_NUMBER")
+      dup_record2.save!(validate: false)
+      record3 = create(:local_census_record, document_type: "3", document_number: "#DOCUMENT_NUMBER")
+
+      expect(LocalCensusRecord.count).to eq(5)
+
+      run_rake_task
+
+      expect(LocalCensusRecord.all).to match_array([record1, record2, record3])
+    end
+  end
+end

From 15b4ff64f4ea8c8683801f65938f78cab4e3d256 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?=
 <senenrodero@gmail.com>
Date: Thu, 7 Nov 2019 13:23:18 +0100
Subject: [PATCH 2/2] Run local census records remove_duplicates rake task
 during deployment

 ... and before applying new migrations.
---
 config/deploy.rb | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/config/deploy.rb b/config/deploy.rb
index 21a0259ba..3129cfd38 100644
--- a/config/deploy.rb
+++ b/config/deploy.rb
@@ -43,6 +43,7 @@ namespace :deploy do
   before :starting, "rvm1:install:rvm"
   before :starting, "rvm1:install:ruby"
   before :starting, "install_bundler_gem"
+  before "deploy:migrate", "remove_local_census_records_duplicates"
 
   after "deploy:migrate", "add_new_settings"
   after :publishing, "deploy:restart"
@@ -68,6 +69,16 @@ task :install_bundler_gem do
   end
 end
 
+task :remove_local_census_records_duplicates do
+  on roles(:db) do
+    within release_path do
+      with rails_env: fetch(:rails_env) do
+        execute :rake, "local_census_records:remove_duplicates"
+      end
+    end
+  end
+end
+
 task :refresh_sitemap do
   on roles(:app) do
     within release_path do