From 615bfadca87fc3a22a71c84cc6d06935eb935e4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= Date: Thu, 16 May 2019 10:56:44 +0200 Subject: [PATCH] Add local_census_records importation model This model without database allow us to validate incoming file extension and headers and also does the following during importation process: * Ignore empty rows * Classifiy rows in two groups: created_records, invalid_records --- app/models/local_census_records/import.rb | 84 ++++++++++++++++++ config/locales/en/activemodel.yml | 14 ++- config/locales/es/activemodel.yml | 12 +++ spec/factories/verifications.rb | 6 ++ .../local_census_records/import/invalid.csv | 5 ++ .../import/valid-without-headers.csv | 5 ++ .../local_census_records/import/valid.csv | 5 ++ .../local_census_records/import_spec.rb | 88 +++++++++++++++++++ 8 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 app/models/local_census_records/import.rb create mode 100644 spec/fixtures/files/local_census_records/import/invalid.csv create mode 100644 spec/fixtures/files/local_census_records/import/valid-without-headers.csv create mode 100644 spec/fixtures/files/local_census_records/import/valid.csv create mode 100644 spec/models/local_census_records/import_spec.rb diff --git a/app/models/local_census_records/import.rb b/app/models/local_census_records/import.rb new file mode 100644 index 000000000..70a48737d --- /dev/null +++ b/app/models/local_census_records/import.rb @@ -0,0 +1,84 @@ +require "csv" + +class LocalCensusRecords::Import + include ActiveModel::Model + + ATTRIBUTES = %w[document_type document_number date_of_birth postal_code].freeze + ALLOWED_FILE_EXTENSIONS = %w[csv].freeze + + attr_accessor :file, :created_records, :invalid_records + + validates :file, presence: true + validate :file_extension, if: -> { @file.present? } + validate :file_headers_definition, if: -> { @file.present? && valid_extension? } + + def initialize(attributes = {}) + if attributes.present? + attributes.each do |attr, value| + public_send("#{attr}=", value) + end + end + @created_records = [] + @invalid_records = [] + end + + def save + return false if invalid? + + CSV.open(file.path, headers: true).each do |row| + next if empty_row?(row) + + process_row row + end + true + end + + private + + def process_row(row) + local_census_record = build_local_census_record(row) + + if local_census_record.invalid? + invalid_records << local_census_record + else + local_census_record.save + created_records << local_census_record + end + end + + def build_local_census_record(row) + local_census_record = LocalCensusRecord.new + local_census_record.attributes = row.to_hash.slice(*ATTRIBUTES) + local_census_record + end + + def empty_row?(row) + row.all? { |_, cell| cell.nil? } + end + + def file_extension + return if valid_extension? + + errors.add :file, :extension, valid_extensions: ALLOWED_FILE_EXTENSIONS.join(", ") + end + + def fetch_file_headers + CSV.open(file.path, &:readline) + end + + def file_headers_definition + headers = fetch_file_headers + return if headers.all? {|header| ATTRIBUTES.include? header } && + ATTRIBUTES.all? {|attr| headers.include? attr } + + errors.add :file, :headers, required_headers: ATTRIBUTES.join(", ") + end + + def valid_extension? + ALLOWED_FILE_EXTENSIONS.include? extension + end + + def extension + File.extname(file.original_filename).delete(".") + end +end diff --git a/config/locales/en/activemodel.yml b/config/locales/en/activemodel.yml index fd378e705..47b239115 100644 --- a/config/locales/en/activemodel.yml +++ b/config/locales/en/activemodel.yml @@ -4,6 +4,9 @@ en: verification: residence: "Residence" sms: "SMS" + local_census_record/import: + one: Local census record import + other: Local census records imports attributes: verification: residence: @@ -19,4 +22,13 @@ en: officing/residence: document_type: "Document type" document_number: "Document number (including letters)" - year_of_birth: "Year born" \ No newline at end of file + year_of_birth: "Year born" + local_census_record/import: + file: File + errors: + models: + local_census_records/import: + attributes: + file: + extension: "Given file format is wrong. The allowed file format is: %{valid_extensions}." + headers: "Given file headers are wrong. The file headers must have the following names: %{required_headers}." diff --git a/config/locales/es/activemodel.yml b/config/locales/es/activemodel.yml index c813edfde..155e61cc0 100644 --- a/config/locales/es/activemodel.yml +++ b/config/locales/es/activemodel.yml @@ -4,6 +4,9 @@ es: verification: residence: "Residencia" sms: "SMS" + local_census_records/import: + one: Importación de registros del censo local + other: Importaciones de registros del censo local attributes: verification: residence: @@ -20,3 +23,12 @@ es: document_type: "Tipo de documento" document_number: "Número de documento (incluida letra)" year_of_birth: "Año de nacimiento" + local_census_records/import: + file: Archivo + errors: + models: + local_census_records/import: + attributes: + file: + extension: "El formato del fichero es incorrecto. El formato de archivo permitido es: %{valid_extensions}." + headers: "Las cabeceras del fichero son incorrectas. Las cabeceras del fichero deben tener los nombres siguientes: %{required_headers}." diff --git a/spec/factories/verifications.rb b/spec/factories/verifications.rb index 3649dfb23..6ce249d58 100644 --- a/spec/factories/verifications.rb +++ b/spec/factories/verifications.rb @@ -5,6 +5,12 @@ FactoryBot.define do date_of_birth Date.new(1970, 1, 31) postal_code "28002" end + factory :local_census_records_import, class: "LocalCensusRecords::Import" do + file { + path = %w[spec fixtures files local_census_records import valid.csv] + Rack::Test::UploadedFile.new(Rails.root.join(*path)) + } + end sequence(:document_number) { |n| "#{n.to_s.rjust(8, "0")}X" } diff --git a/spec/fixtures/files/local_census_records/import/invalid.csv b/spec/fixtures/files/local_census_records/import/invalid.csv new file mode 100644 index 000000000..29750d4a7 --- /dev/null +++ b/spec/fixtures/files/local_census_records/import/invalid.csv @@ -0,0 +1,5 @@ +"document_type","document_number","date_of_birth","postal_code" +,"44556678T","07/08/1984",7008 +"DNI",,"07/08/1985",7009 +"Passport","22556678T",,7010 +"NIE","X11556678","07/08/1987", diff --git a/spec/fixtures/files/local_census_records/import/valid-without-headers.csv b/spec/fixtures/files/local_census_records/import/valid-without-headers.csv new file mode 100644 index 000000000..6f8b6ce79 --- /dev/null +++ b/spec/fixtures/files/local_census_records/import/valid-without-headers.csv @@ -0,0 +1,5 @@ +,,, +"44556678T","DNI","07/08/84",7008 +"33556678T","DNI","07/08/84",7008 +"22556678T","DNI","07/08/84",7008 +"X11556678","NIE","07/08/84",7008 diff --git a/spec/fixtures/files/local_census_records/import/valid.csv b/spec/fixtures/files/local_census_records/import/valid.csv new file mode 100644 index 000000000..12cc4bfe8 --- /dev/null +++ b/spec/fixtures/files/local_census_records/import/valid.csv @@ -0,0 +1,5 @@ +"document_type","document_number","date_of_birth","postal_code" +"DNI","44556678T","07/08/1984",7008 +"DNI","33556678T","07/08/1985",7008 +"DNI","22556678T","07/08/1986",7008 +"NIE","X11556678","07/08/1987",7008 diff --git a/spec/models/local_census_records/import_spec.rb b/spec/models/local_census_records/import_spec.rb new file mode 100644 index 000000000..f561007a2 --- /dev/null +++ b/spec/models/local_census_records/import_spec.rb @@ -0,0 +1,88 @@ +require "rails_helper" + +describe LocalCensusRecords::Import do + + let(:base_files_path) { %w[spec fixtures files local_census_records import] } + let(:import) { build(:local_census_records_import) } + + describe "Validations" do + it "is valid" do + expect(import).to be_valid + end + + it "is not valid without a file to import" do + import.file = nil + + expect(import).not_to be_valid + end + + context "When file extension" do + it "is wrong it should not be valid" do + file = Rack::Test::UploadedFile.new("spec/fixtures/files/clippy.gif") + import = build(:local_census_records_import, file: file) + + expect(import).not_to be_valid + end + + it "is csv it should be valid" do + path = base_files_path << "valid.csv" + file = Rack::Test::UploadedFile.new(Rails.root.join(*path)) + import = build(:local_census_records_import, file: file) + + expect(import).to be_valid + end + end + + context "When file headers" do + it "are all missing it should not be valid" do + path = base_files_path << "valid-without-headers.csv" + file = Rack::Test::UploadedFile.new(Rails.root.join(*path)) + import = build(:local_census_records_import, file: file) + + expect(import).not_to be_valid + end + end + end + + context "#save" do + it "Create valid local census records with provided values" do + import.save + local_census_record = LocalCensusRecord.find_by(document_number: "X11556678") + + expect(local_census_record).not_to be_nil + expect(local_census_record.document_type).to eq("NIE") + expect(local_census_record.document_number).to eq("X11556678") + expect(local_census_record.date_of_birth).to eq(Date.parse("07/08/1987")) + expect(local_census_record.postal_code).to eq("7008") + end + + it "Add successfully created local census records to created_records array" do + import.save + + valid_document_numbers = ["44556678T", "33556678T", "22556678T", "X11556678"] + expect(import.created_records.collect(&:document_number)).to eq(valid_document_numbers) + end + + it "Add invalid local census records to invalid_records array" do + path = base_files_path << "invalid.csv" + file = Rack::Test::UploadedFile.new(Rails.root.join(*path)) + import.file = file + + import.save + + invalid_records_document_types = [nil, "DNI", "Passport", "NIE"] + invalid_records_document_numbers = ["44556678T", nil, "22556678T", "X11556678"] + invalid_records_date_of_births = [Date.parse("07/08/1984"), Date.parse("07/08/1985"), nil, + Date.parse("07/08/1987")] + invalid_records_postal_codes = ["7008", "7009", "7010", nil] + expect(import.invalid_records.collect(&:document_type)) + .to eq(invalid_records_document_types) + expect(import.invalid_records.collect(&:document_number)) + .to eq(invalid_records_document_numbers) + expect(import.invalid_records.collect(&:date_of_birth)) + .to eq(invalid_records_date_of_births) + expect(import.invalid_records.collect(&:postal_code)) + .to eq(invalid_records_postal_codes) + end + end +end