From bc9471b49ef16342f4ced5b433f6f63f055b9ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Sun, 13 Oct 2019 01:11:20 +0200 Subject: [PATCH 1/7] Define SMTP configuration in the secrets file Since SMTP passwords should not be in a file under version control, and they're usually configured in the production.rb file (which is under version control), the natural place to configure it is the secrets.yml file. Until now we were using the capistrano shared folder, but that's a bit inconvenient since changes we've done to the production.rb file (like changing eager_load_paths when we upgraded to Rails 5) won't take effect after a deployment. --- config/deploy.rb | 2 +- config/environments/preproduction.rb | 16 +++++----------- config/environments/production.rb | 16 +++++----------- config/environments/staging.rb | 16 +++++----------- config/secrets.yml.example | 18 ++++++++++++++++++ 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 3129cfd38..5689f7e26 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -21,7 +21,7 @@ set :log_level, :info set :pty, true set :use_sudo, false -set :linked_files, %w[config/database.yml config/secrets.yml config/environments/production.rb] +set :linked_files, %w[config/database.yml config/secrets.yml] set :linked_dirs, %w[log tmp public/system public/assets public/ckeditor_assets] set :keep_releases, 5 diff --git a/config/environments/preproduction.rb b/config/environments/preproduction.rb index 78a443249..17e1b9439 100644 --- a/config/environments/preproduction.rb +++ b/config/environments/preproduction.rb @@ -71,17 +71,11 @@ Rails.application.configure do config.action_mailer.default_url_options = { host: Rails.application.secrets.server_name } config.action_mailer.asset_host = "https://#{Rails.application.secrets.server_name}" - # SMTP configuration to deliver emails - # Uncomment the following block of code and add your SMTP service credentials - # config.action_mailer.delivery_method = :smtp - # config.action_mailer.smtp_settings = { - # address: "smtp.example.com", - # port: 587, - # domain: "example.com", - # user_name: "", - # password: "", - # authentication: "plain", - # enable_starttls_auto: true } + # Configure your SMTP service credentials in secrets.yml + if Rails.application.secrets.smtp_settings + config.action_mailer.delivery_method = Rails.application.secrets.mailer_delivery_method || :smtp + config.action_mailer.smtp_settings = Rails.application.secrets.smtp_settings + end # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # the I18n.default_locale when a translation cannot be found). diff --git a/config/environments/production.rb b/config/environments/production.rb index 6c23b8ab5..245973eb9 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -72,17 +72,11 @@ Rails.application.configure do config.action_mailer.default_url_options = { host: Rails.application.secrets.server_name } config.action_mailer.asset_host = "https://#{Rails.application.secrets.server_name}" - # SMTP configuration to deliver emails - # Uncomment the following block of code and add your SMTP service credentials - # config.action_mailer.delivery_method = :smtp - # config.action_mailer.smtp_settings = { - # address: "smtp.example.com", - # port: 587, - # domain: "example.com", - # user_name: "", - # password: "", - # authentication: "plain", - # enable_starttls_auto: true } + # Configure your SMTP service credentials in secrets.yml + if Rails.application.secrets.smtp_settings + config.action_mailer.delivery_method = Rails.application.secrets.mailer_delivery_method || :smtp + config.action_mailer.smtp_settings = Rails.application.secrets.smtp_settings + end # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # the I18n.default_locale when a translation cannot be found). diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 8e64e9fae..65a15f2b5 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -71,17 +71,11 @@ Rails.application.configure do config.action_mailer.default_url_options = { host: Rails.application.secrets.server_name } config.action_mailer.asset_host = "https://#{Rails.application.secrets.server_name}" - # SMTP configuration to deliver emails - # Uncomment the following block of code and add your SMTP service credentials - # config.action_mailer.delivery_method = :smtp - # config.action_mailer.smtp_settings = { - # address: "smtp.example.com", - # port: 587, - # domain: "example.com", - # user_name: "", - # password: "", - # authentication: "plain", - # enable_starttls_auto: true } + # Configure your SMTP service credentials in secrets.yml + if Rails.application.secrets.smtp_settings + config.action_mailer.delivery_method = Rails.application.secrets.mailer_delivery_method || :smtp + config.action_mailer.smtp_settings = Rails.application.secrets.smtp_settings + end # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # the I18n.default_locale when a translation cannot be found). diff --git a/config/secrets.yml.example b/config/secrets.yml.example index 3a8678766..d0c896c0f 100644 --- a/config/secrets.yml.example +++ b/config/secrets.yml.example @@ -44,6 +44,15 @@ staging: preproduction: secret_key_base: "" server_name: "" + # mailer_delivery_method: "smtp" + # smtp_settings: + # address: "smtp.example.com" + # port: 25 + # domain: "your_domain.com" + # user_name: "" + # password: "" + # authentication: "plain" + # enable_starttls_auto: true rollbar_server_token: "" http_basic_username: "" http_basic_password: "" @@ -61,6 +70,15 @@ preproduction: production: secret_key_base: "" server_name: "" + # mailer_delivery_method: "smtp" + # smtp_settings: + # address: "smtp.example.com" + # port: 25 + # domain: "your_domain.com" + # user_name: "" + # password: "" + # authentication: "plain" + # enable_starttls_auto: true rollbar_server_token: "" http_basic_username: "" http_basic_password: "" From 03c427552540d3d55c2859fe88c86789bf59d90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Wed, 16 Oct 2019 14:29:45 +0200 Subject: [PATCH 2/7] Add task to move SMTP settings to secrets.yml Existing installations having their configuration settings in the capistrano shared folder needed this migration. Note we can't just use `YAML.load` because we'd lose the anchors defined in the file. So we have to parse the file the hard way. --- config/deploy.rb | 13 +++++++++++++ lib/tasks/secrets.rake | 23 +++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 lib/tasks/secrets.rake diff --git a/config/deploy.rb b/config/deploy.rb index 5689f7e26..c3a875367 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -46,6 +46,9 @@ namespace :deploy do before "deploy:migrate", "remove_local_census_records_duplicates" after "deploy:migrate", "add_new_settings" + + before :publishing, "smtp_secrets" + after :publishing, "deploy:restart" after :published, "delayed_job:restart" after :published, "refresh_sitemap" @@ -127,3 +130,13 @@ task :setup_puma do end end end + +task :smtp_secrets do + on roles(:app) do + within release_path do + with rails_env: fetch(:rails_env) do + execute :rake, "secrets:smtp" + end + end + end +end diff --git a/lib/tasks/secrets.rake b/lib/tasks/secrets.rake new file mode 100644 index 000000000..0b7ed7c6b --- /dev/null +++ b/lib/tasks/secrets.rake @@ -0,0 +1,23 @@ +namespace :secrets do + desc "Add SMTP settings to secrets.yml" + task smtp: :environment do + exit if Rails.application.secrets.smtp_settings + + current_settings = { + "mailer_delivery_method" => ActionMailer::Base.delivery_method.to_s, + "smtp_settings" => ActionMailer::Base.smtp_settings.stringify_keys + } + + secrets = Rails.application.config.paths["config/secrets"].first + stream = Psych.parse_stream(File.read(secrets)) + nodes = stream.children.first.children.first + + environment_index = nodes.children.index do |child| + child.is_a?(Psych::Nodes::Scalar) && child.value == Rails.env + end + + nodes.children[environment_index + 1].children.push(*Psych.parse(current_settings.to_yaml).children.first.children) + + File.open(secrets, "w") { |file| file.write stream.to_yaml } + end +end From df4f8810c5e30add875b8f53688679e33431f02f Mon Sep 17 00:00:00 2001 From: Julian Herrero Date: Wed, 30 Oct 2019 20:17:39 +0700 Subject: [PATCH 3/7] Define `force_ssl` configuration in secrets file this is usually configured in the production.rb file (which is under version control), the natural place to configure it is the secrets.yml file. Until now we were using the capistrano shared folder, but that's a bit inconvenient since changes we've done to the production.rb file (like changing eager_load_paths when we upgraded to Rails 5) won't take effect after a deployment. --- config/environments/preproduction.rb | 3 ++- config/environments/production.rb | 3 ++- config/environments/staging.rb | 3 ++- config/secrets.yml.example | 3 +++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/config/environments/preproduction.rb b/config/environments/preproduction.rb index 17e1b9439..39c060f13 100644 --- a/config/environments/preproduction.rb +++ b/config/environments/preproduction.rb @@ -47,7 +47,8 @@ Rails.application.configure do # config.action_dispatch.x_sendfile_header = "X-Accel-Redirect" # for NGINX # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = true + # Configure force_ssl in secrets.yml + config.force_ssl = Rails.application.secrets.force_ssl # Use the lowest log level to ensure availability of diagnostic information # when problems arise. diff --git a/config/environments/production.rb b/config/environments/production.rb index 245973eb9..707331124 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -49,7 +49,8 @@ Rails.application.configure do # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = true + # Configure force_ssl in secrets.yml + config.force_ssl = Rails.application.secrets.force_ssl # Use the lowest log level to ensure availability of diagnostic information # when problems arise. diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 65a15f2b5..f47529f64 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -47,7 +47,8 @@ Rails.application.configure do # config.action_dispatch.x_sendfile_header = "X-Accel-Redirect" # for NGINX # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = true + # Configure force_ssl in secrets.yml + config.force_ssl = Rails.application.secrets.force_ssl # Use the lowest log level to ensure availability of diagnostic information # when problems arise. diff --git a/config/secrets.yml.example b/config/secrets.yml.example index d0c896c0f..05870b09d 100644 --- a/config/secrets.yml.example +++ b/config/secrets.yml.example @@ -32,6 +32,7 @@ test: staging: secret_key_base: "" server_name: "" + force_ssl: true rollbar_server_token: "" http_basic_username: "" http_basic_password: "" @@ -53,6 +54,7 @@ preproduction: # password: "" # authentication: "plain" # enable_starttls_auto: true + force_ssl: true rollbar_server_token: "" http_basic_username: "" http_basic_password: "" @@ -79,6 +81,7 @@ production: # password: "" # authentication: "plain" # enable_starttls_auto: true + force_ssl: true rollbar_server_token: "" http_basic_username: "" http_basic_password: "" From d6130cf6e2e287490f90b4774b96bb71c14053ba Mon Sep 17 00:00:00 2001 From: Julian Herrero Date: Wed, 30 Oct 2019 20:41:35 +0700 Subject: [PATCH 4/7] Add task to move force_ssl settings to secrets.yml Existing installations having their configuration settings in the capistrano shared folder needed this migration. --- config/deploy.rb | 6 +++--- lib/tasks/secrets.rake | 17 +++++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index c3a875367..6dbbca2d4 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -47,7 +47,7 @@ namespace :deploy do after "deploy:migrate", "add_new_settings" - before :publishing, "smtp_secrets" + before :publishing, "smtp_and_ssl_secrets" after :publishing, "deploy:restart" after :published, "delayed_job:restart" @@ -131,11 +131,11 @@ task :setup_puma do end end -task :smtp_secrets do +task :smtp_and_ssl_secrets do on roles(:app) do within release_path do with rails_env: fetch(:rails_env) do - execute :rake, "secrets:smtp" + execute :rake, "secrets:smtp_and_ssl" end end end diff --git a/lib/tasks/secrets.rake b/lib/tasks/secrets.rake index 0b7ed7c6b..e3b19d265 100644 --- a/lib/tasks/secrets.rake +++ b/lib/tasks/secrets.rake @@ -1,13 +1,18 @@ namespace :secrets do - desc "Add SMTP settings to secrets.yml" - task smtp: :environment do - exit if Rails.application.secrets.smtp_settings - + desc "Add SMTP and SSL settings to secrets.yml" + task smtp_and_ssl: :environment do current_settings = { "mailer_delivery_method" => ActionMailer::Base.delivery_method.to_s, - "smtp_settings" => ActionMailer::Base.smtp_settings.stringify_keys + "smtp_settings" => ActionMailer::Base.smtp_settings.stringify_keys, + "force_ssl" => Rails.application.config.force_ssl } + settings_to_add = current_settings.select do |name, _| + Rails.application.secrets[name].nil? + end + + exit if settings_to_add.empty? + secrets = Rails.application.config.paths["config/secrets"].first stream = Psych.parse_stream(File.read(secrets)) nodes = stream.children.first.children.first @@ -16,7 +21,7 @@ namespace :secrets do child.is_a?(Psych::Nodes::Scalar) && child.value == Rails.env end - nodes.children[environment_index + 1].children.push(*Psych.parse(current_settings.to_yaml).children.first.children) + nodes.children[environment_index + 1].children.push(*Psych.parse(settings_to_add.to_yaml).children.first.children) File.open(secrets, "w") { |file| file.write stream.to_yaml } end From 6ecd9e59dce57287c05c3b99c9b6cea893047541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Sat, 2 Nov 2019 13:26:32 +0100 Subject: [PATCH 5/7] Update secrets based on the previous release We were copying the current SMTP and SSL settings to the secrets file after overwriting them, but we need to copy them before overwriting them. The workaround I've found is to copy the tasks to the folder of the previous release and execute them there. --- config/deploy.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 6dbbca2d4..f5c8c5ef7 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -133,9 +133,19 @@ end task :smtp_and_ssl_secrets do on roles(:app) do - within release_path do + within current_path do with rails_env: fetch(:rails_env) do - execute :rake, "secrets:smtp_and_ssl" + tasks_file_path = "lib/tasks/secrets.rake" + + unless test("[ -e #{current_path}/#{tasks_file_path} ]") + begin + execute "cp #{release_path}/#{tasks_file_path} #{current_path}/#{tasks_file_path}" + + execute :rake, "secrets:smtp_and_ssl" + ensure + execute "rm #{current_path}/#{tasks_file_path}" + end + end end end end From a08d42d3f8b178425ef86677fd55ff416f5d0637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Sat, 2 Nov 2019 21:58:28 +0100 Subject: [PATCH 6/7] Enable/disable delayed jobs in the secrets file While this is not a secret and in theory should be in a file under version control, currently the CONSUL installer disables delayed jobs by default, meaning we were keeping two versions of the delayed jobs configuration file, and some existing configurations have their settings defined in a file in capistrano's `shared` folder. So we're moving existing settings to the secrets file. --- config/deploy.rb | 6 +++--- config/initializers/delayed_job_config.rb | 5 ++++- config/secrets.yml.example | 3 +++ lib/tasks/secrets.rake | 7 ++++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index f5c8c5ef7..1112257cb 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -47,7 +47,7 @@ namespace :deploy do after "deploy:migrate", "add_new_settings" - before :publishing, "smtp_and_ssl_secrets" + before :publishing, "smtp_ssl_and_delay_jobs_secrets" after :publishing, "deploy:restart" after :published, "delayed_job:restart" @@ -131,7 +131,7 @@ task :setup_puma do end end -task :smtp_and_ssl_secrets do +task :smtp_ssl_and_delay_jobs_secrets do on roles(:app) do within current_path do with rails_env: fetch(:rails_env) do @@ -141,7 +141,7 @@ task :smtp_and_ssl_secrets do begin execute "cp #{release_path}/#{tasks_file_path} #{current_path}/#{tasks_file_path}" - execute :rake, "secrets:smtp_and_ssl" + execute :rake, "secrets:smtp_ssl_and_delay_jobs" ensure execute "rm #{current_path}/#{tasks_file_path}" end diff --git a/config/initializers/delayed_job_config.rb b/config/initializers/delayed_job_config.rb index 151511dc6..3bcbfa4a1 100644 --- a/config/initializers/delayed_job_config.rb +++ b/config/initializers/delayed_job_config.rb @@ -1,8 +1,11 @@ if Rails.env.test? || Rails.env.development? Delayed::Worker.delay_jobs = false -else +elsif Rails.application.secrets.delay_jobs.nil? Delayed::Worker.delay_jobs = true +else + Delayed::Worker.delay_jobs = Rails.application.secrets.delay_jobs end + Delayed::Worker.destroy_failed_jobs = false Delayed::Worker.sleep_delay = 2 Delayed::Worker.max_attempts = 3 diff --git a/config/secrets.yml.example b/config/secrets.yml.example index 05870b09d..b9f227a46 100644 --- a/config/secrets.yml.example +++ b/config/secrets.yml.example @@ -33,6 +33,7 @@ staging: secret_key_base: "" server_name: "" force_ssl: true + delay_jobs: true rollbar_server_token: "" http_basic_username: "" http_basic_password: "" @@ -55,6 +56,7 @@ preproduction: # authentication: "plain" # enable_starttls_auto: true force_ssl: true + delay_jobs: true rollbar_server_token: "" http_basic_username: "" http_basic_password: "" @@ -82,6 +84,7 @@ production: # authentication: "plain" # enable_starttls_auto: true force_ssl: true + delay_jobs: true rollbar_server_token: "" http_basic_username: "" http_basic_password: "" diff --git a/lib/tasks/secrets.rake b/lib/tasks/secrets.rake index e3b19d265..25d66d3c3 100644 --- a/lib/tasks/secrets.rake +++ b/lib/tasks/secrets.rake @@ -1,10 +1,11 @@ namespace :secrets do - desc "Add SMTP and SSL settings to secrets.yml" - task smtp_and_ssl: :environment do + desc "Add SMTP, SSL and delay jobs settings to secrets.yml" + task smtp_ssl_and_delay_jobs: :environment do current_settings = { "mailer_delivery_method" => ActionMailer::Base.delivery_method.to_s, "smtp_settings" => ActionMailer::Base.smtp_settings.stringify_keys, - "force_ssl" => Rails.application.config.force_ssl + "force_ssl" => Rails.application.config.force_ssl, + "delay_jobs" => Delayed::Worker.delay_jobs } settings_to_add = current_settings.select do |name, _| From d7aab4c929ecb4cefb88652ff8bb86f24dbbd6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Mon, 11 Nov 2019 11:40:57 +0100 Subject: [PATCH 7/7] Check permissions for secrets file Old versions of the installer created this file as root, making it impossible to change it as a regular user. So for old installations we need to make sure we've got write access to this file. We're using `sudo` because in these applications the installer gives `sudo` access to the deploy user, so everything works fine with the default configuration. --- config/deploy.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index 1112257cb..7a10e80ed 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -136,9 +136,15 @@ task :smtp_ssl_and_delay_jobs_secrets do within current_path do with rails_env: fetch(:rails_env) do tasks_file_path = "lib/tasks/secrets.rake" + shared_secrets_path = "#{shared_path}/config/secrets.yml" unless test("[ -e #{current_path}/#{tasks_file_path} ]") begin + unless test("[ -w #{shared_secrets_path} ]") + execute "sudo chown `whoami` #{shared_secrets_path}" + execute "chmod u+w #{shared_secrets_path}" + end + execute "cp #{release_path}/#{tasks_file_path} #{current_path}/#{tasks_file_path}" execute :rake, "secrets:smtp_ssl_and_delay_jobs"