diff --git a/.gitignore b/.gitignore index 21e96def9..d76769206 100644 --- a/.gitignore +++ b/.gitignore @@ -37,5 +37,6 @@ public/sitemap.xml public/assets/ +public/machine_learning/data/ public/system/ /public/ckeditor_assets/ diff --git a/app/assets/javascripts/admin/machine_learning/scripts.js b/app/assets/javascripts/admin/machine_learning/scripts.js new file mode 100644 index 000000000..49e9298e0 --- /dev/null +++ b/app/assets/javascripts/admin/machine_learning/scripts.js @@ -0,0 +1,15 @@ +(function() { + "use strict"; + App.AdminMachineLearningScripts = { + initialize: function() { + $(".admin .machine-learning-scripts select").on({ + change: function() { + var element = document.getElementById($(this).val()); + + $("#script_descriptions > *").not(element).addClass("hide"); + $(element).removeClass("hide"); + } + }).trigger("change"); + } + }; +}).call(this); diff --git a/app/assets/javascripts/application.js b/app/assets/javascripts/application.js index 51a7a4d55..8ff6949cb 100644 --- a/app/assets/javascripts/application.js +++ b/app/assets/javascripts/application.js @@ -168,6 +168,7 @@ var initialize_modules = function() { App.ColumnsSelector.initialize(); } App.AdminBudgetsWizardCreationStep.initialize(); + App.AdminMachineLearningScripts.initialize(); App.BudgetEditAssociations.initialize(); App.Datepicker.initialize(); App.SDGRelatedListSelector.initialize(); diff --git a/app/assets/stylesheets/admin/machine_learning/help.scss b/app/assets/stylesheets/admin/machine_learning/help.scss new file mode 100644 index 000000000..5279a9e63 --- /dev/null +++ b/app/assets/stylesheets/admin/machine_learning/help.scss @@ -0,0 +1,7 @@ +.admin .machine-learning-help { + + > code { + display: block; + white-space: pre-wrap; + } +} diff --git a/app/assets/stylesheets/admin/machine_learning/scripts.scss b/app/assets/stylesheets/admin/machine_learning/scripts.scss new file mode 100644 index 000000000..ad9e738bc --- /dev/null +++ b/app/assets/stylesheets/admin/machine_learning/scripts.scss @@ -0,0 +1,60 @@ +.admin .machine-learning-scripts { + + .alert > :first-child { + @include has-fa-icon(ban, solid); + } + + .success > :first-child { + @include has-fa-icon(check-circle, solid); + } + + .warning > :first-child { + @include has-fa-icon(hourglass-half, solid); + } + + .alert, + .success, + .warning { + > :first-child::before { + margin-right: $font-icon-margin; + } + } + + dl { + font-size: $base-font-size; + + dt, + dd { + display: inline; + } + + * + dt::before { + content: ""; + display: block; + } + } + + form { + max-width: $global-width * 3 / 4; + } + + select { + max-width: 100%; + width: auto; + } + + [type=submit] { + @include regular-button; + display: block; + margin-bottom: 0; + margin-top: $line-height; + + &:focus { + outline: $outline-focus; + } + + &.cancel { + @include hollow-button($alert-color); + } + } +} diff --git a/app/assets/stylesheets/admin/machine_learning/setting.scss b/app/assets/stylesheets/admin/machine_learning/setting.scss new file mode 100644 index 000000000..d81b96210 --- /dev/null +++ b/app/assets/stylesheets/admin/machine_learning/setting.scss @@ -0,0 +1,68 @@ +.admin .machine-learning-setting { + + .card-divider { + background: $primary-color; + color: $white; + + h3 { + margin-top: 0; + } + } + + [aria-pressed] { + @include regular-button; + border-radius: $line-height; + font-weight: bold; + min-width: rem-calc(100); + position: relative; + + &:focus { + outline: $outline-focus; + } + + &::after { + background: $white; + border-radius: 100%; + content: ""; + display: block; + height: 1.75em; + position: absolute; + transform: translateY(-50%); + top: 50%; + width: 1.75em; + } + + &[aria-pressed=true] { + background: $primary-color; + padding-right: 2.5em; + text-align: left; + + &::after { + right: 0.5em; + } + } + + &[aria-pressed=false] { + background: $dark-gray; + padding-left: 2.5em; + text-align: right; + + &::after { + left: 0.5em; + } + } + } + + dl { + @include callout-size(map-get($callout-sizes, small)); + } + + dt { + font-weight: normal; + margin-bottom: 0; + } + + * + dt { + margin-top: $line-height; + } +} diff --git a/app/assets/stylesheets/admin/machine_learning/settings.scss b/app/assets/stylesheets/admin/machine_learning/settings.scss new file mode 100644 index 000000000..788cc95e6 --- /dev/null +++ b/app/assets/stylesheets/admin/machine_learning/settings.scss @@ -0,0 +1,24 @@ +.admin .machine-learning-settings { + + .settings-management { + $gap: rem-calc(map-get($grid-column-gutter, medium)); + display: flex; + flex-wrap: wrap; + margin-left: -$gap; + + > * { + margin-left: $gap; + flex-basis: calc((#{rem-calc(720)} - 100%) * 999); + flex-grow: 1; + } + + .card-section { + display: flex; + flex-direction: column; + + > :last-child { + margin-top: auto; + } + } + } +} diff --git a/app/assets/stylesheets/admin/machine_learning/show.scss b/app/assets/stylesheets/admin/machine_learning/show.scss new file mode 100644 index 000000000..d0024725f --- /dev/null +++ b/app/assets/stylesheets/admin/machine_learning/show.scss @@ -0,0 +1,6 @@ +.admin { + + .experimental-feature { + @include has-fa-icon(flask, solid); + } +} diff --git a/app/assets/stylesheets/admin/menu.scss b/app/assets/stylesheets/admin/menu.scss index b0d72571e..f0d407add 100644 --- a/app/assets/stylesheets/admin/menu.scss +++ b/app/assets/stylesheets/admin/menu.scss @@ -108,6 +108,10 @@ &.users-link { @include icon(user, solid); } + + &.ml-link { + @include icon(brain, solid); + } } li { diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index ff9ea349f..f438ea3a2 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -40,6 +40,7 @@ @import "budgets/**/*"; @import "debates/**/*"; @import "layout/**/*"; +@import "machine_learning/**/*"; @import "proposals/**/*"; @import "relationable/**/*"; @import "sdg/**/*"; diff --git a/app/assets/stylesheets/machine_learning/comments_summary.scss b/app/assets/stylesheets/machine_learning/comments_summary.scss new file mode 100644 index 000000000..e7d492d50 --- /dev/null +++ b/app/assets/stylesheets/machine_learning/comments_summary.scss @@ -0,0 +1,7 @@ +.machine-learning-comments-summary { + border-bottom: 1px solid $medium-gray; + + + * { + margin-top: $line-height * 1.5; + } +} diff --git a/app/assets/stylesheets/machine_learning/info.scss b/app/assets/stylesheets/machine_learning/info.scss new file mode 100644 index 000000000..6638db0ee --- /dev/null +++ b/app/assets/stylesheets/machine_learning/info.scss @@ -0,0 +1,5 @@ +.machine-learning-info { + @include has-fa-icon(info-circle, solid); + float: left; + margin-right: $font-icon-margin; +} diff --git a/app/components/admin/machine_learning/help_component.html.erb b/app/components/admin/machine_learning/help_component.html.erb new file mode 100644 index 000000000..bc1fc5342 --- /dev/null +++ b/app/components/admin/machine_learning/help_component.html.erb @@ -0,0 +1,25 @@ +
+

<%= t("admin.machine_learning.help.title_1") %>

+

<%= t("admin.machine_learning.help.description_1") %>

+ +

<%= t("admin.machine_learning.help.title_2") %>

+

<%= t("admin.machine_learning.help.description_2") %>

+ +

<%= t("admin.machine_learning.help.title_3") %>

+

<%= t("admin.machine_learning.help.description_3") %>

+ +

<%= t("admin.machine_learning.help.title_4") %>

+ + +

<%= t("admin.machine_learning.help.title_5") %>

+

<%= t("admin.machine_learning.help.description_5") %>

+ <%= instructions %> +
diff --git a/app/components/admin/machine_learning/help_component.rb b/app/components/admin/machine_learning/help_component.rb new file mode 100644 index 000000000..18c07c1b2 --- /dev/null +++ b/app/components/admin/machine_learning/help_component.rb @@ -0,0 +1,17 @@ +class Admin::MachineLearning::HelpComponent < ApplicationComponent + private + + def instructions + <<~INSTRUCTIONS + sudo apt update + sudo apt install software-properties-common + sudo add-apt-repository ppa:deadsnakes/ppa + sudo apt install python3.7 + sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + python get-pip.py + sudo update-alternatives --install /usr/bin/pip pip /home/deploy/.local/bin/pip3.7 1 + pip install pandas + INSTRUCTIONS + end +end diff --git a/app/components/admin/machine_learning/scripts_component.html.erb b/app/components/admin/machine_learning/scripts_component.html.erb new file mode 100644 index 000000000..63d65277b --- /dev/null +++ b/app/components/admin/machine_learning/scripts_component.html.erb @@ -0,0 +1,63 @@ +
+ <% if machine_learning_job.errored? %> +
+

+ <%= t("admin.machine_learning.notice.error") %> +

+ +
+
<%= t("admin.machine_learning.executed_by") %>
+
<%= machine_learning_job.user.name %>
+ +
<%= t("admin.machine_learning.script_name") %>
+
<%= machine_learning_job.script %>
+ +
<%= t("admin.machine_learning.error") %>
+
<%= sanitize(machine_learning_job.error) %>
+
+
+ <% elsif machine_learning_job.finished? %> +
+ <%= t("admin.machine_learning.notice.success") %> +
+ <% elsif machine_learning_job.started? %> +
+

+ <%= t("admin.machine_learning.notice.working") %> +

+ +
+
<%= t("admin.machine_learning.executed_by") %>
+
<%= machine_learning_job.user.name %>
+ +
<%= t("admin.machine_learning.script_name") %>
+
<%= machine_learning_job.script %>
+ +
<%= t("admin.machine_learning.started_at") %>
+
<%= machine_learning_job.started_at %>
+
+
+ <% end %> + + <% if machine_learning_job.running_for_too_long? %> + <%= button_to t("admin.machine_learning.cancel"), + cancel_admin_machine_learning_path, + method: :delete, class: "cancel", + data: { confirm: t("admin.machine_learning.cancel_alert") } %> + <% elsif machine_learning_job.errored? || !machine_learning_job.started? || machine_learning_job.finished? %> + <%= form_tag execute_admin_machine_learning_path, method: :post do %> + + <%= select_tag "script", options_for_select(script_select_options) %> + +
+ <% scripts_info.each_with_index do |script_info, index| %> +
+ <%= sanitize(script_info[:description]) %> +
+ <% end %> +
+ + <%= submit_tag t("admin.machine_learning.execute_script") %> + <% end %> + <% end %> +
diff --git a/app/components/admin/machine_learning/scripts_component.rb b/app/components/admin/machine_learning/scripts_component.rb new file mode 100644 index 000000000..1400814ef --- /dev/null +++ b/app/components/admin/machine_learning/scripts_component.rb @@ -0,0 +1,17 @@ +class Admin::MachineLearning::ScriptsComponent < ApplicationComponent + attr_reader :machine_learning_job + + def initialize(machine_learning_job) + @machine_learning_job = machine_learning_job + end + + private + + def script_select_options + scripts_info.map { |info| [info[:name], { "aria-describedby": info[:name] }] } + end + + def scripts_info + @scripts_info ||= ::MachineLearning.scripts_info + end +end diff --git a/app/components/admin/machine_learning/setting_component.html.erb b/app/components/admin/machine_learning/setting_component.html.erb new file mode 100644 index 000000000..5509c1040 --- /dev/null +++ b/app/components/admin/machine_learning/setting_component.html.erb @@ -0,0 +1,40 @@ +
+
+

<%= t("admin.machine_learning.#{kind}") %>

+
+
+

<%= t("admin.machine_learning.#{kind}_description") %>

+ + <% if ml_info.present? %> + <%= form_for(setting, url: admin_setting_path(setting), method: :put) do |f| %> + <%= f.hidden_field :tab, value: "#settings", id: "setting_tab_#{kind}" %> + <%= f.hidden_field :value, value: (setting.enabled? ? "" : "active"), id: "setting_value_#{kind}" %> + <%= f.button(t("shared.#{setting.enabled? ? "yes" : "no"}"), + "aria-labelledby": "machine_learning_#{kind}", + "aria-describedby": "machine_learning_#{kind}_description", + "aria-pressed": setting.enabled?) %> + <% end %> + +
+
<%= t("admin.machine_learning.last_execution") %>
+
+ <%= render Admin::DateRangeComponent.new(ml_info.generated_at, ml_info.updated_at) %> +
+ +
<%= t("admin.machine_learning.executed_script") %>
+
<%= ml_info.script %>
+ +
<%= t("admin.machine_learning.output_files") %>
+
+ <% filenames.each do |filename| %> + <%= filename %>
+ <% end %> +
+
+ <% else %> +
+ <%= t("admin.machine_learning.no_content") %> +
+ <% end %> +
+
diff --git a/app/components/admin/machine_learning/setting_component.rb b/app/components/admin/machine_learning/setting_component.rb new file mode 100644 index 000000000..e6da31334 --- /dev/null +++ b/app/components/admin/machine_learning/setting_component.rb @@ -0,0 +1,25 @@ +class Admin::MachineLearning::SettingComponent < ApplicationComponent + attr_reader :kind + + def initialize(kind) + @kind = kind + end + + private + + def setting + @setting ||= Setting.find_by(key: "machine_learning.#{kind}") + end + + def ml_info + @ml_info ||= MachineLearningInfo.for(kind) + end + + def filenames + ::MachineLearning.data_output_files[ml_info.kind.to_sym].sort + end + + def data_path(filename) + ::MachineLearning.data_path(filename) + end +end diff --git a/app/components/admin/machine_learning/settings_component.html.erb b/app/components/admin/machine_learning/settings_component.html.erb new file mode 100644 index 000000000..8ae6370d0 --- /dev/null +++ b/app/components/admin/machine_learning/settings_component.html.erb @@ -0,0 +1,15 @@ +
+
+ <% script_kinds.each do |kind| %> + <%= render Admin::MachineLearning::SettingComponent.new(kind) %> + <% end %> +
+ +
+

<%= t("admin.machine_learning.data_folder_content") %>

+ + <% filenames.each do |filename| %> + <%= filename %>
+ <% end %> +
+
diff --git a/app/components/admin/machine_learning/settings_component.rb b/app/components/admin/machine_learning/settings_component.rb new file mode 100644 index 000000000..2856f5194 --- /dev/null +++ b/app/components/admin/machine_learning/settings_component.rb @@ -0,0 +1,15 @@ +class Admin::MachineLearning::SettingsComponent < ApplicationComponent + private + + def script_kinds + @script_kinds ||= ::MachineLearning.script_kinds + end + + def filenames + ::MachineLearning.data_intermediate_files + end + + def data_path(filename) + ::MachineLearning.data_path(filename) + end +end diff --git a/app/components/admin/machine_learning/show_component.html.erb b/app/components/admin/machine_learning/show_component.html.erb new file mode 100644 index 000000000..79dc8760c --- /dev/null +++ b/app/components/admin/machine_learning/show_component.html.erb @@ -0,0 +1,39 @@ +<%= header %> + +<% if enabled? %> +
+ <%= sanitize(t("admin.machine_learning.help_text")) %> +
+ + + +
+ <%= render Admin::MachineLearning::ScriptsComponent.new(machine_learning_job) %> + <%= render Admin::MachineLearning::SettingsComponent.new %> + <%= render Admin::MachineLearning::HelpComponent.new %> +
+<% else %> +
+

+ <%= sanitize(t("admin.machine_learning.feature_disabled", + link: link_to(t("admin.machine_learning.feature_disabled_link"), + admin_settings_path(anchor: "tab-feature-flags")))) %> +

+
+<% end %> diff --git a/app/components/admin/machine_learning/show_component.rb b/app/components/admin/machine_learning/show_component.rb new file mode 100644 index 000000000..0986f408b --- /dev/null +++ b/app/components/admin/machine_learning/show_component.rb @@ -0,0 +1,18 @@ +class Admin::MachineLearning::ShowComponent < ApplicationComponent + include Header + attr_reader :machine_learning_job + + def initialize(machine_learning_job) + @machine_learning_job = machine_learning_job + end + + def title + t("admin.machine_learning.title") + end + + private + + def enabled? + ::MachineLearning.enabled? + end +end diff --git a/app/components/admin/menu_component.html.erb b/app/components/admin/menu_component.html.erb index eff6bd32d..40e501dc5 100644 --- a/app/components/admin/menu_component.html.erb +++ b/app/components/admin/menu_component.html.erb @@ -126,4 +126,9 @@ class: ("is-active" if dashboard?) ) %> + <% if ::MachineLearning.enabled? %> +
  • "> + <%= link_to t("admin.menu.machine_learning"), admin_machine_learning_path, class: "ml-link" %> +
    • + <% end %> diff --git a/app/components/machine_learning/comments_summary_component.html.erb b/app/components/machine_learning/comments_summary_component.html.erb new file mode 100644 index 000000000..c66c2303f --- /dev/null +++ b/app/components/machine_learning/comments_summary_component.html.erb @@ -0,0 +1,8 @@ +
    +

    + <%= render MachineLearning::InfoComponent.new %> + <%= t("machine_learning.comments_summary") %> +

    + +

    <%= simple_format(body) %>

    +
    diff --git a/app/components/machine_learning/comments_summary_component.rb b/app/components/machine_learning/comments_summary_component.rb new file mode 100644 index 000000000..832fd8c65 --- /dev/null +++ b/app/components/machine_learning/comments_summary_component.rb @@ -0,0 +1,17 @@ +class MachineLearning::CommentsSummaryComponent < ApplicationComponent + attr_reader :commentable + + def initialize(commentable) + @commentable = commentable + end + + def render? + MachineLearning.enabled? && Setting["machine_learning.comments_summary"].present? && body.present? + end + + private + + def body + commentable.summary_comment&.body + end +end diff --git a/app/components/machine_learning/info_component.html.erb b/app/components/machine_learning/info_component.html.erb new file mode 100644 index 000000000..432c0d500 --- /dev/null +++ b/app/components/machine_learning/info_component.html.erb @@ -0,0 +1,3 @@ +"> + <%= t("machine_learning.info_text") %> + diff --git a/app/components/machine_learning/info_component.rb b/app/components/machine_learning/info_component.rb new file mode 100644 index 000000000..2812ed2f2 --- /dev/null +++ b/app/components/machine_learning/info_component.rb @@ -0,0 +1,3 @@ +class MachineLearning::InfoComponent < ApplicationComponent + delegate :current_user, to: :helpers +end diff --git a/app/components/shared/comments_component.html.erb b/app/components/shared/comments_component.html.erb index 3d165703a..fd6b3d8ed 100644 --- a/app/components/shared/comments_component.html.erb +++ b/app/components/shared/comments_component.html.erb @@ -1,8 +1,8 @@ -<% cache cache_key do %> -
    -
    - <%= content %> +
    +
    + <%= content %> + <% cache cache_key do %> <% if current_user %> <%= render "comments/form", { commentable: record, parent_id: nil } %> <% else %> @@ -12,6 +12,6 @@ <%= render Shared::OrderLinksComponent.new("comments", anchor: "comments") %> <%= render "comments/comment_list", comments: comment_tree.root_comments %> <%= paginate comment_tree.root_comments, params: { anchor: "comments" } %> -
    + <% end %>
    -<% end %> +
    diff --git a/app/components/shared/tag_list_component.html.erb b/app/components/shared/tag_list_component.html.erb index 217f9855c..5ebb0e43f 100644 --- a/app/components/shared/tag_list_component.html.erb +++ b/app/components/shared/tag_list_component.html.erb @@ -1 +1,5 @@ +<% if machine_learning? %> + <%= render MachineLearning::InfoComponent.new %> +<% end %> + <%= link_list(*links, class: "tags", id: "tags_#{dom_id(taggable)}") %> diff --git a/app/components/shared/tag_list_component.rb b/app/components/shared/tag_list_component.rb index 79510718e..4e123f37b 100644 --- a/app/components/shared/tag_list_component.rb +++ b/app/components/shared/tag_list_component.rb @@ -7,6 +7,10 @@ class Shared::TagListComponent < ApplicationComponent @limit = limit end + def render? + taggable.tags_list.any? + end + private def links @@ -23,7 +27,7 @@ class Shared::TagListComponent < ApplicationComponent end def see_more_link - render Shared::SeeMoreLinkComponent.new(taggable, :tags, limit: limit) + render Shared::SeeMoreLinkComponent.new(taggable, :tags_list, limit: limit) end def taggables_path(taggable, tag_name) @@ -34,4 +38,8 @@ class Shared::TagListComponent < ApplicationComponent polymorphic_path(taggable.class, search: tag_name) end end + + def machine_learning? + Tag.machine_learning? + end end diff --git a/app/controllers/admin/machine_learning_controller.rb b/app/controllers/admin/machine_learning_controller.rb new file mode 100644 index 000000000..d87e4f51c --- /dev/null +++ b/app/controllers/admin/machine_learning_controller.rb @@ -0,0 +1,33 @@ +class Admin::MachineLearningController < Admin::BaseController + before_action :load_machine_learning_job, only: [:show, :execute] + + def show + end + + def execute + @machine_learning_job.update!(script: params[:script], + user: current_user, + started_at: Time.current, + finished_at: nil, + error: nil) + + ::MachineLearning.new(@machine_learning_job).run + + redirect_to admin_machine_learning_path, + notice: t("admin.machine_learning.script_info", email: current_user.email) + end + + def cancel + Delayed::Job.where(queue: "machine_learning").destroy_all + MachineLearningJob.destroy_all + + redirect_to admin_machine_learning_path, + notice: t("admin.machine_learning.notice.delete_generated_content") + end + + private + + def load_machine_learning_job + @machine_learning_job = MachineLearningJob.first_or_initialize + end +end diff --git a/app/helpers/site_customization_helper.rb b/app/helpers/site_customization_helper.rb index 17ffdde13..8d30e01cc 100644 --- a/app/helpers/site_customization_helper.rb +++ b/app/helpers/site_customization_helper.rb @@ -19,6 +19,6 @@ module SiteCustomizationHelper end def information_texts_tabs - [:basic, :debates, :community, :proposals, :polls, :layouts, :mailers, :management, :welcome] + [:basic, :debates, :community, :proposals, :polls, :layouts, :mailers, :management, :welcome, :machine_learning] end end diff --git a/app/mailers/mailer.rb b/app/mailers/mailer.rb index 0c0f8ebfe..9fbb8ec88 100644 --- a/app/mailers/mailer.rb +++ b/app/mailers/mailer.rb @@ -127,6 +127,18 @@ class Mailer < ApplicationMailer mail(to: @email_to.email, subject: @email.subject) if @email.can_be_sent? end + def machine_learning_error(user) + @email_to = user.email + + mail(to: @email_to, subject: t("mailers.machine_learning_error.subject")) + end + + def machine_learning_success(user) + @email_to = user.email + + mail(to: @email_to, subject: t("mailers.machine_learning_success.subject")) + end + private def with_user(user, &block) diff --git a/app/models/budget/investment.rb b/app/models/budget/investment.rb index d490a3bf1..b92ac42da 100644 --- a/app/models/budget/investment.rb +++ b/app/models/budget/investment.rb @@ -50,6 +50,7 @@ class Budget has_many :valuator_groups, through: :valuator_group_assignments has_many :comments, -> { where(valuation: false) }, as: :commentable, inverse_of: :commentable + has_one :summary_comment, as: :commentable, class_name: "MlSummaryComment", dependent: :destroy has_many :valuations, -> { where(valuation: true) }, as: :commentable, inverse_of: :commentable, diff --git a/app/models/budget/investment/exporter.rb b/app/models/budget/investment/exporter.rb index cf4972198..6ba272ab2 100644 --- a/app/models/budget/investment/exporter.rb +++ b/app/models/budget/investment/exporter.rb @@ -1,5 +1,6 @@ class Budget::Investment::Exporter require "csv" + include JsonExporter def initialize(investments) @investments = investments @@ -12,6 +13,10 @@ class Budget::Investment::Exporter end end + def model + Budget::Investment + end + private def headers @@ -64,4 +69,12 @@ class Budget::Investment::Exporter I18n.t(price_string) end end + + def json_values(investment) + { + id: investment.id, + title: investment.title, + description: strip_tags(investment.description) + } + end end diff --git a/app/models/comment/exporter.rb b/app/models/comment/exporter.rb new file mode 100644 index 000000000..055ba8611 --- /dev/null +++ b/app/models/comment/exporter.rb @@ -0,0 +1,18 @@ +class Comment::Exporter + include JsonExporter + + def model + Comment + end + + private + + def json_values(comment) + { + id: comment.id, + commentable_id: comment.commentable_id, + commentable_type: comment.commentable_type, + body: strip_tags(comment.body) + } + end +end diff --git a/app/models/concerns/json_exporter.rb b/app/models/concerns/json_exporter.rb new file mode 100644 index 000000000..b06ce42d2 --- /dev/null +++ b/app/models/concerns/json_exporter.rb @@ -0,0 +1,21 @@ +module JsonExporter + def to_json_file(filename) + data = [] + model.find_each { |record| data << json_values(record) } + File.open(filename, "w") { |file| file.write(data.to_json) } + end + + private + + def strip_tags(html_string) + ActionView::Base.full_sanitizer.sanitize(html_string) + end + + def model + raise "This method must be implemented in class #{self.class.name}" + end + + def json_values(record) + raise "This method must be implemented in class #{self.class.name}" + end +end diff --git a/app/models/concerns/relationable.rb b/app/models/concerns/relationable.rb index d0c837d1e..6bbe678ca 100644 --- a/app/models/concerns/relationable.rb +++ b/app/models/concerns/relationable.rb @@ -13,7 +13,14 @@ module Relationable end def relationed_contents - related_contents.not_hidden.map(&:child_relationable) - .reject { |related| related.respond_to?(:retired?) && related.retired? } + if MachineLearning.enabled? && Setting["machine_learning.related_content"].present? + related_content = related_contents.not_hidden.order(machine_learning_score: :desc) + else + related_content = related_contents.not_hidden.from_users + end + + related_content.map(&:child_relationable).reject do |related| + related.respond_to?(:retired?) && related.retired? + end end end diff --git a/app/models/concerns/taggable.rb b/app/models/concerns/taggable.rb index 73889ae81..d4a886e32 100644 --- a/app/models/concerns/taggable.rb +++ b/app/models/concerns/taggable.rb @@ -2,14 +2,22 @@ module Taggable extend ActiveSupport::Concern included do - acts_as_taggable + acts_as_taggable_on :tags, :ml_tags validate :max_number_of_tags, on: :create end - def tag_list_with_limit(limit = nil) - return tags if limit.blank? + def tags_list + if Tag.machine_learning? && (is_a?(Proposal) || is_a?(Budget::Investment)) + ml_tags + else + tags + end + end - tags.sort { |a, b| b.taggings_count <=> a.taggings_count }[0, limit] + def tag_list_with_limit(limit = nil) + return tags_list if limit.blank? + + tags_list.sort { |a, b| b.taggings_count <=> a.taggings_count }[0, limit] end def max_number_of_tags diff --git a/app/models/i18n_content.rb b/app/models/i18n_content.rb index 1bcaea82b..f474aa700 100644 --- a/app/models/i18n_content.rb +++ b/app/models/i18n_content.rb @@ -52,6 +52,8 @@ class I18nContent < ApplicationRecord def self.translations_for(tab) if tab.to_s == "basic" basic_translations + elsif tab.to_s == "machine_learning" + machine_learning_translations else flat_hash(translations_hash_for(tab)).keys end @@ -95,6 +97,20 @@ class I18nContent < ApplicationRecord ] end + def self.machine_learning_translations + %w[ + admin.machine_learning.title + machine_learning.comments_summary + machine_learning.info_text + admin.machine_learning.comments_summary + admin.machine_learning.comments_summary_description + admin.machine_learning.related_content + admin.machine_learning.related_content_description + admin.machine_learning.tags + admin.machine_learning.tags_description + ] + end + def self.translations_hash(locale) Rails.cache.fetch(translation_class.where(locale: locale)) do all.map do |content| diff --git a/app/models/machine_learning.rb b/app/models/machine_learning.rb new file mode 100644 index 000000000..c58b8361a --- /dev/null +++ b/app/models/machine_learning.rb @@ -0,0 +1,436 @@ +class MachineLearning + attr_reader :user, :script, :previous_modified_date + attr_accessor :job + + SCRIPTS_FOLDER = Rails.root.join("public", "machine_learning", "scripts").freeze + DATA_FOLDER = Rails.root.join("public", "machine_learning", "data").freeze + + def initialize(job) + @job = job + @user = job.user + @previous_modified_date = set_previous_modified_date + end + + def run + begin + export_proposals_to_json + export_budget_investments_to_json + export_comments_to_json + + return unless run_machine_learning_scripts + + if updated_file?(MachineLearning.proposals_taggings_filename) && updated_file?(MachineLearning.proposals_tags_filename) + cleanup_proposals_tags! + import_ml_proposals_tags + update_machine_learning_info_for("tags") + end + + if updated_file?(MachineLearning.investments_taggings_filename) && updated_file?(MachineLearning.investments_tags_filename) + cleanup_investments_tags! + import_ml_investments_tags + update_machine_learning_info_for("tags") + end + + if updated_file?(MachineLearning.proposals_related_filename) + cleanup_proposals_related_content! + import_proposals_related_content + update_machine_learning_info_for("related_content") + end + + if updated_file?(MachineLearning.investments_related_filename) + cleanup_investments_related_content! + import_budget_investments_related_content + update_machine_learning_info_for("related_content") + end + + if updated_file?(MachineLearning.proposals_comments_summary_filename) + cleanup_proposals_comments_summary! + import_ml_proposals_comments_summary + update_machine_learning_info_for("comments_summary") + end + + if updated_file?(MachineLearning.investments_comments_summary_filename) + cleanup_investments_comments_summary! + import_ml_investments_comments_summary + update_machine_learning_info_for("comments_summary") + end + + job.update!(finished_at: Time.current) + Mailer.machine_learning_success(user).deliver_later + rescue Exception => error + handle_error(error) + raise error + end + end + handle_asynchronously :run, queue: "machine_learning" + + class << self + def enabled? + Setting["feature.machine_learning"].present? + end + + def proposals_filename + "proposals.json" + end + + def investments_filename + "budget_investments.json" + end + + def comments_filename + "comments.json" + end + + def data_output_files + files = { tags: [], related_content: [], comments_summary: [] } + + files[:tags] << proposals_tags_filename if File.exists?(DATA_FOLDER.join(proposals_tags_filename)) + files[:tags] << proposals_taggings_filename if File.exists?(DATA_FOLDER.join(proposals_taggings_filename)) + files[:tags] << investments_tags_filename if File.exists?(DATA_FOLDER.join(investments_tags_filename)) + files[:tags] << investments_taggings_filename if File.exists?(DATA_FOLDER.join(investments_taggings_filename)) + files[:related_content] << proposals_related_filename if File.exists?(DATA_FOLDER.join(proposals_related_filename)) + files[:related_content] << investments_related_filename if File.exists?(DATA_FOLDER.join(investments_related_filename)) + files[:comments_summary] << proposals_comments_summary_filename if File.exists?(DATA_FOLDER.join(proposals_comments_summary_filename)) + files[:comments_summary] << investments_comments_summary_filename if File.exists?(DATA_FOLDER.join(investments_comments_summary_filename)) + + files + end + + def data_intermediate_files + excluded = [ + proposals_filename, + investments_filename, + comments_filename, + proposals_tags_filename, + proposals_taggings_filename, + investments_tags_filename, + investments_taggings_filename, + proposals_related_filename, + investments_related_filename, + proposals_comments_summary_filename, + investments_comments_summary_filename + ] + json = Dir[DATA_FOLDER.join("*.json")].map do |full_path_filename| + full_path_filename.split("/").last + end + csv = Dir[DATA_FOLDER.join("*.csv")].map do |full_path_filename| + full_path_filename.split("/").last + end + (json + csv - excluded).sort + end + + def proposals_tags_filename + "ml_tags_proposals.json" + end + + def proposals_taggings_filename + "ml_taggings_proposals.json" + end + + def investments_tags_filename + "ml_tags_budgets.json" + end + + def investments_taggings_filename + "ml_taggings_budgets.json" + end + + def proposals_related_filename + "ml_related_content_proposals.json" + end + + def investments_related_filename + "ml_related_content_budgets.json" + end + + def proposals_comments_summary_filename + "ml_comments_summaries_proposals.json" + end + + def investments_comments_summary_filename + "ml_comments_summaries_budgets.json" + end + + def data_path(filename) + "/machine_learning/data/" + filename + end + + def script_kinds + %w[tags related_content comments_summary] + end + + def scripts_info + scripts_info = [] + Dir[SCRIPTS_FOLDER.join("*.py")].each do |full_path_filename| + scripts_info << { + name: full_path_filename.split("/").last, + description: description_from(full_path_filename) + } + end + scripts_info.sort_by { |script_info| script_info[:name] } + end + + def description_from(script_filename) + description = "" + delimiter = '"""' + break_line = "
    " + comment_found = false + File.readlines(script_filename).each do |line| + if line.start_with?(delimiter) && !comment_found + comment_found = true + line.slice!(delimiter) + description << line.strip.concat(break_line) if line.present? + elsif line.include?(delimiter) + line.slice!(delimiter) + description << line.strip if line.present? + break + elsif comment_found + description << line.strip.concat(break_line) + end + end + + description.delete_suffix(break_line) + end + end + + private + + def export_proposals_to_json + filename = DATA_FOLDER.join(MachineLearning.proposals_filename) + Proposal::Exporter.new.to_json_file(filename) + end + + def export_budget_investments_to_json + filename = DATA_FOLDER.join(MachineLearning.investments_filename) + Budget::Investment::Exporter.new(Array.new).to_json_file(filename) + end + + def export_comments_to_json + filename = DATA_FOLDER.join(MachineLearning.comments_filename) + Comment::Exporter.new.to_json_file(filename) + end + + def run_machine_learning_scripts + output = `cd #{SCRIPTS_FOLDER} && python #{job.script} 2>&1` + result = $?.success? + if result == false + job.update!(finished_at: Time.current, error: output) + Mailer.machine_learning_error(user).deliver_later + end + result + end + + def cleanup_proposals_tags! + Tagging.where(context: "ml_tags", taggable_type: "Proposal").find_each(&:destroy!) + Tag.find_each { |tag| tag.destroy! if Tagging.where(tag: tag).empty? } + end + + def cleanup_investments_tags! + Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment").find_each(&:destroy!) + Tag.find_each { |tag| tag.destroy! if Tagging.where(tag: tag).empty? } + end + + def cleanup_proposals_related_content! + RelatedContent.with_hidden.for_proposals.from_machine_learning.find_each(&:really_destroy!) + end + + def cleanup_investments_related_content! + RelatedContent.with_hidden.for_investments.from_machine_learning.find_each(&:really_destroy!) + end + + def cleanup_proposals_comments_summary! + MlSummaryComment.where(commentable_type: "Proposal").find_each(&:destroy!) + end + + def cleanup_investments_comments_summary! + MlSummaryComment.where(commentable_type: "Budget::Investment").find_each(&:destroy!) + end + + def import_ml_proposals_comments_summary + json_file = DATA_FOLDER.join(MachineLearning.proposals_comments_summary_filename) + json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!) + json_data.each do |attributes| + attributes.delete(:id) + unless MlSummaryComment.find_by(commentable_id: attributes[:commentable_id], + commentable_type: "Proposal") + MlSummaryComment.create!(attributes) + end + end + end + + def import_ml_investments_comments_summary + json_file = DATA_FOLDER.join(MachineLearning.investments_comments_summary_filename) + json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!) + json_data.each do |attributes| + attributes.delete(:id) + unless MlSummaryComment.find_by(commentable_id: attributes[:commentable_id], + commentable_type: "Budget::Investment") + MlSummaryComment.create!(attributes) + end + end + end + + def import_proposals_related_content + json_file = DATA_FOLDER.join(MachineLearning.proposals_related_filename) + json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!) + json_data.each do |related| + id = related.delete(:id) + score = related.size + related.each do |_, related_id| + if related_id.present? + attributes = { + parent_relationable_id: id, + parent_relationable_type: "Proposal", + child_relationable_id: related_id, + child_relationable_type: "Proposal" + } + related_content = RelatedContent.find_by(attributes) + if related_content.present? + related_content.update!(machine_learning_score: score) + else + RelatedContent.create!(attributes.merge(machine_learning: true, + author: user, + machine_learning_score: score)) + end + end + score -= 1 + end + end + end + + def import_budget_investments_related_content + json_file = DATA_FOLDER.join(MachineLearning.investments_related_filename) + json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!) + json_data.each do |related| + id = related.delete(:id) + score = related.size + related.each do |_, related_id| + if related_id.present? + attributes = { + parent_relationable_id: id, + parent_relationable_type: "Budget::Investment", + child_relationable_id: related_id, + child_relationable_type: "Budget::Investment" + } + related_content = RelatedContent.find_by(attributes) + if related_content.present? + related_content.update!(machine_learning_score: score) + else + RelatedContent.create!(attributes.merge(machine_learning: true, + author: user, + machine_learning_score: score)) + end + end + score -= 1 + end + end + end + + def import_ml_proposals_tags + ids = {} + json_file = DATA_FOLDER.join(MachineLearning.proposals_tags_filename) + json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!) + json_data.each do |attributes| + if attributes[:name].present? + attributes.delete(:taggings_count) + if attributes[:name].length >= 150 + attributes[:name] = attributes[:name].truncate(150) + end + tag = Tag.find_or_create_by!(name: attributes[:name]) + ids[attributes[:id]] = tag.id + end + end + + json_file = DATA_FOLDER.join(MachineLearning.proposals_taggings_filename) + json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!) + json_data.each do |attributes| + if attributes[:tag_id].present? + tag_id = ids[attributes[:tag_id]] + if Tag.find_by(id: tag_id) && attributes[:taggable_id].present? + attributes[:tag_id] = tag_id + attributes[:taggable_type] = "Proposal" + attributes[:context] = "ml_tags" + Tagging.create!(attributes) + end + end + end + end + + def import_ml_investments_tags + ids = {} + json_file = DATA_FOLDER.join(MachineLearning.investments_tags_filename) + json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!) + json_data.each do |attributes| + if attributes[:name].present? + attributes.delete(:taggings_count) + if attributes[:name].length >= 150 + attributes[:name] = attributes[:name].truncate(150) + end + tag = Tag.find_or_create_by!(name: attributes[:name]) + ids[attributes[:id]] = tag.id + end + end + + json_file = DATA_FOLDER.join(MachineLearning.investments_taggings_filename) + json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!) + json_data.each do |attributes| + if attributes[:tag_id].present? + tag_id = ids[attributes[:tag_id]] + if Tag.find_by(id: tag_id) && attributes[:taggable_id].present? + attributes[:tag_id] = tag_id + attributes[:taggable_type] = "Budget::Investment" + attributes[:context] = "ml_tags" + Tagging.create!(attributes) + end + end + end + end + + def update_machine_learning_info_for(kind) + MachineLearningInfo.find_or_create_by!(kind: kind) + .update!(generated_at: job.started_at, script: job.script) + end + + def set_previous_modified_date + proposals_tags_filename = MachineLearning.proposals_tags_filename + proposals_taggings_filename = MachineLearning.proposals_taggings_filename + investments_tags_filename = MachineLearning.investments_tags_filename + investments_taggings_filename = MachineLearning.investments_taggings_filename + proposals_related_filename = MachineLearning.proposals_related_filename + investments_related_filename = MachineLearning.investments_related_filename + proposals_comments_summary_filename = MachineLearning.proposals_comments_summary_filename + investments_comments_summary_filename = MachineLearning.investments_comments_summary_filename + + { + proposals_tags_filename => last_modified_date_for(proposals_tags_filename), + proposals_taggings_filename => last_modified_date_for(proposals_taggings_filename), + investments_tags_filename => last_modified_date_for(investments_tags_filename), + investments_taggings_filename => last_modified_date_for(investments_taggings_filename), + proposals_related_filename => last_modified_date_for(proposals_related_filename), + investments_related_filename => last_modified_date_for(investments_related_filename), + proposals_comments_summary_filename => last_modified_date_for(proposals_comments_summary_filename), + investments_comments_summary_filename => last_modified_date_for(investments_comments_summary_filename) + } + end + + def last_modified_date_for(filename) + return nil unless File.exists? DATA_FOLDER.join(filename) + + File.mtime DATA_FOLDER.join(filename) + end + + def updated_file?(filename) + return false unless File.exists? DATA_FOLDER.join(filename) + return true unless previous_modified_date[filename].present? + + last_modified_date_for(filename) > previous_modified_date[filename] + end + + def handle_error(error) + message = error.message + backtrace = error.backtrace.select { |line| line.include?("machine_learning.rb") } + full_error = ([message] + backtrace).join("
    ") + job.update!(finished_at: Time.current, error: full_error) + Mailer.machine_learning_error(user).deliver_later + end +end diff --git a/app/models/machine_learning_info.rb b/app/models/machine_learning_info.rb new file mode 100644 index 000000000..793cc5eb4 --- /dev/null +++ b/app/models/machine_learning_info.rb @@ -0,0 +1,5 @@ +class MachineLearningInfo < ApplicationRecord + def self.for(kind) + find_by(kind: kind) + end +end diff --git a/app/models/machine_learning_job.rb b/app/models/machine_learning_job.rb new file mode 100644 index 000000000..feda373a9 --- /dev/null +++ b/app/models/machine_learning_job.rb @@ -0,0 +1,19 @@ +class MachineLearningJob < ApplicationRecord + belongs_to :user, optional: false + + def started? + started_at.present? + end + + def finished? + finished_at.present? + end + + def errored? + error.present? + end + + def running_for_too_long? + started? && !finished? && started_at < 1.day.ago + end +end diff --git a/app/models/ml_summary_comment.rb b/app/models/ml_summary_comment.rb new file mode 100644 index 000000000..c85fd5939 --- /dev/null +++ b/app/models/ml_summary_comment.rb @@ -0,0 +1,3 @@ +class MlSummaryComment < ApplicationRecord + belongs_to :commentable, -> { with_hidden }, polymorphic: true, touch: true +end diff --git a/app/models/proposal.rb b/app/models/proposal.rb index 643c51de5..746b86b2c 100644 --- a/app/models/proposal.rb +++ b/app/models/proposal.rb @@ -41,6 +41,7 @@ class Proposal < ApplicationRecord has_many :dashboard_executed_actions, dependent: :destroy, class_name: "Dashboard::ExecutedAction" has_many :dashboard_actions, through: :dashboard_executed_actions, class_name: "Dashboard::Action" has_many :polls, as: :related, inverse_of: :related + has_one :summary_comment, as: :commentable, class_name: "MlSummaryComment", dependent: :destroy validates_translation :title, presence: true, length: { in: 4..Proposal.title_max_length } validates_translation :description, length: { maximum: Proposal.description_max_length } diff --git a/app/models/proposal/exporter.rb b/app/models/proposal/exporter.rb new file mode 100644 index 000000000..edbba58bd --- /dev/null +++ b/app/models/proposal/exporter.rb @@ -0,0 +1,18 @@ +class Proposal::Exporter + include JsonExporter + + def model + Proposal + end + + private + + def json_values(proposal) + { + id: proposal.id, + title: proposal.title, + summary: strip_tags(proposal.summary), + description: strip_tags(proposal.description) + } + end +end diff --git a/app/models/related_content.rb b/app/models/related_content.rb index 5765dc8ef..6e936f572 100644 --- a/app/models/related_content.rb +++ b/app/models/related_content.rb @@ -9,7 +9,7 @@ class RelatedContent < ApplicationRecord belongs_to :parent_relationable, polymorphic: true, optional: false, touch: true belongs_to :child_relationable, polymorphic: true, optional: false, touch: true has_one :opposite_related_content, class_name: self.name, foreign_key: :related_content_id - has_many :related_content_scores + has_many :related_content_scores, dependent: :destroy validates :parent_relationable_id, uniqueness: { scope: [:parent_relationable_type, :child_relationable_id, :child_relationable_type] } validate :different_parent_and_child @@ -18,6 +18,14 @@ class RelatedContent < ApplicationRecord after_create :create_author_score scope :not_hidden, -> { where(hidden_at: nil) } + scope :from_users, -> { where(machine_learning: false) } + scope :from_machine_learning, -> { where(machine_learning: true) } + scope :for_proposals, -> do + where(parent_relationable_type: "Proposal", child_relationable_type: "Proposal") + end + scope :for_investments, -> do + where(parent_relationable_type: "Budget::Investment", child_relationable_type: "Budget::Investment") + end def score_positive(user) score(RelatedContentScore::SCORES[:POSITIVE], user) @@ -48,8 +56,11 @@ class RelatedContent < ApplicationRecord end def create_opposite_related_content - related_content = RelatedContent.create!(opposite_related_content: self, parent_relationable: child_relationable, - child_relationable: parent_relationable, author: author) + related_content = RelatedContent.create!(opposite_related_content: self, + parent_relationable: child_relationable, + child_relationable: parent_relationable, + machine_learning: machine_learning, + author: author) self.opposite_related_content = related_content end diff --git a/app/models/setting.rb b/app/models/setting.rb index 09c88f96c..cf89f1130 100644 --- a/app/models/setting.rb +++ b/app/models/setting.rb @@ -101,6 +101,7 @@ class Setting < ApplicationRecord "feature.valuation_comment_notification": true, "feature.graphql_api": true, "feature.sdg": false, + "feature.machine_learning": false, "homepage.widgets.feeds.debates": true, "homepage.widgets.feeds.processes": true, "homepage.widgets.feeds.proposals": true, @@ -172,6 +173,9 @@ class Setting < ApplicationRecord "related_content_score_threshold": -0.3, "featured_proposals_number": 3, "feature.dashboard.notification_emails": nil, + "machine_learning.comments_summary": false, + "machine_learning.related_content": false, + "machine_learning.tags": false, "remote_census.general.endpoint": "", "remote_census.request.method_name": "", "remote_census.request.structure": "", diff --git a/app/models/tag.rb b/app/models/tag.rb index ef3d4af06..c2a682a58 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -1,2 +1,5 @@ class Tag < ActsAsTaggableOn::Tag + def self.machine_learning? + MachineLearning.enabled? && Setting["machine_learning.tags"].present? + end end diff --git a/app/models/tag_cloud.rb b/app/models/tag_cloud.rb index 2008685b3..0ab981653 100644 --- a/app/models/tag_cloud.rb +++ b/app/models/tag_cloud.rb @@ -8,12 +8,20 @@ class TagCloud def tags resource_model_scoped. - last_week.tag_counts. + last_week.send(counts). where("lower(name) NOT IN (?)", category_names + geozone_names + default_blacklist). order("#{table_name}_count": :desc, name: :asc). limit(10) end + def counts + if Tag.machine_learning? && [Proposal, Budget::Investment].include?(resource_model) + :ml_tag_counts + else + :tag_counts + end + end + def category_names Tag.category_names.map(&:downcase) end diff --git a/app/models/tagging.rb b/app/models/tagging.rb index 50e0fa9ad..d0e989ad9 100644 --- a/app/models/tagging.rb +++ b/app/models/tagging.rb @@ -1,2 +1,3 @@ class Tagging < ActsAsTaggableOn::Tagging + belongs_to :taggable, polymorphic: true, touch: true end diff --git a/app/views/admin/machine_learning/show.html.erb b/app/views/admin/machine_learning/show.html.erb new file mode 100644 index 000000000..ac9f9cdcd --- /dev/null +++ b/app/views/admin/machine_learning/show.html.erb @@ -0,0 +1 @@ +<%= render Admin::MachineLearning::ShowComponent.new(@machine_learning_job) %> diff --git a/app/views/budgets/investments/show.html.erb b/app/views/budgets/investments/show.html.erb index 8c342c5ef..042e70d46 100644 --- a/app/views/budgets/investments/show.html.erb +++ b/app/views/budgets/investments/show.html.erb @@ -16,6 +16,8 @@
    + <%= render MachineLearning::CommentsSummaryComponent.new(@investment) %> + <%= render "/comments/comment_tree", comment_tree: @comment_tree, display_comments_count: false %>
    diff --git a/app/views/mailer/machine_learning_error.html.erb b/app/views/mailer/machine_learning_error.html.erb new file mode 100644 index 000000000..9798f477c --- /dev/null +++ b/app/views/mailer/machine_learning_error.html.erb @@ -0,0 +1,17 @@ + + +

    + <%= t("mailers.machine_learning_error.title") %> +

    + +

    + <%= t("mailers.machine_learning_error.text") %> +

    + +

    + <%= link_to t("mailers.machine_learning_error.link"), admin_machine_learning_url, + style: "color: #2895F1; text-decoration:none;" %> +

    + diff --git a/app/views/mailer/machine_learning_success.html.erb b/app/views/mailer/machine_learning_success.html.erb new file mode 100644 index 000000000..5f6c5d5e5 --- /dev/null +++ b/app/views/mailer/machine_learning_success.html.erb @@ -0,0 +1,17 @@ + + +

    + <%= t("mailers.machine_learning_success.title") %> +

    + +

    + <%= t("mailers.machine_learning_success.text") %> +

    + +

    + <%= link_to t("mailers.machine_learning_success.link"), admin_machine_learning_url, + style: "color: #2895F1; text-decoration:none;" %> +

    + diff --git a/app/views/proposals/_comments.html.erb b/app/views/proposals/_comments.html.erb index 43d690b35..f39d1861e 100644 --- a/app/views/proposals/_comments.html.erb +++ b/app/views/proposals/_comments.html.erb @@ -1 +1,3 @@ -<%= render Shared::CommentsComponent.new(@proposal, @comment_tree) %> +<%= render Shared::CommentsComponent.new(@proposal, @comment_tree) do %> + <%= render MachineLearning::CommentsSummaryComponent.new(@proposal) %> +<% end %> diff --git a/config/deploy.rb b/config/deploy.rb index 0273ee204..07288e85c 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -22,7 +22,7 @@ set :pty, true set :use_sudo, false set :linked_files, %w[config/database.yml config/secrets.yml] -set :linked_dirs, %w[.bundle log tmp public/system public/assets public/ckeditor_assets] +set :linked_dirs, %w[.bundle log tmp public/system public/assets public/ckeditor_assets public/machine_learning/data] set :keep_releases, 5 diff --git a/config/i18n-tasks.yml b/config/i18n-tasks.yml index a32d18532..42a20478c 100644 --- a/config/i18n-tasks.yml +++ b/config/i18n-tasks.yml @@ -222,7 +222,7 @@ ignore_unused: - "sdg.goals.goal_*" - "sdg.*.filter.more.*" - "sdg_management.relations.index.filter*" - - "tags.filter.more.*" + - "tags.list.filter.more.*" #### ## Exclude these keys from the `i18n-tasks eq-base" report: # ignore_eq_base: diff --git a/config/locales/en/admin.yml b/config/locales/en/admin.yml index 3cdebaab7..6e29759b7 100644 --- a/config/locales/en/admin.yml +++ b/config/locales/en/admin.yml @@ -751,6 +751,7 @@ en: proposals: "Proposals" polls: "Polls" layouts: "Layouts" + machine_learning: "AI / Machine Learning" mailers: "Emails" management: "Management" welcome: "Welcome" @@ -771,6 +772,7 @@ en: debates: "Debates" comments: "Comments" local_census_records: Manage local census + machine_learning: "AI / Machine learning" administrators: index: title: Administrators @@ -1683,3 +1685,53 @@ en: created: Created records local_census_records: no_records_found: No records found. + machine_learning: + cancel: "Cancel operation" + cancel_alert: "This action will cancel the current script and it will be necessary to run the script again." + comments_summary: "Comments summary" + comments_summary_description: "Displays an automatically generated comment summary on all items that can be commented on." + data_folder_content: "Data folder content" + error: "Error:" + execute_script: "Execute script" + executed_by: "Executed by:" + executed_script: "Executed script:" + feature_disabled: "This feature is disabled. To use Machine Learning you can enable it from the %{link}." + feature_disabled_link: "settings page" + help: + title_1: "What is AI / Machine Learning?" + description_1: "Traditionally, computer programs are designed by establishing precise rules about what the program should do in each situation, and how to process the available information at each moment. What is commonly known as AI/Machine Learning is a type of programs for which what is precisely established is what the task to be performed is and how it is assessed whether the task is better or worse performed. But unlike the former, the system discovers or learns what would be the most suitable method to perform the task." + title_2: "What does the AI / Machine Learning module in CONSUL?" + description_2: "This module allows any type of AI/Machine Learning programs to be implemented in CONSUL. The programs can process the information available in CONSUL and produce results that help both users and administrators to carry out more effective and intelligent citizen participation. The module has been developed to make it easy to implement and run new programs, keeping the general code of CONSUL and the code of these new programs independent from each other." + title_3: "How to use this module" + description_3: "To use it for the first time it is necessary to follow the instructions available in the CONSUL documentation to properly activate this module. Once the module is correctly activated, the programs are executed from the \"Execute scripts\" tab. Before executing it, we can see in the same tab an estimation of the execution time and other relevant information. At the end of the execution, we can activate the content that will be shown in CONSUL from the \"Settings\" tab, as well as download other generated files that may be useful to us." + title_4: "How to implement new AI / Machine Learning scripts" + description_4: "Please use previous scripts as examples." + description_4b: "The new scripts should be located in the folder public/machine_learning/scripts" + description_4c: "The input data to be used by the script will be created as JSON files in the folder ../data" + description_4d: "Any generated output should be placed in the folder ../data" + description_4e: "It is recommended to create an independent .ini text file with the settings of the script, with the same name of the script and readable by configparser. This file will facilitate settings change by the administrators." + description_4f: "It is recommended to include as the first line of the script an initial triple-quote string with some brief information about it and links to any relevant information. This string will be automatically shown in the Administration interface." + description_4g: "It is recommended to log the relevant information of the script in a .log text file, with the same name of the script." + title_5: "To use AI / Machine Learning scripts you need to have Python installed on your server" + description_5: "Here you can see the example instructions for an Ubuntu 18.04 server:" + help_text: "This functionality is experimental." + last_execution: "Last execution" + no_content: "No content generated yet." + notice: + success: "The last script has been executed successfully." + error: "An error has occurred. You can see the details below." + working: "The script is running. The administrator who executed it will receive an email when it is finished." + delete_generated_content: "Generated content has been successfully deleted." + output_files: "Output files:" + related_content: "Related content" + related_content_description: "Adds automatically generated related content to proposals and participatory budget projects." + script_info: "You will receive an email in %{email} when the script finishes running." + script_name: "Script name:" + select_script: "Select python script to execute" + started_at: "Started at:" + tab_help: "Help" + tab_scripts: "Execute script" + tab_settings: "Settings / Generated content" + tags: "Tags" + tags_description: "Generates automatic tags on all items that can be tagged on." + title: "AI / Machine learning" diff --git a/config/locales/en/general.yml b/config/locales/en/general.yml index a9abab348..94cd6bd3b 100644 --- a/config/locales/en/general.yml +++ b/config/locales/en/general.yml @@ -965,7 +965,11 @@ en: enqueue_remote_translation: Translations have been correctly requested. button: Translate page tags: - filter: - more: - one: "One more tag" - other: "%{count} more tags" + list: + filter: + more: + one: "One more tag" + other: "%{count} more tags" + machine_learning: + comments_summary: "Comments summary" + info_text: "Content generated by AI / Machine Learning" diff --git a/config/locales/en/mailers.yml b/config/locales/en/mailers.yml index d012cdb71..552aa9831 100644 --- a/config/locales/en/mailers.yml +++ b/config/locales/en/mailers.yml @@ -78,6 +78,16 @@ en: hi: Hi new_comment_by: There is a new evaluation comment from %{commenter} to the budget investment %{investment} commenter_info: "%{commenter}, %{time}:" + machine_learning_error: + link: "Visit Machine Learning panel" + subject: "Machine Learning - An error has occurred running the script" + text: "An error has occurred running the Machine Learning script." + title: "Machine Learning script" + machine_learning_success: + link: "Visit Machine Learning panel" + subject: "Machine Learning - Content has been generated successfully" + text: "Content has been generated successfully." + title: "Machine Learning script" new_actions_notification_rake_created: subject: "More news about your citizen proposal" hi: "Hello %{name}," diff --git a/config/locales/en/settings.yml b/config/locales/en/settings.yml index c6a4fdd6b..fb6fe0613 100644 --- a/config/locales/en/settings.yml +++ b/config/locales/en/settings.yml @@ -106,6 +106,8 @@ en: recommendations_on_proposals_description: "Displays recommendations to users on the proposals page based on the tags of the items followed" community: "Community on proposals and investments" community_description: "Enables the community section in the proposals and investment projects of the Participatory Budgets" + machine_learning: "AI / Machine learning" + machine_learning_description: "Enable the AI / Machine Learning section to run python scripts and enrich content automatically." map: "Proposals and budget investments geolocation" map_description: "Enables geolocation of proposals and investment projects" allow_images: "Allow upload and show images" diff --git a/config/locales/es/admin.yml b/config/locales/es/admin.yml index f3524df58..f27a7c35d 100644 --- a/config/locales/es/admin.yml +++ b/config/locales/es/admin.yml @@ -750,6 +750,7 @@ es: proposals: "Propuestas" polls: "Votaciones" layouts: "Plantillas" + machine_learning: "IA / Machine Learning" mailers: "Correos" management: "Gestión" welcome: "Bienvenido/a" @@ -770,6 +771,7 @@ es: debates: "Debates" comments: "Comentarios" local_census_records: Gestionar censo local + machine_learning: "IA / Machine learning" administrators: index: title: Administradores @@ -1682,3 +1684,53 @@ es: created: Registros creados local_census_records: no_records_found: No se han encontrado registros. + machine_learning: + cancel: "Cancelar operación" + cancel_alert: "Esta acción cancelará el script actual y será necesario ejecutar el script de nuevo." + comments_summary: "Resumen de comentarios" + comments_summary_description: "Muestra un resumen de comentarios generado automáticamente en todos los elementos que pueden ser comentados." + data_folder_content: "Contenido carpeta data" + error: "Error:" + execute_script: "Ejecutar script" + executed_by: "Ejecutado por:" + executed_script: "Script ejecutado:" + feature_disabled: "Esta funcionalidad está deshabilitada. Para utilizar Machine Learning puedes habilitarla desde la %{link}." + feature_disabled_link: "página de configuración" + help: + title_1: "¿Qué es IA / Machine Learning?" + description_1: "Tradicionalmente, los programas informáticos se diseñan estableciendo reglas precisas sobre lo que debe hacer el programa en cada situación, y cómo procesar en cada momento la información disponible. Lo que se conoce comúnmente como IA / Machine Learning es un tipo de programas para los cuales lo que se establece de manera precisa es cuál es la tarea a realizar y cómo se valora el que la tarea esté mejor o peor realizada. Pero a diferencia de los otros, el sistema descubre o aprende cuál sería el método más adecuado para realizar dicha tarea." + title_2: "¿Qué hace el módulo de IA / Machine Learning en CONSUL?" + description_2: "Este módulo permite implementar en CONSUL cualquier tipo de programas de IA / Machine Learning. Los programas pueden procesar la información disponible en CONSUL y producir resultados que ayuden tanto a los usuarios como a los administradores a llevar a cabo una participación ciudadana más eficaz e inteligente. El módulo se ha desarrollado para que sea sencillo implementar y ejecutar nuevos programas, manteniendo por separado el código general de CONSUL y el código de estos nuevos programas." + title_3: "Cómo usar el módulo" + description_3: "Para usarlo por primera vez es necesario seguir las instrucciones disponibles en la documentación de CONSUL para activar correctamente este módulo. Una vez el módulo esté correctamente activado, los programas se ejecutan desde la pestaña \"Ejecutar scripts\". Antes de ejecutarlo podemos ver en esa misma pestaña una estimación del tiempo de ejecución y otras informaciones relevantes. Al terminar la ejecución podemos activar el contenido que se mostrará en CONSUL desde la pestaña de \"Configuración\", así como descargar otros archivos generados que nos puedan resultar útiles." + title_4: "Cómo implementar nuevos scripts de IA / Machine Learning." + description_4: "Utiliza los scripts existentes como ejemplo." + description_4b: "Los nuevos scripts deben estar ubicados en la carpeta public/machine_learning/scripts" + description_4c: "Los datos de entrada que utilizará el script se crearán como archivos JSON en la carpeta ../data" + description_4d: "Cualquier salida generada debe crearse en la carpeta ../data" + description_4e: "Se recomienda crear un archivo de texto .ini independiente con los parámetros de configuración del script, con el mismo nombre del script y legible mediante configparser. Este archivo facilitará cambios de configuración por parte de los administradores." + description_4f: "Se recomienda incluir como primera línea del script una cadena inicial de comillas triples con información breve sobre el mismo y enlaces a cualquier información relevante. Esta cadena se mostrará automáticamente en la interfaz de administración." + description_4g: "Se recomienda crear un log con la información relevante del script en un archivo de texto .log, con el mismo nombre del script." + title_5: "Para utilizar los scripts de IA / Machine Learning necesitas tener Python instalado en tu servidor" + description_5: "Aquí puedes ver las instrucciones de ejemplo para un servidor Ubuntu 18.04:" + help_text: "Esta funcionalidad es experimental." + last_execution: "Última ejecución" + no_content: "Todavía no se ha generado contenido." + notice: + success: "El último script se ha ejecutado correctamente." + error: "Se ha producido un error. Más información sobre el error a continuación." + working: "El script se está ejecutando. El administrador que lo ejecutó recibirá un email cuando termine." + delete_generated_content: "El contenido generado se ha borrado correctamente." + output_files: "Ficheros de salida:" + related_content: "Contenido relacionado" + related_content_description: "Añade contenido relacionado generado automáticamente a las propuestas y proyectos de presupuestos participativos." + script_info: "Recibirás un email en %{email} cuando el script termine de ejecutarse." + script_name: "Nombre del script:" + select_script: "Seleccione el script pyhton a ejecutar" + started_at: "Empezado a las:" + tab_help: "Ayuda" + tab_settings: "Configuración / Contenido generado" + tab_scripts: "Ejecutar script" + tags: "Etiquetas" + tags_description: "Genera etiquetas automáticas para todos los elementos que pueden ser etiquetados." + title: "IA / Machine learning" diff --git a/config/locales/es/general.yml b/config/locales/es/general.yml index fd9494e41..b6b1a1395 100644 --- a/config/locales/es/general.yml +++ b/config/locales/es/general.yml @@ -965,7 +965,11 @@ es: enqueue_remote_translation: Se han solicitado correctamente las traducciones. button: Traducir página tags: - filter: - more: - one: "Una etiqueta más" - other: "%{count} etiquetas más" + list: + filter: + more: + one: "Una etiqueta más" + other: "%{count} etiquetas más" + machine_learning: + comments_summary: "Resumen de comentarios" + info_text: "Contenido generado mediante IA / Machine Learning" diff --git a/config/locales/es/mailers.yml b/config/locales/es/mailers.yml index 74122ddf2..ec4974a4d 100644 --- a/config/locales/es/mailers.yml +++ b/config/locales/es/mailers.yml @@ -78,6 +78,16 @@ es: hi: Hola new_comment_by: Hay un nuevo comentario de evaluación de %{commenter} en el presupuesto participativo %{investment} commenter_info: "%{commenter}, %{time}" + machine_learning_error: + link: "Visitar el panel de Machine Learning" + subject: "Machine learning - Se ha producido un error" + text: "Se ha producido un error ejecutando el script de Machine Learning." + title: "Script Machine Learning" + machine_learning_success: + link: "Visitar el panel de Machine Learning" + subject: "Machine learning - Contenido generado correctamente" + text: "El contenido ha sido generado correctamente." + title: "Script Machine Learning" new_actions_notification_rake_created: subject: "Más novedades de tu propuesta ciudadana" hi: "Hola %{name}," diff --git a/config/locales/es/settings.yml b/config/locales/es/settings.yml index 98c775cd3..4e2086216 100644 --- a/config/locales/es/settings.yml +++ b/config/locales/es/settings.yml @@ -106,6 +106,8 @@ es: recommendations_on_proposals_description: "Muestra a los usuarios recomendaciones en la página de propuestas basado en las etiquetas de los elementos que sigue" community: "Comunidad en propuestas y proyectos de gasto" community_description: "Activa la sección de comunidad en las propuestas y en los proyectos de gasto de los Presupuestos participativos" + machine_learning: "IA / Machine learning" + machine_learning_description: "Habilita la sección de IA / Machine Learning para ejecutar scripts de python y enriquecer el contenido automáticamente." map: "Geolocalización de propuestas y proyectos de gasto" map_description: "Activa la geolocalización de propuestas y proyectos de gasto" allow_images: "Permitir subir y mostrar imágenes" diff --git a/config/routes/admin.rb b/config/routes/admin.rb index 8749178b1..4a358a0c7 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -270,6 +270,11 @@ namespace :admin do namespace :local_census_records do resources :imports, only: [:new, :create, :show] end + + resource :machine_learning, controller: :machine_learning, only: [:show] do + post :execute, on: :collection + delete :cancel, on: :collection + end end resolve "Milestone" do |milestone| diff --git a/db/migrate/20210422104400_create_machine_learning_jobs.rb b/db/migrate/20210422104400_create_machine_learning_jobs.rb new file mode 100644 index 000000000..37ff2b69a --- /dev/null +++ b/db/migrate/20210422104400_create_machine_learning_jobs.rb @@ -0,0 +1,14 @@ +class CreateMachineLearningJobs < ActiveRecord::Migration[5.2] + def change + create_table :machine_learning_jobs do |t| + t.datetime :started_at + t.datetime :finished_at + t.string :script + t.integer :pid + t.string :error + t.references :user, foreign_key: true + + t.timestamps + end + end +end diff --git a/db/migrate/20210423114500_create_ml_summary_comments.rb b/db/migrate/20210423114500_create_ml_summary_comments.rb new file mode 100644 index 000000000..4ece4cd86 --- /dev/null +++ b/db/migrate/20210423114500_create_ml_summary_comments.rb @@ -0,0 +1,11 @@ +class CreateMlSummaryComments < ActiveRecord::Migration[5.2] + def change + create_table :ml_summary_comments do |t| + t.integer :commentable_id + t.string :commentable_type + t.text :body + + t.timestamps + end + end +end diff --git a/db/migrate/20210424092400_add_machine_learning_fields_to_related_contents.rb b/db/migrate/20210424092400_add_machine_learning_fields_to_related_contents.rb new file mode 100644 index 000000000..2193daa03 --- /dev/null +++ b/db/migrate/20210424092400_add_machine_learning_fields_to_related_contents.rb @@ -0,0 +1,6 @@ +class AddMachineLearningFieldsToRelatedContents < ActiveRecord::Migration[5.2] + def change + add_column :related_contents, :machine_learning, :boolean, default: false + add_column :related_contents, :machine_learning_score, :integer, default: 0 + end +end diff --git a/db/migrate/20210519115700_create_machine_learning_infos.rb b/db/migrate/20210519115700_create_machine_learning_infos.rb new file mode 100644 index 000000000..8866ce4d5 --- /dev/null +++ b/db/migrate/20210519115700_create_machine_learning_infos.rb @@ -0,0 +1,11 @@ +class CreateMachineLearningInfos < ActiveRecord::Migration[5.2] + def change + create_table :machine_learning_infos do |t| + t.string :kind + t.datetime :generated_at + t.string :script + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index b69a16a90..d02fe0d92 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_01_23_100638) do +ActiveRecord::Schema.define(version: 2021_05_19_115700) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -875,6 +875,26 @@ ActiveRecord::Schema.define(version: 2021_01_23_100638) do t.index ["user_id"], name: "index_locks_on_user_id" end + create_table "machine_learning_infos", force: :cascade do |t| + t.string "kind" + t.datetime "generated_at" + t.string "script" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + create_table "machine_learning_jobs", force: :cascade do |t| + t.datetime "started_at" + t.datetime "finished_at" + t.string "script" + t.integer "pid" + t.string "error" + t.bigint "user_id" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["user_id"], name: "index_machine_learning_jobs_on_user_id" + end + create_table "managers", id: :serial, force: :cascade do |t| t.integer "user_id" t.index ["user_id"], name: "index_managers_on_user_id" @@ -920,6 +940,14 @@ ActiveRecord::Schema.define(version: 2021_01_23_100638) do t.index ["status_id"], name: "index_milestones_on_status_id" end + create_table "ml_summary_comments", force: :cascade do |t| + t.integer "commentable_id" + t.string "commentable_type" + t.text "body" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "moderators", id: :serial, force: :cascade do |t| t.integer "user_id" t.index ["user_id"], name: "index_moderators_on_user_id" @@ -1278,6 +1306,8 @@ ActiveRecord::Schema.define(version: 2021_01_23_100638) do t.datetime "hidden_at" t.integer "related_content_scores_count", default: 0 t.integer "author_id" + t.boolean "machine_learning", default: false + t.integer "machine_learning_score", default: 0 t.index ["child_relationable_type", "child_relationable_id"], name: "index_related_contents_on_child_relationable" t.index ["hidden_at"], name: "index_related_contents_on_hidden_at" t.index ["parent_relationable_id", "parent_relationable_type", "child_relationable_id", "child_relationable_type"], name: "unique_parent_child_related_content", unique: true @@ -1704,6 +1734,7 @@ ActiveRecord::Schema.define(version: 2021_01_23_100638) do add_foreign_key "legislation_draft_versions", "legislation_processes" add_foreign_key "legislation_proposals", "legislation_processes" add_foreign_key "locks", "users" + add_foreign_key "machine_learning_jobs", "users" add_foreign_key "managers", "users" add_foreign_key "moderators", "users" add_foreign_key "notifications", "users" diff --git a/public/machine_learning/scripts/budgets_related_content_and_tags_nmf.py b/public/machine_learning/scripts/budgets_related_content_and_tags_nmf.py new file mode 100644 index 000000000..e02639be1 --- /dev/null +++ b/public/machine_learning/scripts/budgets_related_content_and_tags_nmf.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +""" +Related Participatory Budgeting projects and Tags - Dummy script + +""" + + +# In[2]: + + +data_path = '../data' +config_file = 'budgets_related_content_and_tags_nmf.ini' +logging_file ='budgets_related_content_and_tags_nmf.log' + + +# In[3]: + + +# Input file: +inputjsonfile = 'budget_investments.json' + +# Output files: +taggings_filename = 'ml_taggings_budgets.json' +tags_filename = 'ml_tags_budgets.json' +related_props_filename = 'ml_related_content_budgets.json' + + +# In[4]: + + +import os +import pandas as pd + + +# ### Read the proposals + +# In[5]: + + +# proposals_input_df = pd.read_json(os.path.join(data_path,inputjsonfile),orient="records") +# col_id = 'id' +# cols_content = ['title','description'] +# proposals_input_df = proposals_input_df[[col_id]+cols_content] + + +# ### Create file: Taggings. Each line is a Tag associated to a Proposal + +# In[6]: + + +taggings_file_cols = ['tag_id','taggable_id','taggable_type'] +taggings_file_df = pd.DataFrame(columns=taggings_file_cols) +row = [0,1,'Budget::Investment'] +taggings_file_df = taggings_file_df.append(dict(zip(taggings_file_cols,row)), ignore_index=True) +taggings_file_df.to_json(os.path.join(data_path,taggings_filename),orient="records", force_ascii=False) + + +# ### Create file: Tags. List of Tags with the number of times they have been used + +# In[7]: + + +tags_file_cols = ['id','name','taggings_count','kind'] +tags_file_df = pd.DataFrame(columns=tags_file_cols) +row = [0,'tag',0,''] +tags_file_df = tags_file_df.append(dict(zip(tags_file_cols,row)), ignore_index=True) +tags_file_df.to_json(os.path.join(data_path,tags_filename),orient="records", force_ascii=False) + + +# ### Create file: List of related proposals + +# In[8]: + + +numb_related_proposals = 2 +related_props_cols = ['id']+['related'+str(num) for num in range(1,numb_related_proposals+1)] +related_props_df = pd.DataFrame(columns=related_props_cols) +row = [1]+['' for num in range(1,numb_related_proposals+1)] +related_props_df = related_props_df.append(dict(zip(related_props_cols,row)), ignore_index=True) +related_props_df.to_json(os.path.join(data_path,related_props_filename),orient="records", force_ascii=False) + diff --git a/public/machine_learning/scripts/budgets_summary_comments_textrank.py b/public/machine_learning/scripts/budgets_summary_comments_textrank.py new file mode 100644 index 000000000..9f81ad66d --- /dev/null +++ b/public/machine_learning/scripts/budgets_summary_comments_textrank.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +""" +Participatory Budgeting comments summaries - Dummy script + +""" + + +# In[2]: + + +data_path = '../data' +config_file = 'budgets_summary_comments_textrank.ini' +logging_file ='budgets_summary_comments_textrank.log' + + +# In[3]: + + +# Input file: +inputjsonfile = 'comments.json' + +# Output files: +comments_summaries_filename = 'ml_comments_summaries_budgets.json' + + +# In[4]: + + +import os +import pandas as pd + + +# ### Read the comments + +# In[5]: + + +# comments_input_df = pd.read_json(os.path.join(data_path,inputjsonfile),orient="records") +# col_id = 'commentable_id' +# col_content = 'body' +# comments_input_df = comments_input_df[[col_id]+[col_content]] + + +# ### Create file. Comments summaries + +# In[6]: + + +comments_summaries_cols = ['id','commentable_id','commentable_type','body'] +comments_summaries_df = pd.DataFrame(columns=comments_summaries_cols) +row = [0,0,'Budget::Investment','Summary'] +comments_summaries_df = comments_summaries_df.append(dict(zip(comments_summaries_cols,row)), ignore_index=True) +comments_summaries_df.to_json(os.path.join(data_path,comments_summaries_filename),orient="records", force_ascii=False) + diff --git a/public/machine_learning/scripts/proposals_related_content_and_tags_nmf.py b/public/machine_learning/scripts/proposals_related_content_and_tags_nmf.py new file mode 100644 index 000000000..75d2ef02b --- /dev/null +++ b/public/machine_learning/scripts/proposals_related_content_and_tags_nmf.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +""" +Related Proposals and Tags - Dummy script + +""" + + +# In[2]: + + +data_path = '../data' +config_file = 'proposals_related_content_and_tags_nmf.ini' +logging_file ='proposals_related_content_and_tags_nmf.log' + + +# In[3]: + + +# Input file: +inputjsonfile = 'proposals.json' + +# Output files: +taggings_filename = 'ml_taggings_proposals.json' +tags_filename = 'ml_tags_proposals.json' +related_props_filename = 'ml_related_content_proposals.json' + + +# In[4]: + + +import os +import pandas as pd + + +# ### Read the proposals + +# In[5]: + + +# proposals_input_df = pd.read_json(os.path.join(data_path,inputjsonfile),orient="records") +# col_id = 'id' +# cols_content = ['title','description','summary'] +# proposals_input_df = proposals_input_df[[col_id]+cols_content] + + +# ### Create file: Taggings. Each line is a Tag associated to a Proposal + +# In[6]: + + +taggings_file_cols = ['tag_id','taggable_id','taggable_type'] +taggings_file_df = pd.DataFrame(columns=taggings_file_cols) +row = [0,1,'Proposal'] +taggings_file_df = taggings_file_df.append(dict(zip(taggings_file_cols,row)), ignore_index=True) +taggings_file_df.to_json(os.path.join(data_path,taggings_filename),orient="records", force_ascii=False) + + +# ### Create file: Tags. List of Tags with the number of times they have been used + +# In[7]: + + +tags_file_cols = ['id','name','taggings_count','kind'] +tags_file_df = pd.DataFrame(columns=tags_file_cols) +row = [0,'tag',0,''] +tags_file_df = tags_file_df.append(dict(zip(tags_file_cols,row)), ignore_index=True) +tags_file_df.to_json(os.path.join(data_path,tags_filename),orient="records", force_ascii=False) + + +# ### Create file: List of related proposals + +# In[8]: + + +numb_related_proposals = 2 +related_props_cols = ['id']+['related'+str(num) for num in range(1,numb_related_proposals+1)] +related_props_df = pd.DataFrame(columns=related_props_cols) +row = [1]+['' for num in range(1,numb_related_proposals+1)] +related_props_df = related_props_df.append(dict(zip(related_props_cols,row)), ignore_index=True) +related_props_df.to_json(os.path.join(data_path,related_props_filename),orient="records", force_ascii=False) + diff --git a/public/machine_learning/scripts/proposals_summary_comments_textrank.py b/public/machine_learning/scripts/proposals_summary_comments_textrank.py new file mode 100644 index 000000000..2af827979 --- /dev/null +++ b/public/machine_learning/scripts/proposals_summary_comments_textrank.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +""" +Proposals comments summaries - Dummy script + +""" + + +# In[2]: + + +data_path = '../data' +config_file = 'proposals_summary_comments_textrank.ini' +logging_file ='proposals_summary_comments_textrank.log' + + +# In[3]: + + +# Input file: +inputjsonfile = 'comments.json' + +# Output files: +comments_summaries_filename = 'ml_comments_summaries_proposals.json' + + +# In[4]: + + +import os +import pandas as pd + + +# ### Read the comments + +# In[5]: + + +# comments_input_df = pd.read_json(os.path.join(data_path,inputjsonfile),orient="records") +# col_id = 'commentable_id' +# col_content = 'body' +# comments_input_df = comments_input_df[[col_id]+[col_content]] + + +# ### Create file. Comments summaries + +# In[6]: + + +comments_summaries_cols = ['id','commentable_id','commentable_type','body'] +comments_summaries_df = pd.DataFrame(columns=comments_summaries_cols) +row = [0,0,'Proposal','Summary'] +comments_summaries_df = comments_summaries_df.append(dict(zip(comments_summaries_cols,row)), ignore_index=True) +comments_summaries_df.to_json(os.path.join(data_path,comments_summaries_filename),orient="records", force_ascii=False) + diff --git a/spec/components/machine_learning/comments_summary_component_spec.rb b/spec/components/machine_learning/comments_summary_component_spec.rb new file mode 100644 index 000000000..993077b65 --- /dev/null +++ b/spec/components/machine_learning/comments_summary_component_spec.rb @@ -0,0 +1,44 @@ +require "rails_helper" + +describe MachineLearning::CommentsSummaryComponent, type: :component do + let(:commentable) { double(summary_comment: double(body: "There's a general agreement")) } + let(:component) { MachineLearning::CommentsSummaryComponent.new(commentable) } + + before do + Setting["feature.machine_learning"] = true + Setting["machine_learning.comments_summary"] = true + allow(controller).to receive(:current_user).and_return(nil) + end + + it "is displayed when the setting is enabled" do + render_inline component + + expect(page).to have_content "Comments summary" + expect(page).to have_content "There's a general agreement" + expect(page).to have_content "Content generated by AI / Machine Learning" + end + + it "is not displayed when the setting is disabled" do + Setting["machine_learning.comments_summary"] = false + + render_inline component + + expect(page.native.inner_html).to be_empty + end + + it "is not displayed when the machine learning feature is disabled" do + Setting["feature.machine_learning"] = false + + render_inline component + + expect(page.native.inner_html).to be_empty + end + + it "is not displayed when there's no summary" do + commentable = double(summary_comment: double(body: "")) + + render_inline MachineLearning::CommentsSummaryComponent.new(commentable) + + expect(page.native.inner_html).to be_empty + end +end diff --git a/spec/components/relationable/related_list_component_spec.rb b/spec/components/relationable/related_list_component_spec.rb new file mode 100644 index 000000000..55ab43729 --- /dev/null +++ b/spec/components/relationable/related_list_component_spec.rb @@ -0,0 +1,48 @@ +require "rails_helper" + +describe Relationable::RelatedListComponent, type: :component do + let(:proposal) { create(:proposal) } + let(:user_proposal) { create(:proposal, title: "I am user related") } + let(:machine_proposal) { create(:proposal, title: "I am machine related") } + let(:component) { Relationable::RelatedListComponent.new(proposal) } + + before do + Setting["feature.machine_learning"] = true + Setting["machine_learning.related_content"] = true + + create(:related_content, parent_relationable: proposal, child_relationable: user_proposal) + create(:related_content, parent_relationable: proposal, + child_relationable: machine_proposal, + machine_learning: true) + + allow(controller).to receive(:current_user).and_return(nil) + end + + it "displays machine learning and user content when machine learning is enabled" do + render_inline component + + expect(page).to have_css "li", count: 2 + expect(page).to have_content "I am machine related" + expect(page).to have_content "I am user related" + end + + it "displays user related content when machine learning is disabled" do + Setting["feature.machine_learning"] = false + + render_inline component + + expect(page).to have_css "li", count: 1 + expect(page).to have_content "I am user related" + expect(page).not_to have_content "I am machine related" + end + + it "displays user related content when machine learning related content is disabled" do + Setting["machine_learning.related_content"] = false + + render_inline component + + expect(page).to have_css "li", count: 1 + expect(page).to have_content "I am user related" + expect(page).not_to have_content "I am machine related" + end +end diff --git a/spec/components/shared/tag_list_component_spec.rb b/spec/components/shared/tag_list_component_spec.rb new file mode 100644 index 000000000..be128370f --- /dev/null +++ b/spec/components/shared/tag_list_component_spec.rb @@ -0,0 +1,48 @@ +require "rails_helper" + +describe Shared::TagListComponent, type: :component do + let(:user_tag) { create(:tag, name: "user tag") } + let(:ml_tag) { create(:tag, name: "machine learning tag") } + let(:proposal) { create(:proposal, tag_list: [user_tag], ml_tag_list: [ml_tag]) } + let(:component) { Shared::TagListComponent.new(proposal, limit: nil) } + + before do + Setting["feature.machine_learning"] = true + Setting["machine_learning.tags"] = true + allow(controller).to receive(:current_user).and_return(create(:administrator).user) + end + + it "displays machine learning tags when machine learning is enabled" do + render_inline component + + expect(page).not_to have_link "user tag" + expect(page).to have_link "machine learning tag" + expect(page).to have_content "Content generated by AI / Machine Learning" + end + + it "displays user tags when machine learning is disabled" do + Setting["feature.machine_learning"] = false + + render_inline component + + expect(page).to have_link "user tag" + expect(page).not_to have_link "machine learning tag" + expect(page).not_to have_content "Content generated by AI / Machine Learning" + end + + it "displays user tags when machine learning tags are disabled" do + Setting["machine_learning.tags"] = false + + render_inline component + + expect(page).to have_link "user tag" + expect(page).not_to have_link "machine learning tag" + expect(page).not_to have_content "Content generated by AI / Machine Learning" + end + + it "is not rendered when there are no tags" do + render_inline Shared::TagListComponent.new(Proposal.new, limit: nil) + + expect(page.native.inner_html).to be_empty + end +end diff --git a/spec/factories/classifications.rb b/spec/factories/classifications.rb index d1f107814..c48c0e0f6 100644 --- a/spec/factories/classifications.rb +++ b/spec/factories/classifications.rb @@ -41,6 +41,20 @@ FactoryBot.define do association :author, factory: :user association :parent_relationable, factory: [:proposal, :debate].sample association :child_relationable, factory: [:proposal, :debate].sample + + trait :proposals do + association :parent_relationable, factory: :proposal + association :child_relationable, factory: :proposal + end + + trait :budget_investments do + association :parent_relationable, factory: :budget_investment + association :child_relationable, factory: :budget_investment + end + + trait :from_machine_learning do + machine_learning { true } + end end factory :related_content_score do diff --git a/spec/factories/machine_learning.rb b/spec/factories/machine_learning.rb new file mode 100644 index 000000000..98e77984e --- /dev/null +++ b/spec/factories/machine_learning.rb @@ -0,0 +1,19 @@ +FactoryBot.define do + factory :machine_learning_job do + association :user, factory: :user + script { "script.py" } + started_at { Time.current } + finished_at { nil } + error { nil } + end + + factory :machine_learning_info do + kind { "tags" } + generated_at { Time.current } + script { "script.py" } + end + + factory :ml_summary_comment do + body { "Summary comment generated by Machine Learning" } + end +end diff --git a/spec/models/machine_learning_spec.rb b/spec/models/machine_learning_spec.rb new file mode 100644 index 000000000..ebbb1c607 --- /dev/null +++ b/spec/models/machine_learning_spec.rb @@ -0,0 +1,610 @@ +require "rails_helper" + +describe MachineLearning do + def full_sanitizer(string) + ActionView::Base.full_sanitizer.sanitize(string) + end + + let(:job) { create(:machine_learning_job) } + + describe "#cleanup_proposals_tags!" do + it "does not delete other machine learning generated data" do + create(:ml_summary_comment, commentable: create(:proposal)) + create(:ml_summary_comment, commentable: create(:budget_investment)) + + create(:related_content, :proposals, :from_machine_learning) + create(:related_content, :budget_investments, :from_machine_learning) + + expect(MlSummaryComment.count).to be 2 + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_proposals_tags!) + + expect(MlSummaryComment.count).to be 2 + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + end + + it "deletes proposals tags machine learning generated data" do + proposal = create(:proposal) + investment = create(:budget_investment) + + user_tag = create(:tag) + create(:tagging, tag: user_tag, taggable: proposal) + + ml_proposal_tag = create(:tag) + create(:tagging, tag: ml_proposal_tag, taggable: proposal, context: "ml_tags") + + ml_investment_tag = create(:tag) + create(:tagging, tag: ml_investment_tag, taggable: investment, context: "ml_tags") + + common_tag = create(:tag) + create(:tagging, tag: common_tag, taggable: proposal) + create(:tagging, tag: common_tag, taggable: proposal, context: "ml_tags") + create(:tagging, tag: common_tag, taggable: investment, context: "ml_tags") + + expect(Tag.count).to be 4 + expect(Tagging.count).to be 6 + expect(Tagging.where(context: "tags").count).to be 2 + expect(Tagging.where(context: "ml_tags", taggable_type: "Proposal").count).to be 2 + expect(Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment").count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_proposals_tags!) + + expect(Tag.count).to be 3 + expect(Tag.all).not_to include ml_proposal_tag + expect(Tagging.count).to be 4 + expect(Tagging.where(context: "tags").count).to be 2 + expect(Tagging.where(context: "ml_tags", taggable_type: "Proposal")).to be_empty + expect(Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment").count).to be 2 + end + end + + describe "#cleanup_investments_tags!" do + it "does not delete other machine learning generated data" do + create(:ml_summary_comment, commentable: create(:proposal)) + create(:ml_summary_comment, commentable: create(:budget_investment)) + + create(:related_content, :proposals, :from_machine_learning) + create(:related_content, :budget_investments, :from_machine_learning) + + expect(MlSummaryComment.count).to be 2 + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_investments_tags!) + + expect(MlSummaryComment.count).to be 2 + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + end + + it "deletes investments tags machine learning generated data" do + proposal = create(:proposal) + investment = create(:budget_investment) + + user_tag = create(:tag) + create(:tagging, tag: user_tag, taggable: investment) + + ml_investment_tag = create(:tag) + create(:tagging, tag: ml_investment_tag, taggable: investment, context: "ml_tags") + + ml_proposal_tag = create(:tag) + create(:tagging, tag: ml_proposal_tag, taggable: proposal, context: "ml_tags") + + common_tag = create(:tag) + create(:tagging, tag: common_tag, taggable: investment) + create(:tagging, tag: common_tag, taggable: investment, context: "ml_tags") + create(:tagging, tag: common_tag, taggable: proposal, context: "ml_tags") + + expect(Tag.count).to be 4 + expect(Tagging.count).to be 6 + expect(Tagging.where(context: "tags").count).to be 2 + expect(Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment").count).to be 2 + expect(Tagging.where(context: "ml_tags", taggable_type: "Proposal").count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_investments_tags!) + + expect(Tag.count).to be 3 + expect(Tag.all).not_to include ml_investment_tag + expect(Tagging.count).to be 4 + expect(Tagging.where(context: "tags").count).to be 2 + expect(Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment")).to be_empty + expect(Tagging.where(context: "ml_tags", taggable_type: "Proposal").count).to be 2 + end + end + + describe "#cleanup_proposals_related_content!" do + it "does not delete other machine learning generated data" do + proposal = create(:proposal) + investment = create(:budget_investment) + + create(:ml_summary_comment, commentable: proposal) + create(:ml_summary_comment, commentable: investment) + + create(:tagging, tag: create(:tag)) + create(:tagging, tag: create(:tag), context: "ml_tags", taggable: proposal) + create(:tagging, tag: create(:tag), context: "ml_tags", taggable: investment) + + expect(MlSummaryComment.count).to be 2 + expect(Tag.count).to be 3 + expect(Tagging.count).to be 3 + expect(Tagging.where(context: "tags").count).to be 1 + expect(Tagging.where(context: "ml_tags").count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_proposals_related_content!) + + expect(MlSummaryComment.count).to be 2 + expect(Tag.count).to be 3 + expect(Tagging.count).to be 3 + expect(Tagging.where(context: "tags").count).to be 1 + expect(Tagging.where(context: "ml_tags").count).to be 2 + end + + it "deletes proposals related content machine learning generated data" do + create(:related_content, :proposals) + create(:related_content, :budget_investments) + create(:related_content, :proposals, :from_machine_learning) + create(:related_content, :budget_investments, :from_machine_learning) + + expect(RelatedContent.for_proposals.from_users.count).to be 2 + expect(RelatedContent.for_investments.from_users.count).to be 2 + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_proposals_related_content!) + + expect(RelatedContent.for_proposals.from_users.count).to be 2 + expect(RelatedContent.for_investments.from_users.count).to be 2 + expect(RelatedContent.for_proposals.from_machine_learning).to be_empty + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + end + end + + describe "#cleanup_investments_related_content!" do + it "does not delete other machine learning generated data" do + proposal = create(:proposal) + investment = create(:budget_investment) + + create(:ml_summary_comment, commentable: proposal) + create(:ml_summary_comment, commentable: investment) + + create(:tagging, tag: create(:tag)) + create(:tagging, tag: create(:tag), context: "ml_tags", taggable: proposal) + create(:tagging, tag: create(:tag), context: "ml_tags", taggable: investment) + + expect(MlSummaryComment.count).to be 2 + expect(Tag.count).to be 3 + expect(Tagging.count).to be 3 + expect(Tagging.where(context: "tags").count).to be 1 + expect(Tagging.where(context: "ml_tags").count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_investments_related_content!) + + expect(MlSummaryComment.count).to be 2 + expect(Tag.count).to be 3 + expect(Tagging.count).to be 3 + expect(Tagging.where(context: "tags").count).to be 1 + expect(Tagging.where(context: "ml_tags").count).to be 2 + end + + it "deletes proposals related content machine learning generated data" do + create(:related_content, :proposals) + create(:related_content, :budget_investments) + create(:related_content, :proposals, :from_machine_learning) + create(:related_content, :budget_investments, :from_machine_learning) + + expect(RelatedContent.for_proposals.from_users.count).to be 2 + expect(RelatedContent.for_investments.from_users.count).to be 2 + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_investments_related_content!) + + expect(RelatedContent.for_proposals.from_users.count).to be 2 + expect(RelatedContent.for_investments.from_users.count).to be 2 + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning).to be_empty + end + end + + describe "#cleanup_proposals_comments_summary!" do + it "does not delete other machine learning generated data" do + create(:related_content, :proposals, :from_machine_learning) + create(:related_content, :budget_investments, :from_machine_learning) + + create(:tagging, tag: create(:tag)) + create(:tagging, tag: create(:tag), context: "ml_tags", taggable: create(:proposal)) + create(:tagging, tag: create(:tag), context: "ml_tags", taggable: create(:budget_investment)) + + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + expect(Tag.count).to be 3 + expect(Tagging.count).to be 3 + expect(Tagging.where(context: "tags").count).to be 1 + expect(Tagging.where(context: "ml_tags").count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_proposals_comments_summary!) + + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + expect(Tag.count).to be 3 + expect(Tagging.count).to be 3 + expect(Tagging.where(context: "tags").count).to be 1 + expect(Tagging.where(context: "ml_tags").count).to be 2 + end + + it "deletes proposals comments summary machine learning generated data" do + create(:ml_summary_comment, commentable: create(:proposal)) + create(:ml_summary_comment, commentable: create(:budget_investment)) + + expect(MlSummaryComment.where(commentable_type: "Proposal").count).to be 1 + expect(MlSummaryComment.where(commentable_type: "Budget::Investment").count).to be 1 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_proposals_comments_summary!) + + expect(MlSummaryComment.where(commentable_type: "Proposal")).to be_empty + expect(MlSummaryComment.where(commentable_type: "Budget::Investment").count).to be 1 + end + end + + describe "#cleanup_investments_comments_summary!" do + it "does not delete other machine learning generated data" do + create(:related_content, :proposals, :from_machine_learning) + create(:related_content, :budget_investments, :from_machine_learning) + + create(:tagging, tag: create(:tag)) + create(:tagging, tag: create(:tag), context: "ml_tags", taggable: create(:proposal)) + create(:tagging, tag: create(:tag), context: "ml_tags", taggable: create(:budget_investment)) + + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + expect(Tag.count).to be 3 + expect(Tagging.count).to be 3 + expect(Tagging.where(context: "tags").count).to be 1 + expect(Tagging.where(context: "ml_tags").count).to be 2 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_investments_comments_summary!) + + expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2 + expect(RelatedContent.for_investments.from_machine_learning.count).to be 2 + expect(Tag.count).to be 3 + expect(Tagging.count).to be 3 + expect(Tagging.where(context: "tags").count).to be 1 + expect(Tagging.where(context: "ml_tags").count).to be 2 + end + + it "deletes budget investments comments summary machine learning generated data" do + create(:ml_summary_comment, commentable: create(:proposal)) + create(:ml_summary_comment, commentable: create(:budget_investment)) + + expect(MlSummaryComment.where(commentable_type: "Proposal").count).to be 1 + expect(MlSummaryComment.where(commentable_type: "Budget::Investment").count).to be 1 + + machine_learning = MachineLearning.new(job) + machine_learning.send(:cleanup_investments_comments_summary!) + + expect(MlSummaryComment.where(commentable_type: "Proposal").count).to be 1 + expect(MlSummaryComment.where(commentable_type: "Budget::Investment")).to be_empty + end + end + + describe "#export_proposals_to_json" do + it "creates a JSON file with all proposals" do + require "fileutils" + FileUtils.mkdir_p Rails.root.join("public", "machine_learning", "data") + + first_proposal = create(:proposal) + last_proposal = create(:proposal) + + machine_learning = MachineLearning.new(job) + machine_learning.send(:export_proposals_to_json) + + json_file = MachineLearning::DATA_FOLDER.join("proposals.json") + json = JSON.parse(File.read(json_file)) + + expect(json).to be_an Array + expect(json.size).to be 2 + + expect(json.first["id"]).to eq first_proposal.id + expect(json.first["title"]).to eq first_proposal.title + expect(json.first["summary"]).to eq full_sanitizer(first_proposal.summary) + expect(json.first["description"]).to eq full_sanitizer(first_proposal.description) + + expect(json.last["id"]).to eq last_proposal.id + expect(json.last["title"]).to eq last_proposal.title + expect(json.last["summary"]).to eq full_sanitizer(last_proposal.summary) + expect(json.last["description"]).to eq full_sanitizer(last_proposal.description) + end + end + + describe "#export_budget_investments_to_json" do + it "creates a JSON file with all budget investments" do + require "fileutils" + FileUtils.mkdir_p Rails.root.join("public", "machine_learning", "data") + + first_budget_investment = create(:budget_investment) + last_budget_investment = create(:budget_investment) + + machine_learning = MachineLearning.new(job) + machine_learning.send(:export_budget_investments_to_json) + + json_file = MachineLearning::DATA_FOLDER.join("budget_investments.json") + json = JSON.parse(File.read(json_file)) + + expect(json).to be_an Array + expect(json.size).to be 2 + + expect(json.first["id"]).to eq first_budget_investment.id + expect(json.first["title"]).to eq first_budget_investment.title + expect(json.first["description"]).to eq full_sanitizer(first_budget_investment.description) + + expect(json.last["id"]).to eq last_budget_investment.id + expect(json.last["title"]).to eq last_budget_investment.title + expect(json.last["description"]).to eq full_sanitizer(last_budget_investment.description) + end + end + + describe "#export_comments_to_json" do + it "creates a JSON file with all comments" do + require "fileutils" + FileUtils.mkdir_p Rails.root.join("public", "machine_learning", "data") + + first_comment = create(:comment) + last_comment = create(:comment) + + machine_learning = MachineLearning.new(job) + machine_learning.send(:export_comments_to_json) + + json_file = MachineLearning::DATA_FOLDER.join("comments.json") + json = JSON.parse(File.read(json_file)) + + expect(json).to be_an Array + expect(json.size).to be 2 + + expect(json.first["id"]).to eq first_comment.id + expect(json.first["commentable_id"]).to eq first_comment.commentable_id + expect(json.first["commentable_type"]).to eq first_comment.commentable_type + expect(json.first["body"]).to eq full_sanitizer(first_comment.body) + + expect(json.last["id"]).to eq last_comment.id + expect(json.last["commentable_id"]).to eq last_comment.commentable_id + expect(json.last["commentable_type"]).to eq last_comment.commentable_type + expect(json.last["body"]).to eq full_sanitizer(last_comment.body) + end + end + + describe "#run_machine_learning_scripts" do + it "returns true if python script executed correctly" do + machine_learning = MachineLearning.new(job) + + command = "cd #{MachineLearning::SCRIPTS_FOLDER} && python script.py 2>&1" + expect(machine_learning).to receive(:`).with(command) do + Process.waitpid Process.fork { exit 0 } + end + + expect(Mailer).not_to receive(:machine_learning_error) + + expect(machine_learning.send(:run_machine_learning_scripts)).to be true + + job.reload + expect(job.finished_at).not_to be_present + expect(job.error).not_to be_present + end + + it "returns false if python script errored" do + machine_learning = MachineLearning.new(job) + + command = "cd #{MachineLearning::SCRIPTS_FOLDER} && python script.py 2>&1" + expect(machine_learning).to receive(:`).with(command) do + Process.waitpid Process.fork { abort "error message" } + end + + mailer = double("mailer") + expect(mailer).to receive(:deliver_later) + expect(Mailer).to receive(:machine_learning_error).and_return mailer + + expect(machine_learning.send(:run_machine_learning_scripts)).to be false + + job.reload + expect(job.finished_at).to be_present + expect(job.error).not_to eq "error message" + end + end + + describe "#import_ml_proposals_comments_summary" do + it "feeds the database using content from the JSON file generated by the machine learning script" do + machine_learning = MachineLearning.new(job) + + proposal = create(:proposal) + + data = [ + { commentable_id: proposal.id, + commentable_type: "Proposal", + body: "Summary comment for proposal with ID #{proposal.id}" } + ] + + filename = "ml_comments_summaries_proposals.json" + json_file = MachineLearning::DATA_FOLDER.join(filename) + expect(File).to receive(:read).with(json_file).and_return data.to_json + + machine_learning.send(:import_ml_proposals_comments_summary) + + expect(proposal.summary_comment.body).to eq "Summary comment for proposal with ID #{proposal.id}" + end + end + + describe "#import_ml_investments_comments_summary" do + it "feeds the database using content from the JSON file generated by the machine learning script" do + machine_learning = MachineLearning.new(job) + + investment = create(:budget_investment) + + data = [ + { commentable_id: investment.id, + commentable_type: "Budget::Investment", + body: "Summary comment for investment with ID #{investment.id}" } + ] + + filename = "ml_comments_summaries_budgets.json" + json_file = MachineLearning::DATA_FOLDER.join(filename) + expect(File).to receive(:read).with(json_file).and_return data.to_json + + machine_learning.send(:import_ml_investments_comments_summary) + + expect(investment.summary_comment.body).to eq "Summary comment for investment with ID #{investment.id}" + end + end + + describe "#import_proposals_related_content" do + it "feeds the database using content from the JSON file generated by the machine learning script" do + machine_learning = MachineLearning.new(job) + + proposal = create(:proposal) + related_proposal = create(:proposal) + other_related_proposal = create(:proposal) + + data = [ + { + "id" => proposal.id, + "related1" => related_proposal.id, + "related2" => other_related_proposal.id + } + ] + + filename = "ml_related_content_proposals.json" + json_file = MachineLearning::DATA_FOLDER.join(filename) + expect(File).to receive(:read).with(json_file).and_return data.to_json + + machine_learning.send(:import_proposals_related_content) + + expect(proposal.related_contents.count).to be 2 + expect(proposal.related_contents.first.child_relationable).to eq related_proposal + expect(proposal.related_contents.last.child_relationable).to eq other_related_proposal + end + end + + describe "#import_budget_investments_related_content" do + it "feeds the database using content from the JSON file generated by the machine learning script" do + machine_learning = MachineLearning.new(job) + + investment = create(:budget_investment) + related_investment = create(:budget_investment) + other_related_investment = create(:budget_investment) + + data = [ + { + "id" => investment.id, + "related1" => related_investment.id, + "related2" => other_related_investment.id + } + ] + + filename = "ml_related_content_budgets.json" + json_file = MachineLearning::DATA_FOLDER.join(filename) + expect(File).to receive(:read).with(json_file).and_return data.to_json + + machine_learning.send(:import_budget_investments_related_content) + + expect(investment.related_contents.count).to be 2 + expect(investment.related_contents.first.child_relationable).to eq related_investment + expect(investment.related_contents.last.child_relationable).to eq other_related_investment + end + end + + describe "#import_ml_proposals_tags" do + it "feeds the database using content from the JSON file generated by the machine learning script" do + create(:tag, name: "Existing tag") + proposal = create(:proposal) + machine_learning = MachineLearning.new(job) + + tags_data = [ + { id: 0, + name: "Existing tag" }, + { id: 1, + name: "Machine learning tag" } + ] + + taggings_data = [ + { tag_id: 0, + taggable_id: proposal.id + }, + { tag_id: 1, + taggable_id: proposal.id + } + ] + + tags_filename = "ml_tags_proposals.json" + tags_json_file = MachineLearning::DATA_FOLDER.join(tags_filename) + expect(File).to receive(:read).with(tags_json_file).and_return tags_data.to_json + + taggings_filename = "ml_taggings_proposals.json" + taggings_json_file = MachineLearning::DATA_FOLDER.join(taggings_filename) + expect(File).to receive(:read).with(taggings_json_file).and_return taggings_data.to_json + + machine_learning.send(:import_ml_proposals_tags) + + expect(Tag.count).to be 2 + expect(Tag.first.name).to eq "Existing tag" + expect(Tag.last.name).to eq "Machine learning tag" + expect(proposal.tags).to be_empty + expect(proposal.ml_tags.count).to be 2 + expect(proposal.ml_tags.first.name).to eq "Existing tag" + expect(proposal.ml_tags.last.name).to eq "Machine learning tag" + end + end + + describe "#import_ml_investments_tags" do + it "feeds the database using content from the JSON file generated by the machine learning script" do + create(:tag, name: "Existing tag") + investment = create(:budget_investment) + machine_learning = MachineLearning.new(job) + + tags_data = [ + { id: 0, + name: "Existing tag" }, + { id: 1, + name: "Machine learning tag" } + ] + + taggings_data = [ + { tag_id: 0, + taggable_id: investment.id + }, + { tag_id: 1, + taggable_id: investment.id + } + ] + + tags_filename = "ml_tags_budgets.json" + tags_json_file = MachineLearning::DATA_FOLDER.join(tags_filename) + expect(File).to receive(:read).with(tags_json_file).and_return tags_data.to_json + + taggings_filename = "ml_taggings_budgets.json" + taggings_json_file = MachineLearning::DATA_FOLDER.join(taggings_filename) + expect(File).to receive(:read).with(taggings_json_file).and_return taggings_data.to_json + + machine_learning.send(:import_ml_investments_tags) + + expect(Tag.count).to be 2 + expect(Tag.first.name).to eq "Existing tag" + expect(Tag.last.name).to eq "Machine learning tag" + expect(investment.tags).to be_empty + expect(investment.ml_tags.count).to be 2 + expect(investment.ml_tags.first.name).to eq "Existing tag" + expect(investment.ml_tags.last.name).to eq "Machine learning tag" + end + end +end diff --git a/spec/system/admin/machine_learning_spec.rb b/spec/system/admin/machine_learning_spec.rb new file mode 100644 index 000000000..d5db40a76 --- /dev/null +++ b/spec/system/admin/machine_learning_spec.rb @@ -0,0 +1,255 @@ +require "rails_helper" + +describe "Machine learning" do + let(:admin) { create(:administrator) } + + before do + login_as(admin.user) + Setting["feature.machine_learning"] = true + end + + scenario "Section does not appear if feature is not enabled" do + Setting["feature.machine_learning"] = false + + visit admin_root_path + + within "#admin_menu" do + expect(page).not_to have_link "AI / Machine learning" + end + end + + scenario "Section appears if feature is enabled" do + visit admin_root_path + + within "#admin_menu" do + expect(page).to have_link "AI / Machine learning" + end + + click_link "AI / Machine learning" + + expect(page).to have_content "AI / Machine learning" + expect(page).to have_content "This functionality is experimental" + expect(page).to have_link "Execute script" + expect(page).to have_link "Settings / Generated content" + expect(page).to have_link "Help" + expect(page).to have_current_path(admin_machine_learning_path) + end + + scenario "Show message if feature is disabled" do + Setting["feature.machine_learning"] = false + + visit admin_machine_learning_path + + expect(page).to have_content "This feature is disabled. To use Machine Learning you can enable it from "\ + "the settings page" + expect(page).to have_link "settings page", href: admin_settings_path(anchor: "tab-feature-flags") + end + + scenario "Script executed sucessfully" do + allow_any_instance_of(MachineLearning).to receive(:run) do + MachineLearningJob.first.update!(finished_at: Time.current) + end + + visit admin_machine_learning_path + + select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute" + click_button "Execute script" + + expect(page).to have_content "The last script has been executed successfully." + expect(page).to have_content "You will receive an email in #{admin.email} when the script "\ + "finishes running." + + expect(page).to have_field "Select python script to execute" + expect(page).to have_button "Execute script" + end + + scenario "Settings" do + visit admin_machine_learning_path + + within "#machine_learning_tabs" do + click_link "Settings / Generated content" + end + + expect(page).to have_content "Related content" + expect(page).to have_content "Adds automatically generated related content to proposals and "\ + "participatory budget projects" + + expect(page).to have_content "Comments summary" + expect(page).to have_content "Displays an automatically generated comment summary on all items that "\ + "can be commented on." + + expect(page).to have_content "Tags" + expect(page).to have_content "Generates automatic tags on all items that can be tagged on." + + expect(page).to have_content "No content generated yet", count: 3 + expect(page).not_to have_button "Yes" + expect(page).not_to have_button "No" + end + + scenario "Script started but not finished yet" do + allow_any_instance_of(MachineLearning).to receive(:run) + + visit admin_machine_learning_path + + select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute" + click_button "Execute script" + + expect(page).to have_content "The script is running. The administrator who executed it will receive "\ + "an email when it is finished." + + expect(page).to have_content "Executed by: #{admin.name}" + expect(page).to have_content "Script name: proposals_related_content_and_tags_nmf.py" + expect(page).to have_content "Started at:" + + expect(page).not_to have_content "Select python script to execute" + expect(page).not_to have_button "Execute script" + end + + scenario "Admin can cancel operation if script is working for too long" do + allow_any_instance_of(MachineLearning).to receive(:run) do + MachineLearningJob.first.update!(started_at: 25.hours.ago) + end + + visit admin_machine_learning_path + + select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute" + click_button "Execute script" + + accept_confirm { click_button "Cancel operation" } + + expect(page).to have_content "Generated content has been successfully deleted." + + expect(page).to have_field "Select python script to execute" + expect(page).to have_button "Execute script" + end + + scenario "Script finished with an error" do + allow_any_instance_of(MachineLearning).to receive(:run) do + MachineLearningJob.first.update!(finished_at: Time.current, error: "Error description") + end + + visit admin_machine_learning_path + + select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute" + click_button "Execute script" + + expect(page).to have_content "An error has occurred. You can see the details below." + + expect(page).to have_content "Executed by: #{admin.name}" + expect(page).to have_content "Script name: proposals_related_content_and_tags_nmf.py" + expect(page).to have_content "Error: Error description" + + expect(page).to have_content "You will receive an email in #{admin.email} when the script "\ + "finishes running." + + expect(page).to have_field "Select python script to execute" + expect(page).to have_button "Execute script" + end + + scenario "Email content received by the user who execute the script" do + reset_mailer + Mailer.machine_learning_success(admin.user).deliver + + email = open_last_email + expect(email).to have_subject "Machine Learning - Content has been generated successfully" + expect(email).to have_content "Machine Learning script" + expect(email).to have_content "Content has been generated successfully." + expect(email).to have_link "Visit Machine Learning panel" + expect(email).to deliver_to(admin.user.email) + + reset_mailer + Mailer.machine_learning_error(admin.user).deliver + + email = open_last_email + expect(email).to have_subject "Machine Learning - An error has occurred running the script" + expect(email).to have_content "Machine Learning script" + expect(email).to have_content "An error has occurred running the Machine Learning script." + expect(email).to have_link "Visit Machine Learning panel" + expect(email).to deliver_to(admin.user.email) + end + + scenario "Machine Learning visualization settings are disabled by default" do + allow_any_instance_of(MachineLearning).to receive(:run) do + MachineLearningJob.first.update!(finished_at: Time.current) + end + + visit admin_machine_learning_path + + select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute" + click_button "Execute script" + + expect(page).to have_content "The last script has been executed successfully." + + within "#machine_learning_tabs" do + click_link "Settings / Generated content" + end + + expect(page).to have_content "No content generated yet", count: 3 + expect(page).not_to have_button "Yes" + expect(page).not_to have_button "No" + end + + scenario "Show script descriptions" do + visit admin_machine_learning_path + + select "proposals_summary_comments_textrank.py", from: "Select python script to execute" + + within "#script_descriptions" do + expect(page).to have_content "Proposals comments summaries - Dummy script" + end + + select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute" + + within "#script_descriptions" do + expect(page).to have_content "Related Proposals and Tags - Dummy script" + expect(page).not_to have_content "Proposals comments summaries - Dummy script" + end + end + + scenario "Show output files info on settins page" do + require "fileutils" + FileUtils.mkdir_p Rails.root.join("public", "machine_learning", "data") + + allow_any_instance_of(MachineLearning).to receive(:run) do + MachineLearningJob.first.update!(finished_at: Time.current + 2.minutes) + create(:machine_learning_info, + script: "proposals_summary_comments_textrank.py", + kind: "comments_summary", + updated_at: Time.current + 2.minutes) + comments_file = MachineLearning::DATA_FOLDER.join(MachineLearning.comments_filename) + File.open(comments_file, "w") { |file| file.write([].to_json) } + proposals_comments_summary_file = MachineLearning::DATA_FOLDER.join(MachineLearning.proposals_comments_summary_filename) + File.open(proposals_comments_summary_file, "w") { |file| file.write([].to_json) } + end + + visit admin_machine_learning_path + + within "#machine_learning_tabs" do + click_link "Settings / Generated content" + end + + expect(page).to have_content "No content generated yet.", count: 3 + + within "#machine_learning_tabs" do + click_link "Execute script" + end + + travel_to(Time.zone.local(2019, 10, 20, 14, 57, 25)) do + select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute" + click_button "Execute script" + + expect(page).to have_content "The last script has been executed successfully." + end + + within "#machine_learning_tabs" do + click_link "Settings / Generated content" + end + + expect(page).to have_content "Last execution\n2019-10-20 14:57 - 2019-10-20 14:59" + expect(page).to have_content "Executed script:" + expect(page).to have_content "proposals_summary_comments_textrank.py" + expect(page).to have_content "Output files:" + expect(page).to have_link "ml_comments_summaries_proposals.json", + href: "/machine_learning/data/ml_comments_summaries_proposals.json" + end +end diff --git a/spec/system/machine_learning_spec.rb b/spec/system/machine_learning_spec.rb new file mode 100644 index 000000000..0017b5a76 --- /dev/null +++ b/spec/system/machine_learning_spec.rb @@ -0,0 +1,75 @@ +require "rails_helper" + +describe "Machine learning" do + let(:user_tag) { create(:tag, name: "user tag") } + let(:ml_proposal_tag) { create(:tag, name: "machine learning proposal tag") } + let(:ml_investment_tag) { create(:tag, name: "machine learning investment tag") } + let(:proposal) { create(:proposal) } + let(:related_proposal) { create(:proposal) } + let(:investment) { create(:budget_investment) } + let(:related_investment) { create(:budget_investment) } + + before do + Setting["feature.machine_learning"] = true + Setting["machine_learning.comments_summary"] = true + Setting["machine_learning.related_content"] = true + Setting["machine_learning.tags"] = true + + proposal.update!(tag_list: [user_tag]) + proposal.update!(ml_tag_list: [ml_proposal_tag]) + investment.update!(tag_list: [user_tag]) + investment.update!(ml_tag_list: [ml_investment_tag]) + end + + scenario "proposal view" do + create(:ml_summary_comment, commentable: proposal, body: "Life is wonderful") + create(:related_content, parent_relationable: proposal, + child_relationable: related_proposal, + machine_learning: true) + + visit proposal_path(proposal) + + within "#tags_proposal_#{proposal.id}" do + expect(page).not_to have_link "user tag" + expect(page).to have_link "machine learning proposal tag" + expect(page).not_to have_link "machine learning investment tag" + end + + within ".related-content" do + expect(page).to have_content "Related content (1)" + expect(page).to have_css ".related-content-title" + expect(page).to have_content related_proposal.title + end + + within "#comments" do + expect(page).to have_content "Comments summary" + expect(page).to have_content "Life is wonderful" + end + end + + scenario "investment view" do + create(:ml_summary_comment, commentable: investment, body: "Build in the main square") + create(:related_content, parent_relationable: investment, + child_relationable: related_investment, + machine_learning: true) + + visit budget_investment_path(investment.budget, investment) + + within "#tags_budget_investment_#{investment.id}" do + expect(page).not_to have_link "user tag" + expect(page).not_to have_link "machine learning proposal tag" + expect(page).to have_link "machine learning investment tag" + end + + within ".related-content" do + expect(page).to have_content "Related content (1)" + expect(page).to have_css ".related-content-title", count: 1 + expect(page).to have_content related_investment.title + end + + within "#tab-comments" do + expect(page).to have_content "Comments summary" + expect(page).to have_content "Build in the main square" + end + end +end