consul/grecia
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add experimental machine learning

Browse Source
This commit is contained in:
Machine Learning
2021-06-18 12:27:29 +07:00
committed by Javi Martín
parent c8d8fae98d
commit 4d27bbebad
84 changed files with 2845 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -37,5 +37,6 @@
public/sitemap.xml public/sitemap.xml
public/assets/ public/assets/
public/machine_learning/data/
public/system/ public/system/
/public/ckeditor_assets/ /public/ckeditor_assets/

15
app/assets/javascripts/admin/machine_learning/scripts.js Normal file
View File

@@ -0,0 +1,15 @@
(function() {
"use strict";
App.AdminMachineLearningScripts = {
initialize: function() {
$(".admin .machine-learning-scripts select").on({
change: function() {
var element = document.getElementById($(this).val());
$("#script_descriptions > *").not(element).addClass("hide");
$(element).removeClass("hide");
}
}).trigger("change");
}
};
}).call(this);

1
app/assets/javascripts/application.js
View File

@@ -168,6 +168,7 @@ var initialize_modules = function() {
App.ColumnsSelector.initialize(); App.ColumnsSelector.initialize();
} }
App.AdminBudgetsWizardCreationStep.initialize(); App.AdminBudgetsWizardCreationStep.initialize();
App.AdminMachineLearningScripts.initialize();
App.BudgetEditAssociations.initialize(); App.BudgetEditAssociations.initialize();
App.Datepicker.initialize(); App.Datepicker.initialize();
App.SDGRelatedListSelector.initialize(); App.SDGRelatedListSelector.initialize();

7
app/assets/stylesheets/admin/machine_learning/help.scss Normal file
View File

@@ -0,0 +1,7 @@
.admin .machine-learning-help {
> code {
display: block;
white-space: pre-wrap;
}
}

60
app/assets/stylesheets/admin/machine_learning/scripts.scss Normal file
View File

@@ -0,0 +1,60 @@
.admin .machine-learning-scripts {
.alert > :first-child {
@include has-fa-icon(ban, solid);
}
.success > :first-child {
@include has-fa-icon(check-circle, solid);
}
.warning > :first-child {
@include has-fa-icon(hourglass-half, solid);
}
.alert,
.success,
.warning {
> :first-child::before {
margin-right: $font-icon-margin;
}
}
dl {
font-size: $base-font-size;
dt,
dd {
display: inline;
}
* + dt::before {
content: "";
display: block;
}
}
form {
max-width: $global-width * 3 / 4;
}
select {
max-width: 100%;
width: auto;
}
[type=submit] {
@include regular-button;
display: block;
margin-bottom: 0;
margin-top: $line-height;
&:focus {
outline: $outline-focus;
}
&.cancel {
@include hollow-button($alert-color);
}
}
}

68
app/assets/stylesheets/admin/machine_learning/setting.scss Normal file
View File

@@ -0,0 +1,68 @@
.admin .machine-learning-setting {
.card-divider {
background: $primary-color;
color: $white;
h3 {
margin-top: 0;
}
}
[aria-pressed] {
@include regular-button;
border-radius: $line-height;
font-weight: bold;
min-width: rem-calc(100);
position: relative;
&:focus {
outline: $outline-focus;
}
&::after {
background: $white;
border-radius: 100%;
content: "";
display: block;
height: 1.75em;
position: absolute;
transform: translateY(-50%);
top: 50%;
width: 1.75em;
}
&[aria-pressed=true] {
background: $primary-color;
padding-right: 2.5em;
text-align: left;
&::after {
right: 0.5em;
}
}
&[aria-pressed=false] {
background: $dark-gray;
padding-left: 2.5em;
text-align: right;
&::after {
left: 0.5em;
}
}
}
dl {
@include callout-size(map-get($callout-sizes, small));
}
dt {
font-weight: normal;
margin-bottom: 0;
}
* + dt {
margin-top: $line-height;
}
}

24
app/assets/stylesheets/admin/machine_learning/settings.scss Normal file
View File

@@ -0,0 +1,24 @@
.admin .machine-learning-settings {
.settings-management {
$gap: rem-calc(map-get($grid-column-gutter, medium));
display: flex;
flex-wrap: wrap;
margin-left: -$gap;
> * {
margin-left: $gap;
flex-basis: calc((#{rem-calc(720)} - 100%) * 999);
flex-grow: 1;
}
.card-section {
display: flex;
flex-direction: column;
> :last-child {
margin-top: auto;
}
}
}
}

6
app/assets/stylesheets/admin/machine_learning/show.scss Normal file
View File

@@ -0,0 +1,6 @@
.admin {
.experimental-feature {
@include has-fa-icon(flask, solid);
}
}

4
app/assets/stylesheets/admin/menu.scss
View File

@@ -108,6 +108,10 @@
&.users-link { &.users-link {
@include icon(user, solid); @include icon(user, solid);
} }
&.ml-link {
@include icon(brain, solid);
}
} }
li { li {

1
app/assets/stylesheets/application.scss
View File

@@ -40,6 +40,7 @@
@import "budgets/**/*"; @import "budgets/**/*";
@import "debates/**/*"; @import "debates/**/*";
@import "layout/**/*"; @import "layout/**/*";
@import "machine_learning/**/*";
@import "proposals/**/*"; @import "proposals/**/*";
@import "relationable/**/*"; @import "relationable/**/*";
@import "sdg/**/*"; @import "sdg/**/*";

7
app/assets/stylesheets/machine_learning/comments_summary.scss Normal file
View File

@@ -0,0 +1,7 @@
.machine-learning-comments-summary {
border-bottom: 1px solid $medium-gray;
+ * {
margin-top: $line-height * 1.5;
}
}

5
app/assets/stylesheets/machine_learning/info.scss Normal file
View File

@@ -0,0 +1,5 @@
.machine-learning-info {
@include has-fa-icon(info-circle, solid);
float: left;
margin-right: $font-icon-margin;
}

25
app/components/admin/machine_learning/help_component.html.erb Normal file
View File

@@ -0,0 +1,25 @@
<div class="tabs-panel machine-learning-help" id="help">
<h3><%= t("admin.machine_learning.help.title_1") %></h3>
<p><%= t("admin.machine_learning.help.description_1") %></p>
<h3><%= t("admin.machine_learning.help.title_2") %></h3>
<p><%= t("admin.machine_learning.help.description_2") %></p>
<h3><%= t("admin.machine_learning.help.title_3") %></h3>
<p><%= t("admin.machine_learning.help.description_3") %></p>
<h3><%= t("admin.machine_learning.help.title_4") %></h3>
<ul>
<li><%= t("admin.machine_learning.help.description_4") %></li>
<li><%= sanitize(t("admin.machine_learning.help.description_4b")) %></li>
<li><%= sanitize(t("admin.machine_learning.help.description_4c")) %></li>
<li><%= sanitize(t("admin.machine_learning.help.description_4d")) %></li>
<li><%= sanitize(t("admin.machine_learning.help.description_4e")) %></li>
<li><%= t("admin.machine_learning.help.description_4f") %></li>
<li><%= sanitize(t("admin.machine_learning.help.description_4g")) %></li>
</ul>
<h3><%= t("admin.machine_learning.help.title_5") %></h3>
<p><%= t("admin.machine_learning.help.description_5") %></p>
<code><%= instructions %></code>
</div>

17
app/components/admin/machine_learning/help_component.rb Normal file
View File

@@ -0,0 +1,17 @@
class Admin::MachineLearning::HelpComponent < ApplicationComponent
private
def instructions
<<~INSTRUCTIONS
sudo apt update
sudo apt install software-properties-common
sudo add-apt-repository ppa:deadsnakes/ppa
sudo apt install python3.7
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python get-pip.py
sudo update-alternatives --install /usr/bin/pip pip /home/deploy/.local/bin/pip3.7 1
pip install pandas
INSTRUCTIONS
end
end

63
app/components/admin/machine_learning/scripts_component.html.erb Normal file
View File

@@ -0,0 +1,63 @@
<div class="tabs-panel is-active machine-learning-scripts" id="scripts">
<% if machine_learning_job.errored? %>
<div class="callout alert">
<p>
<strong><%= t("admin.machine_learning.notice.error") %></strong>
</p>
<dl>
<dt><%= t("admin.machine_learning.executed_by") %></dt>
<dd><%= machine_learning_job.user.name %></dd>
<dt><%= t("admin.machine_learning.script_name") %></dt>
<dd><%= machine_learning_job.script %></dd>
<dt><%= t("admin.machine_learning.error") %></dt>
<dd><%= sanitize(machine_learning_job.error) %></dd>
</dl>
</div>
<% elsif machine_learning_job.finished? %>
<div class="callout success">
<strong><%= t("admin.machine_learning.notice.success") %></strong>
</div>
<% elsif machine_learning_job.started? %>
<div class="callout warning">
<p>
<strong><%= t("admin.machine_learning.notice.working") %></strong>
</p>
<dl>
<dt><%= t("admin.machine_learning.executed_by") %></dt>
<dd><%= machine_learning_job.user.name %></dd>
<dt><%= t("admin.machine_learning.script_name") %></dt>
<dd><%= machine_learning_job.script %></dd>
<dt><%= t("admin.machine_learning.started_at") %></dt>
<dd><%= machine_learning_job.started_at %></dd>
</dl>
</div>
<% end %>
<% if machine_learning_job.running_for_too_long? %>
<%= button_to t("admin.machine_learning.cancel"),
cancel_admin_machine_learning_path,
method: :delete, class: "cancel",
data: { confirm: t("admin.machine_learning.cancel_alert") } %>
<% elsif machine_learning_job.errored? || !machine_learning_job.started? || machine_learning_job.finished? %>
<%= form_tag execute_admin_machine_learning_path, method: :post do %>
<label for="script"><%= t("admin.machine_learning.select_script") %></label>
<%= select_tag "script", options_for_select(script_select_options) %>
<div id="script_descriptions">
<% scripts_info.each_with_index do |script_info, index| %>
<div id="<%= script_info[:name] %>" class="help-text">
<%= sanitize(script_info[:description]) %>
</div>
<% end %>
</div>
<%= submit_tag t("admin.machine_learning.execute_script") %>
<% end %>
<% end %>
</div>

17
app/components/admin/machine_learning/scripts_component.rb Normal file
View File

@@ -0,0 +1,17 @@
class Admin::MachineLearning::ScriptsComponent < ApplicationComponent
attr_reader :machine_learning_job
def initialize(machine_learning_job)
@machine_learning_job = machine_learning_job
end
private
def script_select_options
scripts_info.map { |info| [info[:name], { "aria-describedby": info[:name] }] }
end
def scripts_info
@scripts_info ||= ::MachineLearning.scripts_info
end
end

40
app/components/admin/machine_learning/setting_component.html.erb Normal file
View File

@@ -0,0 +1,40 @@
<div class="card machine-learning-setting" id="<%= dom_id(setting) %>">
<div class="card-divider">
<h3 id="machine_learning_<%= kind %>"><%= t("admin.machine_learning.#{kind}") %></h3>
</div>
<div class="card-section">
<p id="machine_learning_<%= kind %>_description"><%= t("admin.machine_learning.#{kind}_description") %></p>
<% if ml_info.present? %>
<%= form_for(setting, url: admin_setting_path(setting), method: :put) do |f| %>
<%= f.hidden_field :tab, value: "#settings", id: "setting_tab_#{kind}" %>
<%= f.hidden_field :value, value: (setting.enabled? ? "" : "active"), id: "setting_value_#{kind}" %>
<%= f.button(t("shared.#{setting.enabled? ? "yes" : "no"}"),
"aria-labelledby": "machine_learning_#{kind}",
"aria-describedby": "machine_learning_#{kind}_description",
"aria-pressed": setting.enabled?) %>
<% end %>
<dl class="callout success">
<dt><strong><%= t("admin.machine_learning.last_execution") %></strong></dt>
<dd>
<strong><%= render Admin::DateRangeComponent.new(ml_info.generated_at, ml_info.updated_at) %></strong>
</dd>
<dt><%= t("admin.machine_learning.executed_script") %></dt>
<dd><%= ml_info.script %></dd>
<dt><%= t("admin.machine_learning.output_files") %></dt>
<dd>
<% filenames.each do |filename| %>
<a href="<%= data_path(filename) %>" target="_blank"><%= filename %></a><br>
<% end %>
</dd>
</dl>
<% else %>
<div class="callout secondary">
<%= t("admin.machine_learning.no_content") %>
</div>
<% end %>
</div>
</div>

25
app/components/admin/machine_learning/setting_component.rb Normal file
View File

@@ -0,0 +1,25 @@
class Admin::MachineLearning::SettingComponent < ApplicationComponent
attr_reader :kind
def initialize(kind)
@kind = kind
end
private
def setting
@setting ||= Setting.find_by(key: "machine_learning.#{kind}")
end
def ml_info
@ml_info ||= MachineLearningInfo.for(kind)
end
def filenames
::MachineLearning.data_output_files[ml_info.kind.to_sym].sort
end
def data_path(filename)
::MachineLearning.data_path(filename)
end
end

15
app/components/admin/machine_learning/settings_component.html.erb Normal file
View File

@@ -0,0 +1,15 @@
<div class="tabs-panel machine-learning-settings" id="settings">
<div class="settings-management">
<% script_kinds.each do |kind| %>
<%= render Admin::MachineLearning::SettingComponent.new(kind) %>
<% end %>
</div>
<div class="callout secondary">
<p><strong><%= t("admin.machine_learning.data_folder_content") %></strong></p>
<% filenames.each do |filename| %>
<a href="<%= data_path(filename) %>" target="_blank"><%= filename %></a><br>
<% end %>
</div>
</div>

15
app/components/admin/machine_learning/settings_component.rb Normal file
View File

@@ -0,0 +1,15 @@
class Admin::MachineLearning::SettingsComponent < ApplicationComponent
private
def script_kinds
@script_kinds ||= ::MachineLearning.script_kinds
end
def filenames
::MachineLearning.data_intermediate_files
end
def data_path(filename)
::MachineLearning.data_path(filename)
end
end

39
app/components/admin/machine_learning/show_component.html.erb Normal file
View File

@@ -0,0 +1,39 @@
<%= header %>
<% if enabled? %>
<div class="callout primary experimental-feature">
<strong><%= sanitize(t("admin.machine_learning.help_text")) %></strong>
</div>
<ul class="tabs" data-tabs id="machine_learning_tabs" data-deep-link="true">
<li class="tabs-title is-active">
<a href="#scripts">
<%= t("admin.machine_learning.tab_scripts") %>
</a>
</li>
<li class="tabs-title">
<a href="#settings">
<%= t("admin.machine_learning.tab_settings") %>
</a>
</li>
<li class="tabs-title">
<a href="#help">
<%= t("admin.machine_learning.tab_help") %>
</a>
</li>
</ul>
<div class="tabs-content" data-tabs-content="machine_learning_tabs">
<%= render Admin::MachineLearning::ScriptsComponent.new(machine_learning_job) %>
<%= render Admin::MachineLearning::SettingsComponent.new %>
<%= render Admin::MachineLearning::HelpComponent.new %>
</div>
<% else %>
<div class="callout primary">
<p>
<%= sanitize(t("admin.machine_learning.feature_disabled",
link: link_to(t("admin.machine_learning.feature_disabled_link"),
admin_settings_path(anchor: "tab-feature-flags")))) %>
</p>
</div>
<% end %>

18
app/components/admin/machine_learning/show_component.rb Normal file
View File

@@ -0,0 +1,18 @@
class Admin::MachineLearning::ShowComponent < ApplicationComponent
include Header
attr_reader :machine_learning_job
def initialize(machine_learning_job)
@machine_learning_job = machine_learning_job
end
def title
t("admin.machine_learning.title")
end
private
def enabled?
::MachineLearning.enabled?
end
end

5
app/components/admin/menu_component.html.erb
View File

@@ -126,4 +126,9 @@
class: ("is-active" if dashboard?) class: ("is-active" if dashboard?)
) %> ) %>
</li> </li>
<% if ::MachineLearning.enabled? %>
<li class="<%= "is-active" if controller_name == "machine_learning" %>">
<%= link_to t("admin.menu.machine_learning"), admin_machine_learning_path, class: "ml-link" %>
</li>
<% end %>
</ul> </ul>

8
app/components/machine_learning/comments_summary_component.html.erb Normal file
View File

@@ -0,0 +1,8 @@
<div class="machine-learning-comments-summary">
<p>
<%= render MachineLearning::InfoComponent.new %>
<strong><%= t("machine_learning.comments_summary") %></strong>
</p>
<p><%= simple_format(body) %></p>
</div>

17
app/components/machine_learning/comments_summary_component.rb Normal file
View File

@@ -0,0 +1,17 @@
class MachineLearning::CommentsSummaryComponent < ApplicationComponent
attr_reader :commentable
def initialize(commentable)
@commentable = commentable
end
def render?
MachineLearning.enabled? && Setting["machine_learning.comments_summary"].present? && body.present?
end
private
def body
commentable.summary_comment&.body
end
end

3
app/components/machine_learning/info_component.html.erb Normal file
View File

@@ -0,0 +1,3 @@
<span class="machine-learning-info top" data-tooltip tabindex="0" title="<%= t("machine_learning.info_text") %>">
<span class="show-for-sr"><%= t("machine_learning.info_text") %></span>
</span>

3
app/components/machine_learning/info_component.rb Normal file
View File

@@ -0,0 +1,3 @@
class MachineLearning::InfoComponent < ApplicationComponent
delegate :current_user, to: :helpers
end

12
app/components/shared/comments_component.html.erb
View File

@@ -1,8 +1,8 @@
<% cache cache_key do %> <div class="row comments">
<div class="row comments"> <div id="comments" class="small-12 column">
<div id="comments" class="small-12 column"> <%= content %>
<%= content %>
<% cache cache_key do %>
<% if current_user %> <% if current_user %>
<%= render "comments/form", { commentable: record, parent_id: nil } %> <%= render "comments/form", { commentable: record, parent_id: nil } %>
<% else %> <% else %>
@@ -12,6 +12,6 @@
<%= render Shared::OrderLinksComponent.new("comments", anchor: "comments") %> <%= render Shared::OrderLinksComponent.new("comments", anchor: "comments") %>
<%= render "comments/comment_list", comments: comment_tree.root_comments %> <%= render "comments/comment_list", comments: comment_tree.root_comments %>
<%= paginate comment_tree.root_comments, params: { anchor: "comments" } %> <%= paginate comment_tree.root_comments, params: { anchor: "comments" } %>
</div> <% end %>
</div> </div>
<% end %> </div>

4
app/components/shared/tag_list_component.html.erb
View File

@@ -1 +1,5 @@
<% if machine_learning? %>
<%= render MachineLearning::InfoComponent.new %>
<% end %>
<%= link_list(*links, class: "tags", id: "tags_#{dom_id(taggable)}") %> <%= link_list(*links, class: "tags", id: "tags_#{dom_id(taggable)}") %>

10
app/components/shared/tag_list_component.rb
View File

@@ -7,6 +7,10 @@ class Shared::TagListComponent < ApplicationComponent
@limit = limit @limit = limit
end end
def render?
taggable.tags_list.any?
end
private private
def links def links
@@ -23,7 +27,7 @@ class Shared::TagListComponent < ApplicationComponent
end end
def see_more_link def see_more_link
render Shared::SeeMoreLinkComponent.new(taggable, :tags, limit: limit) render Shared::SeeMoreLinkComponent.new(taggable, :tags_list, limit: limit)
end end
def taggables_path(taggable, tag_name) def taggables_path(taggable, tag_name)
@@ -34,4 +38,8 @@ class Shared::TagListComponent < ApplicationComponent
polymorphic_path(taggable.class, search: tag_name) polymorphic_path(taggable.class, search: tag_name)
end end
end end
def machine_learning?
Tag.machine_learning?
end
end end

33
app/controllers/admin/machine_learning_controller.rb Normal file
View File

@@ -0,0 +1,33 @@
class Admin::MachineLearningController < Admin::BaseController
before_action :load_machine_learning_job, only: [:show, :execute]
def show
end
def execute
@machine_learning_job.update!(script: params[:script],
user: current_user,
started_at: Time.current,
finished_at: nil,
error: nil)
::MachineLearning.new(@machine_learning_job).run
redirect_to admin_machine_learning_path,
notice: t("admin.machine_learning.script_info", email: current_user.email)
end
def cancel
Delayed::Job.where(queue: "machine_learning").destroy_all
MachineLearningJob.destroy_all
redirect_to admin_machine_learning_path,
notice: t("admin.machine_learning.notice.delete_generated_content")
end
private
def load_machine_learning_job
@machine_learning_job = MachineLearningJob.first_or_initialize
end
end

2
app/helpers/site_customization_helper.rb
View File

@@ -19,6 +19,6 @@ module SiteCustomizationHelper
end end
def information_texts_tabs def information_texts_tabs
[:basic, :debates, :community, :proposals, :polls, :layouts, :mailers, :management, :welcome] [:basic, :debates, :community, :proposals, :polls, :layouts, :mailers, :management, :welcome, :machine_learning]
end end
end end

12
app/mailers/mailer.rb
View File

@@ -127,6 +127,18 @@ class Mailer < ApplicationMailer
mail(to: @email_to.email, subject: @email.subject) if @email.can_be_sent? mail(to: @email_to.email, subject: @email.subject) if @email.can_be_sent?
end end
def machine_learning_error(user)
@email_to = user.email
mail(to: @email_to, subject: t("mailers.machine_learning_error.subject"))
end
def machine_learning_success(user)
@email_to = user.email
mail(to: @email_to, subject: t("mailers.machine_learning_success.subject"))
end
private private
def with_user(user, &block) def with_user(user, &block)

1
app/models/budget/investment.rb
View File

@@ -50,6 +50,7 @@ class Budget
has_many :valuator_groups, through: :valuator_group_assignments has_many :valuator_groups, through: :valuator_group_assignments
has_many :comments, -> { where(valuation: false) }, as: :commentable, inverse_of: :commentable has_many :comments, -> { where(valuation: false) }, as: :commentable, inverse_of: :commentable
has_one :summary_comment, as: :commentable, class_name: "MlSummaryComment", dependent: :destroy
has_many :valuations, -> { where(valuation: true) }, has_many :valuations, -> { where(valuation: true) },
as: :commentable, as: :commentable,
inverse_of: :commentable, inverse_of: :commentable,

13
app/models/budget/investment/exporter.rb
View File

@@ -1,5 +1,6 @@
class Budget::Investment::Exporter class Budget::Investment::Exporter
require "csv" require "csv"
include JsonExporter
def initialize(investments) def initialize(investments)
@investments = investments @investments = investments
@@ -12,6 +13,10 @@ class Budget::Investment::Exporter
end end
end end
def model
Budget::Investment
end
private private
def headers def headers
@@ -64,4 +69,12 @@ class Budget::Investment::Exporter
I18n.t(price_string) I18n.t(price_string)
end end
end end
def json_values(investment)
{
id: investment.id,
title: investment.title,
description: strip_tags(investment.description)
}
end
end end

18
app/models/comment/exporter.rb Normal file
View File

@@ -0,0 +1,18 @@
class Comment::Exporter
include JsonExporter
def model
Comment
end
private
def json_values(comment)
{
id: comment.id,
commentable_id: comment.commentable_id,
commentable_type: comment.commentable_type,
body: strip_tags(comment.body)
}
end
end

21
app/models/concerns/json_exporter.rb Normal file
View File

@@ -0,0 +1,21 @@
module JsonExporter
def to_json_file(filename)
data = []
model.find_each { |record| data << json_values(record) }
File.open(filename, "w") { |file| file.write(data.to_json) }
end
private
def strip_tags(html_string)
ActionView::Base.full_sanitizer.sanitize(html_string)
end
def model
raise "This method must be implemented in class #{self.class.name}"
end
def json_values(record)
raise "This method must be implemented in class #{self.class.name}"
end
end

11
app/models/concerns/relationable.rb
View File

@@ -13,7 +13,14 @@ module Relationable
end end
def relationed_contents def relationed_contents
related_contents.not_hidden.map(&:child_relationable) if MachineLearning.enabled? && Setting["machine_learning.related_content"].present?
.reject { |related| related.respond_to?(:retired?) && related.retired? } related_content = related_contents.not_hidden.order(machine_learning_score: :desc)
else
related_content = related_contents.not_hidden.from_users
end
related_content.map(&:child_relationable).reject do |related|
related.respond_to?(:retired?) && related.retired?
end
end end
end end

16
app/models/concerns/taggable.rb
View File

@@ -2,14 +2,22 @@ module Taggable
extend ActiveSupport::Concern extend ActiveSupport::Concern
included do included do
acts_as_taggable acts_as_taggable_on :tags, :ml_tags
validate :max_number_of_tags, on: :create validate :max_number_of_tags, on: :create
end end
def tag_list_with_limit(limit = nil) def tags_list
return tags if limit.blank? if Tag.machine_learning? && (is_a?(Proposal) || is_a?(Budget::Investment))
ml_tags
else
tags
end
end
tags.sort { |a, b| b.taggings_count <=> a.taggings_count }[0, limit] def tag_list_with_limit(limit = nil)
return tags_list if limit.blank?
tags_list.sort { |a, b| b.taggings_count <=> a.taggings_count }[0, limit]
end end
def max_number_of_tags def max_number_of_tags

16
app/models/i18n_content.rb
View File

@@ -52,6 +52,8 @@ class I18nContent < ApplicationRecord
def self.translations_for(tab) def self.translations_for(tab)
if tab.to_s == "basic" if tab.to_s == "basic"
basic_translations basic_translations
elsif tab.to_s == "machine_learning"
machine_learning_translations
else else
flat_hash(translations_hash_for(tab)).keys flat_hash(translations_hash_for(tab)).keys
end end
@@ -95,6 +97,20 @@ class I18nContent < ApplicationRecord
] ]
end end
def self.machine_learning_translations
%w[
admin.machine_learning.title
machine_learning.comments_summary
machine_learning.info_text
admin.machine_learning.comments_summary
admin.machine_learning.comments_summary_description
admin.machine_learning.related_content
admin.machine_learning.related_content_description
admin.machine_learning.tags
admin.machine_learning.tags_description
]
end
def self.translations_hash(locale) def self.translations_hash(locale)
Rails.cache.fetch(translation_class.where(locale: locale)) do Rails.cache.fetch(translation_class.where(locale: locale)) do
all.map do |content| all.map do |content|

436
app/models/machine_learning.rb Normal file
View File

@@ -0,0 +1,436 @@
class MachineLearning
attr_reader :user, :script, :previous_modified_date
attr_accessor :job
SCRIPTS_FOLDER = Rails.root.join("public", "machine_learning", "scripts").freeze
DATA_FOLDER = Rails.root.join("public", "machine_learning", "data").freeze
def initialize(job)
@job = job
@user = job.user
@previous_modified_date = set_previous_modified_date
end
def run
begin
export_proposals_to_json
export_budget_investments_to_json
export_comments_to_json
return unless run_machine_learning_scripts
if updated_file?(MachineLearning.proposals_taggings_filename) && updated_file?(MachineLearning.proposals_tags_filename)
cleanup_proposals_tags!
import_ml_proposals_tags
update_machine_learning_info_for("tags")
end
if updated_file?(MachineLearning.investments_taggings_filename) && updated_file?(MachineLearning.investments_tags_filename)
cleanup_investments_tags!
import_ml_investments_tags
update_machine_learning_info_for("tags")
end
if updated_file?(MachineLearning.proposals_related_filename)
cleanup_proposals_related_content!
import_proposals_related_content
update_machine_learning_info_for("related_content")
end
if updated_file?(MachineLearning.investments_related_filename)
cleanup_investments_related_content!
import_budget_investments_related_content
update_machine_learning_info_for("related_content")
end
if updated_file?(MachineLearning.proposals_comments_summary_filename)
cleanup_proposals_comments_summary!
import_ml_proposals_comments_summary
update_machine_learning_info_for("comments_summary")
end
if updated_file?(MachineLearning.investments_comments_summary_filename)
cleanup_investments_comments_summary!
import_ml_investments_comments_summary
update_machine_learning_info_for("comments_summary")
end
job.update!(finished_at: Time.current)
Mailer.machine_learning_success(user).deliver_later
rescue Exception => error
handle_error(error)
raise error
end
end
handle_asynchronously :run, queue: "machine_learning"
class << self
def enabled?
Setting["feature.machine_learning"].present?
end
def proposals_filename
"proposals.json"
end
def investments_filename
"budget_investments.json"
end
def comments_filename
"comments.json"
end
def data_output_files
files = { tags: [], related_content: [], comments_summary: [] }
files[:tags] << proposals_tags_filename if File.exists?(DATA_FOLDER.join(proposals_tags_filename))
files[:tags] << proposals_taggings_filename if File.exists?(DATA_FOLDER.join(proposals_taggings_filename))
files[:tags] << investments_tags_filename if File.exists?(DATA_FOLDER.join(investments_tags_filename))
files[:tags] << investments_taggings_filename if File.exists?(DATA_FOLDER.join(investments_taggings_filename))
files[:related_content] << proposals_related_filename if File.exists?(DATA_FOLDER.join(proposals_related_filename))
files[:related_content] << investments_related_filename if File.exists?(DATA_FOLDER.join(investments_related_filename))
files[:comments_summary] << proposals_comments_summary_filename if File.exists?(DATA_FOLDER.join(proposals_comments_summary_filename))
files[:comments_summary] << investments_comments_summary_filename if File.exists?(DATA_FOLDER.join(investments_comments_summary_filename))
files
end
def data_intermediate_files
excluded = [
proposals_filename,
investments_filename,
comments_filename,
proposals_tags_filename,
proposals_taggings_filename,
investments_tags_filename,
investments_taggings_filename,
proposals_related_filename,
investments_related_filename,
proposals_comments_summary_filename,
investments_comments_summary_filename
]
json = Dir[DATA_FOLDER.join("*.json")].map do |full_path_filename|
full_path_filename.split("/").last
end
csv = Dir[DATA_FOLDER.join("*.csv")].map do |full_path_filename|
full_path_filename.split("/").last
end
(json + csv - excluded).sort
end
def proposals_tags_filename
"ml_tags_proposals.json"
end
def proposals_taggings_filename
"ml_taggings_proposals.json"
end
def investments_tags_filename
"ml_tags_budgets.json"
end
def investments_taggings_filename
"ml_taggings_budgets.json"
end
def proposals_related_filename
"ml_related_content_proposals.json"
end
def investments_related_filename
"ml_related_content_budgets.json"
end
def proposals_comments_summary_filename
"ml_comments_summaries_proposals.json"
end
def investments_comments_summary_filename
"ml_comments_summaries_budgets.json"
end
def data_path(filename)
"/machine_learning/data/" + filename
end
def script_kinds
%w[tags related_content comments_summary]
end
def scripts_info
scripts_info = []
Dir[SCRIPTS_FOLDER.join("*.py")].each do |full_path_filename|
scripts_info << {
name: full_path_filename.split("/").last,
description: description_from(full_path_filename)
}
end
scripts_info.sort_by { |script_info| script_info[:name] }
end
def description_from(script_filename)
description = ""
delimiter = '"""'
break_line = "<br>"
comment_found = false
File.readlines(script_filename).each do |line|
if line.start_with?(delimiter) && !comment_found
comment_found = true
line.slice!(delimiter)
description << line.strip.concat(break_line) if line.present?
elsif line.include?(delimiter)
line.slice!(delimiter)
description << line.strip if line.present?
break
elsif comment_found
description << line.strip.concat(break_line)
end
end
description.delete_suffix(break_line)
end
end
private
def export_proposals_to_json
filename = DATA_FOLDER.join(MachineLearning.proposals_filename)
Proposal::Exporter.new.to_json_file(filename)
end
def export_budget_investments_to_json
filename = DATA_FOLDER.join(MachineLearning.investments_filename)
Budget::Investment::Exporter.new(Array.new).to_json_file(filename)
end
def export_comments_to_json
filename = DATA_FOLDER.join(MachineLearning.comments_filename)
Comment::Exporter.new.to_json_file(filename)
end
def run_machine_learning_scripts
output = `cd #{SCRIPTS_FOLDER} && python #{job.script} 2>&1`
result = $?.success?
if result == false
job.update!(finished_at: Time.current, error: output)
Mailer.machine_learning_error(user).deliver_later
end
result
end
def cleanup_proposals_tags!
Tagging.where(context: "ml_tags", taggable_type: "Proposal").find_each(&:destroy!)
Tag.find_each { |tag| tag.destroy! if Tagging.where(tag: tag).empty? }
end
def cleanup_investments_tags!
Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment").find_each(&:destroy!)
Tag.find_each { |tag| tag.destroy! if Tagging.where(tag: tag).empty? }
end
def cleanup_proposals_related_content!
RelatedContent.with_hidden.for_proposals.from_machine_learning.find_each(&:really_destroy!)
end
def cleanup_investments_related_content!
RelatedContent.with_hidden.for_investments.from_machine_learning.find_each(&:really_destroy!)
end
def cleanup_proposals_comments_summary!
MlSummaryComment.where(commentable_type: "Proposal").find_each(&:destroy!)
end
def cleanup_investments_comments_summary!
MlSummaryComment.where(commentable_type: "Budget::Investment").find_each(&:destroy!)
end
def import_ml_proposals_comments_summary
json_file = DATA_FOLDER.join(MachineLearning.proposals_comments_summary_filename)
json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!)
json_data.each do |attributes|
attributes.delete(:id)
unless MlSummaryComment.find_by(commentable_id: attributes[:commentable_id],
commentable_type: "Proposal")
MlSummaryComment.create!(attributes)
end
end
end
def import_ml_investments_comments_summary
json_file = DATA_FOLDER.join(MachineLearning.investments_comments_summary_filename)
json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!)
json_data.each do |attributes|
attributes.delete(:id)
unless MlSummaryComment.find_by(commentable_id: attributes[:commentable_id],
commentable_type: "Budget::Investment")
MlSummaryComment.create!(attributes)
end
end
end
def import_proposals_related_content
json_file = DATA_FOLDER.join(MachineLearning.proposals_related_filename)
json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!)
json_data.each do |related|
id = related.delete(:id)
score = related.size
related.each do |_, related_id|
if related_id.present?
attributes = {
parent_relationable_id: id,
parent_relationable_type: "Proposal",
child_relationable_id: related_id,
child_relationable_type: "Proposal"
}
related_content = RelatedContent.find_by(attributes)
if related_content.present?
related_content.update!(machine_learning_score: score)
else
RelatedContent.create!(attributes.merge(machine_learning: true,
author: user,
machine_learning_score: score))
end
end
score -= 1
end
end
end
def import_budget_investments_related_content
json_file = DATA_FOLDER.join(MachineLearning.investments_related_filename)
json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!)
json_data.each do |related|
id = related.delete(:id)
score = related.size
related.each do |_, related_id|
if related_id.present?
attributes = {
parent_relationable_id: id,
parent_relationable_type: "Budget::Investment",
child_relationable_id: related_id,
child_relationable_type: "Budget::Investment"
}
related_content = RelatedContent.find_by(attributes)
if related_content.present?
related_content.update!(machine_learning_score: score)
else
RelatedContent.create!(attributes.merge(machine_learning: true,
author: user,
machine_learning_score: score))
end
end
score -= 1
end
end
end
def import_ml_proposals_tags
ids = {}
json_file = DATA_FOLDER.join(MachineLearning.proposals_tags_filename)
json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!)
json_data.each do |attributes|
if attributes[:name].present?
attributes.delete(:taggings_count)
if attributes[:name].length >= 150
attributes[:name] = attributes[:name].truncate(150)
end
tag = Tag.find_or_create_by!(name: attributes[:name])
ids[attributes[:id]] = tag.id
end
end
json_file = DATA_FOLDER.join(MachineLearning.proposals_taggings_filename)
json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!)
json_data.each do |attributes|
if attributes[:tag_id].present?
tag_id = ids[attributes[:tag_id]]
if Tag.find_by(id: tag_id) && attributes[:taggable_id].present?
attributes[:tag_id] = tag_id
attributes[:taggable_type] = "Proposal"
attributes[:context] = "ml_tags"
Tagging.create!(attributes)
end
end
end
end
def import_ml_investments_tags
ids = {}
json_file = DATA_FOLDER.join(MachineLearning.investments_tags_filename)
json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!)
json_data.each do |attributes|
if attributes[:name].present?
attributes.delete(:taggings_count)
if attributes[:name].length >= 150
attributes[:name] = attributes[:name].truncate(150)
end
tag = Tag.find_or_create_by!(name: attributes[:name])
ids[attributes[:id]] = tag.id
end
end
json_file = DATA_FOLDER.join(MachineLearning.investments_taggings_filename)
json_data = JSON.parse(File.read(json_file)).each(&:deep_symbolize_keys!)
json_data.each do |attributes|
if attributes[:tag_id].present?
tag_id = ids[attributes[:tag_id]]
if Tag.find_by(id: tag_id) && attributes[:taggable_id].present?
attributes[:tag_id] = tag_id
attributes[:taggable_type] = "Budget::Investment"
attributes[:context] = "ml_tags"
Tagging.create!(attributes)
end
end
end
end
def update_machine_learning_info_for(kind)
MachineLearningInfo.find_or_create_by!(kind: kind)
.update!(generated_at: job.started_at, script: job.script)
end
def set_previous_modified_date
proposals_tags_filename = MachineLearning.proposals_tags_filename
proposals_taggings_filename = MachineLearning.proposals_taggings_filename
investments_tags_filename = MachineLearning.investments_tags_filename
investments_taggings_filename = MachineLearning.investments_taggings_filename
proposals_related_filename = MachineLearning.proposals_related_filename
investments_related_filename = MachineLearning.investments_related_filename
proposals_comments_summary_filename = MachineLearning.proposals_comments_summary_filename
investments_comments_summary_filename = MachineLearning.investments_comments_summary_filename
{
proposals_tags_filename => last_modified_date_for(proposals_tags_filename),
proposals_taggings_filename => last_modified_date_for(proposals_taggings_filename),
investments_tags_filename => last_modified_date_for(investments_tags_filename),
investments_taggings_filename => last_modified_date_for(investments_taggings_filename),
proposals_related_filename => last_modified_date_for(proposals_related_filename),
investments_related_filename => last_modified_date_for(investments_related_filename),
proposals_comments_summary_filename => last_modified_date_for(proposals_comments_summary_filename),
investments_comments_summary_filename => last_modified_date_for(investments_comments_summary_filename)
}
end
def last_modified_date_for(filename)
return nil unless File.exists? DATA_FOLDER.join(filename)
File.mtime DATA_FOLDER.join(filename)
end
def updated_file?(filename)
return false unless File.exists? DATA_FOLDER.join(filename)
return true unless previous_modified_date[filename].present?
last_modified_date_for(filename) > previous_modified_date[filename]
end
def handle_error(error)
message = error.message
backtrace = error.backtrace.select { |line| line.include?("machine_learning.rb") }
full_error = ([message] + backtrace).join("<br>")
job.update!(finished_at: Time.current, error: full_error)
Mailer.machine_learning_error(user).deliver_later
end
end

5
app/models/machine_learning_info.rb Normal file
View File

@@ -0,0 +1,5 @@
class MachineLearningInfo < ApplicationRecord
def self.for(kind)
find_by(kind: kind)
end
end

19
app/models/machine_learning_job.rb Normal file
View File

@@ -0,0 +1,19 @@
class MachineLearningJob < ApplicationRecord
belongs_to :user, optional: false
def started?
started_at.present?
end
def finished?
finished_at.present?
end
def errored?
error.present?
end
def running_for_too_long?
started? && !finished? && started_at < 1.day.ago
end
end

3
app/models/ml_summary_comment.rb Normal file
View File

@@ -0,0 +1,3 @@
class MlSummaryComment < ApplicationRecord
belongs_to :commentable, -> { with_hidden }, polymorphic: true, touch: true
end

1
app/models/proposal.rb
View File

@@ -41,6 +41,7 @@ class Proposal < ApplicationRecord
has_many :dashboard_executed_actions, dependent: :destroy, class_name: "Dashboard::ExecutedAction" has_many :dashboard_executed_actions, dependent: :destroy, class_name: "Dashboard::ExecutedAction"
has_many :dashboard_actions, through: :dashboard_executed_actions, class_name: "Dashboard::Action" has_many :dashboard_actions, through: :dashboard_executed_actions, class_name: "Dashboard::Action"
has_many :polls, as: :related, inverse_of: :related has_many :polls, as: :related, inverse_of: :related
has_one :summary_comment, as: :commentable, class_name: "MlSummaryComment", dependent: :destroy
validates_translation :title, presence: true, length: { in: 4..Proposal.title_max_length } validates_translation :title, presence: true, length: { in: 4..Proposal.title_max_length }
validates_translation :description, length: { maximum: Proposal.description_max_length } validates_translation :description, length: { maximum: Proposal.description_max_length }

18
app/models/proposal/exporter.rb Normal file
View File

@@ -0,0 +1,18 @@
class Proposal::Exporter
include JsonExporter
def model
Proposal
end
private
def json_values(proposal)
{
id: proposal.id,
title: proposal.title,
summary: strip_tags(proposal.summary),
description: strip_tags(proposal.description)
}
end
end

17
app/models/related_content.rb
View File

@@ -9,7 +9,7 @@ class RelatedContent < ApplicationRecord
belongs_to :parent_relationable, polymorphic: true, optional: false, touch: true belongs_to :parent_relationable, polymorphic: true, optional: false, touch: true
belongs_to :child_relationable, polymorphic: true, optional: false, touch: true belongs_to :child_relationable, polymorphic: true, optional: false, touch: true
has_one :opposite_related_content, class_name: self.name, foreign_key: :related_content_id has_one :opposite_related_content, class_name: self.name, foreign_key: :related_content_id
has_many :related_content_scores has_many :related_content_scores, dependent: :destroy
validates :parent_relationable_id, uniqueness: { scope: [:parent_relationable_type, :child_relationable_id, :child_relationable_type] } validates :parent_relationable_id, uniqueness: { scope: [:parent_relationable_type, :child_relationable_id, :child_relationable_type] }
validate :different_parent_and_child validate :different_parent_and_child
@@ -18,6 +18,14 @@ class RelatedContent < ApplicationRecord
after_create :create_author_score after_create :create_author_score
scope :not_hidden, -> { where(hidden_at: nil) } scope :not_hidden, -> { where(hidden_at: nil) }
scope :from_users, -> { where(machine_learning: false) }
scope :from_machine_learning, -> { where(machine_learning: true) }
scope :for_proposals, -> do
where(parent_relationable_type: "Proposal", child_relationable_type: "Proposal")
end
scope :for_investments, -> do
where(parent_relationable_type: "Budget::Investment", child_relationable_type: "Budget::Investment")
end
def score_positive(user) def score_positive(user)
score(RelatedContentScore::SCORES[:POSITIVE], user) score(RelatedContentScore::SCORES[:POSITIVE], user)
@@ -48,8 +56,11 @@ class RelatedContent < ApplicationRecord
end end
def create_opposite_related_content def create_opposite_related_content
related_content = RelatedContent.create!(opposite_related_content: self, parent_relationable: child_relationable, related_content = RelatedContent.create!(opposite_related_content: self,
child_relationable: parent_relationable, author: author) parent_relationable: child_relationable,
child_relationable: parent_relationable,
machine_learning: machine_learning,
author: author)
self.opposite_related_content = related_content self.opposite_related_content = related_content
end end

4
app/models/setting.rb
View File

@@ -101,6 +101,7 @@ class Setting < ApplicationRecord
"feature.valuation_comment_notification": true, "feature.valuation_comment_notification": true,
"feature.graphql_api": true, "feature.graphql_api": true,
"feature.sdg": false, "feature.sdg": false,
"feature.machine_learning": false,
"homepage.widgets.feeds.debates": true, "homepage.widgets.feeds.debates": true,
"homepage.widgets.feeds.processes": true, "homepage.widgets.feeds.processes": true,
"homepage.widgets.feeds.proposals": true, "homepage.widgets.feeds.proposals": true,
@@ -172,6 +173,9 @@ class Setting < ApplicationRecord
"related_content_score_threshold": -0.3, "related_content_score_threshold": -0.3,
"featured_proposals_number": 3, "featured_proposals_number": 3,
"feature.dashboard.notification_emails": nil, "feature.dashboard.notification_emails": nil,
"machine_learning.comments_summary": false,
"machine_learning.related_content": false,
"machine_learning.tags": false,
"remote_census.general.endpoint": "", "remote_census.general.endpoint": "",
"remote_census.request.method_name": "", "remote_census.request.method_name": "",
"remote_census.request.structure": "", "remote_census.request.structure": "",

3
app/models/tag.rb
View File

@@ -1,2 +1,5 @@
class Tag < ActsAsTaggableOn::Tag class Tag < ActsAsTaggableOn::Tag
def self.machine_learning?
MachineLearning.enabled? && Setting["machine_learning.tags"].present?
end
end end

10
app/models/tag_cloud.rb
View File

@@ -8,12 +8,20 @@ class TagCloud
def tags def tags
resource_model_scoped. resource_model_scoped.
last_week.tag_counts. last_week.send(counts).
where("lower(name) NOT IN (?)", category_names + geozone_names + default_blacklist). where("lower(name) NOT IN (?)", category_names + geozone_names + default_blacklist).
order("#{table_name}_count": :desc, name: :asc). order("#{table_name}_count": :desc, name: :asc).
limit(10) limit(10)
end end
def counts
if Tag.machine_learning? && [Proposal, Budget::Investment].include?(resource_model)
:ml_tag_counts
else
:tag_counts
end
end
def category_names def category_names
Tag.category_names.map(&:downcase) Tag.category_names.map(&:downcase)
end end

1
app/models/tagging.rb
View File

@@ -1,2 +1,3 @@
class Tagging < ActsAsTaggableOn::Tagging class Tagging < ActsAsTaggableOn::Tagging
belongs_to :taggable, polymorphic: true, touch: true
end end

1
app/views/admin/machine_learning/show.html.erb Normal file
View File

@@ -0,0 +1 @@
<%= render Admin::MachineLearning::ShowComponent.new(@machine_learning_job) %>

2
app/views/budgets/investments/show.html.erb
View File

@@ -16,6 +16,8 @@
<div class="tabs-content" data-tabs-content="investments_tabs"> <div class="tabs-content" data-tabs-content="investments_tabs">
<div class="tabs-panel is-active" id="tab-comments"> <div class="tabs-panel is-active" id="tab-comments">
<%= render MachineLearning::CommentsSummaryComponent.new(@investment) %>
<%= render "/comments/comment_tree", comment_tree: @comment_tree, <%= render "/comments/comment_tree", comment_tree: @comment_tree,
display_comments_count: false %> display_comments_count: false %>
</div> </div>

17
app/views/mailer/machine_learning_error.html.erb Normal file
View File

@@ -0,0 +1,17 @@
<td style="padding-bottom: 20px; padding-left: 10px;">
<h1 style="font-family: 'Open Sans','Helvetica Neue',sans-serif;">
<%= t("mailers.machine_learning_error.title") %>
</h1>
<p style="font-family: 'Open Sans','Helvetica Neue',arial,sans-serif;font-size: 14px;
font-weight: normal;line-height: 24px;">
<%= t("mailers.machine_learning_error.text") %>
</p>
<p style="font-family: 'Open Sans','Helvetica Neue',arial,sans-serif;font-size: 14px;
font-weight: normal;line-height: 24px;">
<%= link_to t("mailers.machine_learning_error.link"), admin_machine_learning_url,
style: "color: #2895F1; text-decoration:none;" %>
</p>
</td>

17
app/views/mailer/machine_learning_success.html.erb Normal file
View File

@@ -0,0 +1,17 @@
<td style="padding-bottom: 20px; padding-left: 10px;">
<h1 style="font-family: 'Open Sans','Helvetica Neue',sans-serif;">
<%= t("mailers.machine_learning_success.title") %>
</h1>
<p style="font-family: 'Open Sans','Helvetica Neue',arial,sans-serif;font-size: 14px;
font-weight: normal;line-height: 24px;">
<%= t("mailers.machine_learning_success.text") %>
</p>
<p style="font-family: 'Open Sans','Helvetica Neue',arial,sans-serif;font-size: 14px;
font-weight: normal;line-height: 24px;">
<%= link_to t("mailers.machine_learning_success.link"), admin_machine_learning_url,
style: "color: #2895F1; text-decoration:none;" %>
</p>
</td>

4
app/views/proposals/_comments.html.erb
View File

@@ -1 +1,3 @@
<%= render Shared::CommentsComponent.new(@proposal, @comment_tree) %> <%= render Shared::CommentsComponent.new(@proposal, @comment_tree) do %>
<%= render MachineLearning::CommentsSummaryComponent.new(@proposal) %>
<% end %>

2
config/deploy.rb
View File

@@ -22,7 +22,7 @@ set :pty, true
set :use_sudo, false set :use_sudo, false
set :linked_files, %w[config/database.yml config/secrets.yml] set :linked_files, %w[config/database.yml config/secrets.yml]
set :linked_dirs, %w[.bundle log tmp public/system public/assets public/ckeditor_assets] set :linked_dirs, %w[.bundle log tmp public/system public/assets public/ckeditor_assets public/machine_learning/data]
set :keep_releases, 5 set :keep_releases, 5

2
config/i18n-tasks.yml
View File

@@ -222,7 +222,7 @@ ignore_unused:
- "sdg.goals.goal_*" - "sdg.goals.goal_*"
- "sdg.*.filter.more.*" - "sdg.*.filter.more.*"
- "sdg_management.relations.index.filter*" - "sdg_management.relations.index.filter*"
- "tags.filter.more.*" - "tags.list.filter.more.*"
#### ####
## Exclude these keys from the `i18n-tasks eq-base" report: ## Exclude these keys from the `i18n-tasks eq-base" report:
# ignore_eq_base: # ignore_eq_base:

52
config/locales/en/admin.yml
View File

@@ -751,6 +751,7 @@ en:
proposals: "Proposals" proposals: "Proposals"
polls: "Polls" polls: "Polls"
layouts: "Layouts" layouts: "Layouts"
machine_learning: "AI / Machine Learning"
mailers: "Emails" mailers: "Emails"
management: "Management" management: "Management"
welcome: "Welcome" welcome: "Welcome"
@@ -771,6 +772,7 @@ en:
debates: "Debates" debates: "Debates"
comments: "Comments" comments: "Comments"
local_census_records: Manage local census local_census_records: Manage local census
machine_learning: "AI / Machine learning"
administrators: administrators:
index: index:
title: Administrators title: Administrators
@@ -1683,3 +1685,53 @@ en:
created: Created records created: Created records
local_census_records: local_census_records:
no_records_found: No records found. no_records_found: No records found.
machine_learning:
cancel: "Cancel operation"
cancel_alert: "This action will cancel the current script and it will be necessary to run the script again."
comments_summary: "Comments summary"
comments_summary_description: "Displays an automatically generated comment summary on all items that can be commented on."
data_folder_content: "Data folder content"
error: "Error:"
execute_script: "Execute script"
executed_by: "Executed by:"
executed_script: "Executed script:"
feature_disabled: "This feature is disabled. To use Machine Learning you can enable it from the %{link}."
feature_disabled_link: "settings page"
help:
title_1: "What is AI / Machine Learning?"
description_1: "Traditionally, computer programs are designed by establishing precise rules about what the program should do in each situation, and how to process the available information at each moment. What is commonly known as AI/Machine Learning is a type of programs for which what is precisely established is what the task to be performed is and how it is assessed whether the task is better or worse performed. But unlike the former, the system discovers or learns what would be the most suitable method to perform the task."
title_2: "What does the AI / Machine Learning module in CONSUL?"
description_2: "This module allows any type of AI/Machine Learning programs to be implemented in CONSUL. The programs can process the information available in CONSUL and produce results that help both users and administrators to carry out more effective and intelligent citizen participation. The module has been developed to make it easy to implement and run new programs, keeping the general code of CONSUL and the code of these new programs independent from each other."
title_3: "How to use this module"
description_3: "To use it for the first time it is necessary to follow the instructions available in the CONSUL documentation to properly activate this module. Once the module is correctly activated, the programs are executed from the \"Execute scripts\" tab. Before executing it, we can see in the same tab an estimation of the execution time and other relevant information. At the end of the execution, we can activate the content that will be shown in CONSUL from the \"Settings\" tab, as well as download other generated files that may be useful to us."
title_4: "How to implement new AI / Machine Learning scripts"
description_4: "Please use previous scripts as examples."
description_4b: "The new scripts should be located in the folder <code>public/machine_learning/scripts</code>"
description_4c: "The input data to be used by the script will be created as JSON files in the folder <code>../data</code>"
description_4d: "Any generated output should be placed in the folder <code>../data</code>"
description_4e: "It is recommended to create an independent <code>.ini</code> text file with the settings of the script, with the same name of the script and readable by configparser. This file will facilitate settings change by the administrators."
description_4f: "It is recommended to include as the first line of the script an initial triple-quote string with some brief information about it and links to any relevant information. This string will be automatically shown in the Administration interface."
description_4g: "It is recommended to log the relevant information of the script in a <code>.log</code> text file, with the same name of the script."
title_5: "To use AI / Machine Learning scripts you need to have Python installed on your server"
description_5: "Here you can see the example instructions for an Ubuntu 18.04 server:"
help_text: "This functionality is experimental."
last_execution: "Last execution"
no_content: "No content generated yet."
notice:
success: "The last script has been executed successfully."
error: "An error has occurred. You can see the details below."
working: "The script is running. The administrator who executed it will receive an email when it is finished."
delete_generated_content: "Generated content has been successfully deleted."
output_files: "Output files:"
related_content: "Related content"
related_content_description: "Adds automatically generated related content to proposals and participatory budget projects."
script_info: "You will receive an email in %{email} when the script finishes running."
script_name: "Script name:"
select_script: "Select python script to execute"
started_at: "Started at:"
tab_help: "Help"
tab_scripts: "Execute script"
tab_settings: "Settings / Generated content"
tags: "Tags"
tags_description: "Generates automatic tags on all items that can be tagged on."
title: "AI / Machine learning"

12
config/locales/en/general.yml
View File

@@ -965,7 +965,11 @@ en:
enqueue_remote_translation: Translations have been correctly requested. enqueue_remote_translation: Translations have been correctly requested.
button: Translate page button: Translate page
tags: tags:
filter: list:
more: filter:
one: "One more tag" more:
other: "%{count} more tags" one: "One more tag"
other: "%{count} more tags"
machine_learning:
comments_summary: "Comments summary"
info_text: "Content generated by AI / Machine Learning"

10
config/locales/en/mailers.yml
View File

@@ -78,6 +78,16 @@ en:
hi: Hi hi: Hi
new_comment_by: There is a new evaluation comment from <strong>%{commenter}</strong> to the budget investment %{investment} new_comment_by: There is a new evaluation comment from <strong>%{commenter}</strong> to the budget investment %{investment}
commenter_info: "%{commenter}, %{time}:" commenter_info: "%{commenter}, %{time}:"
machine_learning_error:
link: "Visit Machine Learning panel"
subject: "Machine Learning - An error has occurred running the script"
text: "An error has occurred running the Machine Learning script."
title: "Machine Learning script"
machine_learning_success:
link: "Visit Machine Learning panel"
subject: "Machine Learning - Content has been generated successfully"
text: "Content has been generated successfully."
title: "Machine Learning script"
new_actions_notification_rake_created: new_actions_notification_rake_created:
subject: "More news about your citizen proposal" subject: "More news about your citizen proposal"
hi: "Hello %{name}," hi: "Hello %{name},"

2
config/locales/en/settings.yml
View File

@@ -106,6 +106,8 @@ en:
recommendations_on_proposals_description: "Displays recommendations to users on the proposals page based on the tags of the items followed" recommendations_on_proposals_description: "Displays recommendations to users on the proposals page based on the tags of the items followed"
community: "Community on proposals and investments" community: "Community on proposals and investments"
community_description: "Enables the community section in the proposals and investment projects of the Participatory Budgets" community_description: "Enables the community section in the proposals and investment projects of the Participatory Budgets"
machine_learning: "AI / Machine learning"
machine_learning_description: "Enable the AI / Machine Learning section to run python scripts and enrich content automatically."
map: "Proposals and budget investments geolocation" map: "Proposals and budget investments geolocation"
map_description: "Enables geolocation of proposals and investment projects" map_description: "Enables geolocation of proposals and investment projects"
allow_images: "Allow upload and show images" allow_images: "Allow upload and show images"

52
config/locales/es/admin.yml
View File

@@ -750,6 +750,7 @@ es:
proposals: "Propuestas" proposals: "Propuestas"
polls: "Votaciones" polls: "Votaciones"
layouts: "Plantillas" layouts: "Plantillas"
machine_learning: "IA / Machine Learning"
mailers: "Correos" mailers: "Correos"
management: "Gestión" management: "Gestión"
welcome: "Bienvenido/a" welcome: "Bienvenido/a"
@@ -770,6 +771,7 @@ es:
debates: "Debates" debates: "Debates"
comments: "Comentarios" comments: "Comentarios"
local_census_records: Gestionar censo local local_census_records: Gestionar censo local
machine_learning: "IA / Machine learning"
administrators: administrators:
index: index:
title: Administradores title: Administradores
@@ -1682,3 +1684,53 @@ es:
created: Registros creados created: Registros creados
local_census_records: local_census_records:
no_records_found: No se han encontrado registros. no_records_found: No se han encontrado registros.
machine_learning:
cancel: "Cancelar operación"
cancel_alert: "Esta acción cancelará el script actual y será necesario ejecutar el script de nuevo."
comments_summary: "Resumen de comentarios"
comments_summary_description: "Muestra un resumen de comentarios generado automáticamente en todos los elementos que pueden ser comentados."
data_folder_content: "Contenido carpeta data"
error: "Error:"
execute_script: "Ejecutar script"
executed_by: "Ejecutado por:"
executed_script: "Script ejecutado:"
feature_disabled: "Esta funcionalidad está deshabilitada. Para utilizar Machine Learning puedes habilitarla desde la %{link}."
feature_disabled_link: "página de configuración"
help:
title_1: "¿Qué es IA / Machine Learning?"
description_1: "Tradicionalmente, los programas informáticos se diseñan estableciendo reglas precisas sobre lo que debe hacer el programa en cada situación, y cómo procesar en cada momento la información disponible. Lo que se conoce comúnmente como IA / Machine Learning es un tipo de programas para los cuales lo que se establece de manera precisa es cuál es la tarea a realizar y cómo se valora el que la tarea esté mejor o peor realizada. Pero a diferencia de los otros, el sistema descubre o aprende cuál sería el método más adecuado para realizar dicha tarea."
title_2: "¿Qué hace el módulo de IA / Machine Learning en CONSUL?"
description_2: "Este módulo permite implementar en CONSUL cualquier tipo de programas de IA / Machine Learning. Los programas pueden procesar la información disponible en CONSUL y producir resultados que ayuden tanto a los usuarios como a los administradores a llevar a cabo una participación ciudadana más eficaz e inteligente. El módulo se ha desarrollado para que sea sencillo implementar y ejecutar nuevos programas, manteniendo por separado el código general de CONSUL y el código de estos nuevos programas."
title_3: "Cómo usar el módulo"
description_3: "Para usarlo por primera vez es necesario seguir las instrucciones disponibles en la documentación de CONSUL para activar correctamente este módulo. Una vez el módulo esté correctamente activado, los programas se ejecutan desde la pestaña \"Ejecutar scripts\". Antes de ejecutarlo podemos ver en esa misma pestaña una estimación del tiempo de ejecución y otras informaciones relevantes. Al terminar la ejecución podemos activar el contenido que se mostrará en CONSUL desde la pestaña de \"Configuración\", así como descargar otros archivos generados que nos puedan resultar útiles."
title_4: "Cómo implementar nuevos scripts de IA / Machine Learning."
description_4: "Utiliza los scripts existentes como ejemplo."
description_4b: "Los nuevos scripts deben estar ubicados en la carpeta <code>public/machine_learning/scripts</code>"
description_4c: "Los datos de entrada que utilizará el script se crearán como archivos JSON en la carpeta <code>../data</code>"
description_4d: "Cualquier salida generada debe crearse en la carpeta <code>../data</code>"
description_4e: "Se recomienda crear un archivo de texto <code>.ini</code> independiente con los parámetros de configuración del script, con el mismo nombre del script y legible mediante configparser. Este archivo facilitará cambios de configuración por parte de los administradores."
description_4f: "Se recomienda incluir como primera línea del script una cadena inicial de comillas triples con información breve sobre el mismo y enlaces a cualquier información relevante. Esta cadena se mostrará automáticamente en la interfaz de administración."
description_4g: "Se recomienda crear un log con la información relevante del script en un archivo de texto <code>.log</code>, con el mismo nombre del script."
title_5: "Para utilizar los scripts de IA / Machine Learning necesitas tener Python instalado en tu servidor"
description_5: "Aquí puedes ver las instrucciones de ejemplo para un servidor Ubuntu 18.04:"
help_text: "Esta funcionalidad es experimental."
last_execution: "Última ejecución"
no_content: "Todavía no se ha generado contenido."
notice:
success: "El último script se ha ejecutado correctamente."
error: "Se ha producido un error. Más información sobre el error a continuación."
working: "El script se está ejecutando. El administrador que lo ejecutó recibirá un email cuando termine."
delete_generated_content: "El contenido generado se ha borrado correctamente."
output_files: "Ficheros de salida:"
related_content: "Contenido relacionado"
related_content_description: "Añade contenido relacionado generado automáticamente a las propuestas y proyectos de presupuestos participativos."
script_info: "Recibirás un email en %{email} cuando el script termine de ejecutarse."
script_name: "Nombre del script:"
select_script: "Seleccione el script pyhton a ejecutar"
started_at: "Empezado a las:"
tab_help: "Ayuda"
tab_settings: "Configuración / Contenido generado"
tab_scripts: "Ejecutar script"
tags: "Etiquetas"
tags_description: "Genera etiquetas automáticas para todos los elementos que pueden ser etiquetados."
title: "IA / Machine learning"

12
config/locales/es/general.yml
View File

@@ -965,7 +965,11 @@ es:
enqueue_remote_translation: Se han solicitado correctamente las traducciones. enqueue_remote_translation: Se han solicitado correctamente las traducciones.
button: Traducir página button: Traducir página
tags: tags:
filter: list:
more: filter:
one: "Una etiqueta más" more:
other: "%{count} etiquetas más" one: "Una etiqueta más"
other: "%{count} etiquetas más"
machine_learning:
comments_summary: "Resumen de comentarios"
info_text: "Contenido generado mediante IA / Machine Learning"

10
config/locales/es/mailers.yml
View File

@@ -78,6 +78,16 @@ es:
hi: Hola hi: Hola
new_comment_by: Hay un nuevo comentario de evaluación de <strong>%{commenter}</strong> en el presupuesto participativo %{investment} new_comment_by: Hay un nuevo comentario de evaluación de <strong>%{commenter}</strong> en el presupuesto participativo %{investment}
commenter_info: "%{commenter}, %{time}" commenter_info: "%{commenter}, %{time}"
machine_learning_error:
link: "Visitar el panel de Machine Learning"
subject: "Machine learning - Se ha producido un error"
text: "Se ha producido un error ejecutando el script de Machine Learning."
title: "Script Machine Learning"
machine_learning_success:
link: "Visitar el panel de Machine Learning"
subject: "Machine learning - Contenido generado correctamente"
text: "El contenido ha sido generado correctamente."
title: "Script Machine Learning"
new_actions_notification_rake_created: new_actions_notification_rake_created:
subject: "Más novedades de tu propuesta ciudadana" subject: "Más novedades de tu propuesta ciudadana"
hi: "Hola %{name}," hi: "Hola %{name},"

2
config/locales/es/settings.yml
View File

@@ -106,6 +106,8 @@ es:
recommendations_on_proposals_description: "Muestra a los usuarios recomendaciones en la página de propuestas basado en las etiquetas de los elementos que sigue" recommendations_on_proposals_description: "Muestra a los usuarios recomendaciones en la página de propuestas basado en las etiquetas de los elementos que sigue"
community: "Comunidad en propuestas y proyectos de gasto" community: "Comunidad en propuestas y proyectos de gasto"
community_description: "Activa la sección de comunidad en las propuestas y en los proyectos de gasto de los Presupuestos participativos" community_description: "Activa la sección de comunidad en las propuestas y en los proyectos de gasto de los Presupuestos participativos"
machine_learning: "IA / Machine learning"
machine_learning_description: "Habilita la sección de IA / Machine Learning para ejecutar scripts de python y enriquecer el contenido automáticamente."
map: "Geolocalización de propuestas y proyectos de gasto" map: "Geolocalización de propuestas y proyectos de gasto"
map_description: "Activa la geolocalización de propuestas y proyectos de gasto" map_description: "Activa la geolocalización de propuestas y proyectos de gasto"
allow_images: "Permitir subir y mostrar imágenes" allow_images: "Permitir subir y mostrar imágenes"

5
config/routes/admin.rb
View File

@@ -270,6 +270,11 @@ namespace :admin do
namespace :local_census_records do namespace :local_census_records do
resources :imports, only: [:new, :create, :show] resources :imports, only: [:new, :create, :show]
end end
resource :machine_learning, controller: :machine_learning, only: [:show] do
post :execute, on: :collection
delete :cancel, on: :collection
end
end end
resolve "Milestone" do |milestone| resolve "Milestone" do |milestone|

14
db/migrate/20210422104400_create_machine_learning_jobs.rb Normal file
View File

@@ -0,0 +1,14 @@
class CreateMachineLearningJobs < ActiveRecord::Migration[5.2]
def change
create_table :machine_learning_jobs do |t|
t.datetime :started_at
t.datetime :finished_at
t.string :script
t.integer :pid
t.string :error
t.references :user, foreign_key: true
t.timestamps
end
end
end

11
db/migrate/20210423114500_create_ml_summary_comments.rb Normal file
View File

@@ -0,0 +1,11 @@
class CreateMlSummaryComments < ActiveRecord::Migration[5.2]
def change
create_table :ml_summary_comments do |t|
t.integer :commentable_id
t.string :commentable_type
t.text :body
t.timestamps
end
end
end

6
db/migrate/20210424092400_add_machine_learning_fields_to_related_contents.rb Normal file
View File

@@ -0,0 +1,6 @@
class AddMachineLearningFieldsToRelatedContents < ActiveRecord::Migration[5.2]
def change
add_column :related_contents, :machine_learning, :boolean, default: false
add_column :related_contents, :machine_learning_score, :integer, default: 0
end
end

11
db/migrate/20210519115700_create_machine_learning_infos.rb Normal file
View File

@@ -0,0 +1,11 @@
class CreateMachineLearningInfos < ActiveRecord::Migration[5.2]
def change
create_table :machine_learning_infos do |t|
t.string :kind
t.datetime :generated_at
t.string :script
t.timestamps
end
end
end

33
db/schema.rb
View File

@@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2021_01_23_100638) do ActiveRecord::Schema.define(version: 2021_05_19_115700) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "pg_trgm" enable_extension "pg_trgm"
@@ -875,6 +875,26 @@ ActiveRecord::Schema.define(version: 2021_01_23_100638) do
t.index ["user_id"], name: "index_locks_on_user_id" t.index ["user_id"], name: "index_locks_on_user_id"
end end
create_table "machine_learning_infos", force: :cascade do |t|
t.string "kind"
t.datetime "generated_at"
t.string "script"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "machine_learning_jobs", force: :cascade do |t|
t.datetime "started_at"
t.datetime "finished_at"
t.string "script"
t.integer "pid"
t.string "error"
t.bigint "user_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["user_id"], name: "index_machine_learning_jobs_on_user_id"
end
create_table "managers", id: :serial, force: :cascade do |t| create_table "managers", id: :serial, force: :cascade do |t|
t.integer "user_id" t.integer "user_id"
t.index ["user_id"], name: "index_managers_on_user_id" t.index ["user_id"], name: "index_managers_on_user_id"
@@ -920,6 +940,14 @@ ActiveRecord::Schema.define(version: 2021_01_23_100638) do
t.index ["status_id"], name: "index_milestones_on_status_id" t.index ["status_id"], name: "index_milestones_on_status_id"
end end
create_table "ml_summary_comments", force: :cascade do |t|
t.integer "commentable_id"
t.string "commentable_type"
t.text "body"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "moderators", id: :serial, force: :cascade do |t| create_table "moderators", id: :serial, force: :cascade do |t|
t.integer "user_id" t.integer "user_id"
t.index ["user_id"], name: "index_moderators_on_user_id" t.index ["user_id"], name: "index_moderators_on_user_id"
@@ -1278,6 +1306,8 @@ ActiveRecord::Schema.define(version: 2021_01_23_100638) do
t.datetime "hidden_at" t.datetime "hidden_at"
t.integer "related_content_scores_count", default: 0 t.integer "related_content_scores_count", default: 0
t.integer "author_id" t.integer "author_id"
t.boolean "machine_learning", default: false
t.integer "machine_learning_score", default: 0
t.index ["child_relationable_type", "child_relationable_id"], name: "index_related_contents_on_child_relationable" t.index ["child_relationable_type", "child_relationable_id"], name: "index_related_contents_on_child_relationable"
t.index ["hidden_at"], name: "index_related_contents_on_hidden_at" t.index ["hidden_at"], name: "index_related_contents_on_hidden_at"
t.index ["parent_relationable_id", "parent_relationable_type", "child_relationable_id", "child_relationable_type"], name: "unique_parent_child_related_content", unique: true t.index ["parent_relationable_id", "parent_relationable_type", "child_relationable_id", "child_relationable_type"], name: "unique_parent_child_related_content", unique: true
@@ -1704,6 +1734,7 @@ ActiveRecord::Schema.define(version: 2021_01_23_100638) do
add_foreign_key "legislation_draft_versions", "legislation_processes" add_foreign_key "legislation_draft_versions", "legislation_processes"
add_foreign_key "legislation_proposals", "legislation_processes" add_foreign_key "legislation_proposals", "legislation_processes"
add_foreign_key "locks", "users" add_foreign_key "locks", "users"
add_foreign_key "machine_learning_jobs", "users"
add_foreign_key "managers", "users" add_foreign_key "managers", "users"
add_foreign_key "moderators", "users" add_foreign_key "moderators", "users"
add_foreign_key "notifications", "users" add_foreign_key "notifications", "users"

86
public/machine_learning/scripts/budgets_related_content_and_tags_nmf.py Normal file
View File

@@ -0,0 +1,86 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
"""
Related Participatory Budgeting projects and Tags - Dummy script
"""
# In[2]:
data_path = '../data'
config_file = 'budgets_related_content_and_tags_nmf.ini'
logging_file ='budgets_related_content_and_tags_nmf.log'
# In[3]:
# Input file:
inputjsonfile = 'budget_investments.json'
# Output files:
taggings_filename = 'ml_taggings_budgets.json'
tags_filename = 'ml_tags_budgets.json'
related_props_filename = 'ml_related_content_budgets.json'
# In[4]:
import os
import pandas as pd
# ### Read the proposals
# In[5]:
# proposals_input_df = pd.read_json(os.path.join(data_path,inputjsonfile),orient="records")
# col_id = 'id'
# cols_content = ['title','description']
# proposals_input_df = proposals_input_df[[col_id]+cols_content]
# ### Create file: Taggings. Each line is a Tag associated to a Proposal
# In[6]:
taggings_file_cols = ['tag_id','taggable_id','taggable_type']
taggings_file_df = pd.DataFrame(columns=taggings_file_cols)
row = [0,1,'Budget::Investment']
taggings_file_df = taggings_file_df.append(dict(zip(taggings_file_cols,row)), ignore_index=True)
taggings_file_df.to_json(os.path.join(data_path,taggings_filename),orient="records", force_ascii=False)
# ### Create file: Tags. List of Tags with the number of times they have been used
# In[7]:
tags_file_cols = ['id','name','taggings_count','kind']
tags_file_df = pd.DataFrame(columns=tags_file_cols)
row = [0,'tag',0,'']
tags_file_df = tags_file_df.append(dict(zip(tags_file_cols,row)), ignore_index=True)
tags_file_df.to_json(os.path.join(data_path,tags_filename),orient="records", force_ascii=False)
# ### Create file: List of related proposals
# In[8]:
numb_related_proposals = 2
related_props_cols = ['id']+['related'+str(num) for num in range(1,numb_related_proposals+1)]
related_props_df = pd.DataFrame(columns=related_props_cols)
row = [1]+['' for num in range(1,numb_related_proposals+1)]
related_props_df = related_props_df.append(dict(zip(related_props_cols,row)), ignore_index=True)
related_props_df.to_json(os.path.join(data_path,related_props_filename),orient="records", force_ascii=False)

59
public/machine_learning/scripts/budgets_summary_comments_textrank.py Normal file
View File

@@ -0,0 +1,59 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
"""
Participatory Budgeting comments summaries - Dummy script
"""
# In[2]:
data_path = '../data'
config_file = 'budgets_summary_comments_textrank.ini'
logging_file ='budgets_summary_comments_textrank.log'
# In[3]:
# Input file:
inputjsonfile = 'comments.json'
# Output files:
comments_summaries_filename = 'ml_comments_summaries_budgets.json'
# In[4]:
import os
import pandas as pd
# ### Read the comments
# In[5]:
# comments_input_df = pd.read_json(os.path.join(data_path,inputjsonfile),orient="records")
# col_id = 'commentable_id'
# col_content = 'body'
# comments_input_df = comments_input_df[[col_id]+[col_content]]
# ### Create file. Comments summaries
# In[6]:
comments_summaries_cols = ['id','commentable_id','commentable_type','body']
comments_summaries_df = pd.DataFrame(columns=comments_summaries_cols)
row = [0,0,'Budget::Investment','Summary']
comments_summaries_df = comments_summaries_df.append(dict(zip(comments_summaries_cols,row)), ignore_index=True)
comments_summaries_df.to_json(os.path.join(data_path,comments_summaries_filename),orient="records", force_ascii=False)

86
public/machine_learning/scripts/proposals_related_content_and_tags_nmf.py Normal file
View File

@@ -0,0 +1,86 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
"""
Related Proposals and Tags - Dummy script
"""
# In[2]:
data_path = '../data'
config_file = 'proposals_related_content_and_tags_nmf.ini'
logging_file ='proposals_related_content_and_tags_nmf.log'
# In[3]:
# Input file:
inputjsonfile = 'proposals.json'
# Output files:
taggings_filename = 'ml_taggings_proposals.json'
tags_filename = 'ml_tags_proposals.json'
related_props_filename = 'ml_related_content_proposals.json'
# In[4]:
import os
import pandas as pd
# ### Read the proposals
# In[5]:
# proposals_input_df = pd.read_json(os.path.join(data_path,inputjsonfile),orient="records")
# col_id = 'id'
# cols_content = ['title','description','summary']
# proposals_input_df = proposals_input_df[[col_id]+cols_content]
# ### Create file: Taggings. Each line is a Tag associated to a Proposal
# In[6]:
taggings_file_cols = ['tag_id','taggable_id','taggable_type']
taggings_file_df = pd.DataFrame(columns=taggings_file_cols)
row = [0,1,'Proposal']
taggings_file_df = taggings_file_df.append(dict(zip(taggings_file_cols,row)), ignore_index=True)
taggings_file_df.to_json(os.path.join(data_path,taggings_filename),orient="records", force_ascii=False)
# ### Create file: Tags. List of Tags with the number of times they have been used
# In[7]:
tags_file_cols = ['id','name','taggings_count','kind']
tags_file_df = pd.DataFrame(columns=tags_file_cols)
row = [0,'tag',0,'']
tags_file_df = tags_file_df.append(dict(zip(tags_file_cols,row)), ignore_index=True)
tags_file_df.to_json(os.path.join(data_path,tags_filename),orient="records", force_ascii=False)
# ### Create file: List of related proposals
# In[8]:
numb_related_proposals = 2
related_props_cols = ['id']+['related'+str(num) for num in range(1,numb_related_proposals+1)]
related_props_df = pd.DataFrame(columns=related_props_cols)
row = [1]+['' for num in range(1,numb_related_proposals+1)]
related_props_df = related_props_df.append(dict(zip(related_props_cols,row)), ignore_index=True)
related_props_df.to_json(os.path.join(data_path,related_props_filename),orient="records", force_ascii=False)

59
public/machine_learning/scripts/proposals_summary_comments_textrank.py Normal file
View File

@@ -0,0 +1,59 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
"""
Proposals comments summaries - Dummy script
"""
# In[2]:
data_path = '../data'
config_file = 'proposals_summary_comments_textrank.ini'
logging_file ='proposals_summary_comments_textrank.log'
# In[3]:
# Input file:
inputjsonfile = 'comments.json'
# Output files:
comments_summaries_filename = 'ml_comments_summaries_proposals.json'
# In[4]:
import os
import pandas as pd
# ### Read the comments
# In[5]:
# comments_input_df = pd.read_json(os.path.join(data_path,inputjsonfile),orient="records")
# col_id = 'commentable_id'
# col_content = 'body'
# comments_input_df = comments_input_df[[col_id]+[col_content]]
# ### Create file. Comments summaries
# In[6]:
comments_summaries_cols = ['id','commentable_id','commentable_type','body']
comments_summaries_df = pd.DataFrame(columns=comments_summaries_cols)
row = [0,0,'Proposal','Summary']
comments_summaries_df = comments_summaries_df.append(dict(zip(comments_summaries_cols,row)), ignore_index=True)
comments_summaries_df.to_json(os.path.join(data_path,comments_summaries_filename),orient="records", force_ascii=False)

44
spec/components/machine_learning/comments_summary_component_spec.rb Normal file
View File

@@ -0,0 +1,44 @@
require "rails_helper"
describe MachineLearning::CommentsSummaryComponent, type: :component do
let(:commentable) { double(summary_comment: double(body: "There's a general agreement")) }
let(:component) { MachineLearning::CommentsSummaryComponent.new(commentable) }
before do
Setting["feature.machine_learning"] = true
Setting["machine_learning.comments_summary"] = true
allow(controller).to receive(:current_user).and_return(nil)
end
it "is displayed when the setting is enabled" do
render_inline component
expect(page).to have_content "Comments summary"
expect(page).to have_content "There's a general agreement"
expect(page).to have_content "Content generated by AI / Machine Learning"
end
it "is not displayed when the setting is disabled" do
Setting["machine_learning.comments_summary"] = false
render_inline component
expect(page.native.inner_html).to be_empty
end
it "is not displayed when the machine learning feature is disabled" do
Setting["feature.machine_learning"] = false
render_inline component
expect(page.native.inner_html).to be_empty
end
it "is not displayed when there's no summary" do
commentable = double(summary_comment: double(body: ""))
render_inline MachineLearning::CommentsSummaryComponent.new(commentable)
expect(page.native.inner_html).to be_empty
end
end

48
spec/components/relationable/related_list_component_spec.rb Normal file
View File

@@ -0,0 +1,48 @@
require "rails_helper"
describe Relationable::RelatedListComponent, type: :component do
let(:proposal) { create(:proposal) }
let(:user_proposal) { create(:proposal, title: "I am user related") }
let(:machine_proposal) { create(:proposal, title: "I am machine related") }
let(:component) { Relationable::RelatedListComponent.new(proposal) }
before do
Setting["feature.machine_learning"] = true
Setting["machine_learning.related_content"] = true
create(:related_content, parent_relationable: proposal, child_relationable: user_proposal)
create(:related_content, parent_relationable: proposal,
child_relationable: machine_proposal,
machine_learning: true)
allow(controller).to receive(:current_user).and_return(nil)
end
it "displays machine learning and user content when machine learning is enabled" do
render_inline component
expect(page).to have_css "li", count: 2
expect(page).to have_content "I am machine related"
expect(page).to have_content "I am user related"
end
it "displays user related content when machine learning is disabled" do
Setting["feature.machine_learning"] = false
render_inline component
expect(page).to have_css "li", count: 1
expect(page).to have_content "I am user related"
expect(page).not_to have_content "I am machine related"
end
it "displays user related content when machine learning related content is disabled" do
Setting["machine_learning.related_content"] = false
render_inline component
expect(page).to have_css "li", count: 1
expect(page).to have_content "I am user related"
expect(page).not_to have_content "I am machine related"
end
end

48
spec/components/shared/tag_list_component_spec.rb Normal file
View File

@@ -0,0 +1,48 @@
require "rails_helper"
describe Shared::TagListComponent, type: :component do
let(:user_tag) { create(:tag, name: "user tag") }
let(:ml_tag) { create(:tag, name: "machine learning tag") }
let(:proposal) { create(:proposal, tag_list: [user_tag], ml_tag_list: [ml_tag]) }
let(:component) { Shared::TagListComponent.new(proposal, limit: nil) }
before do
Setting["feature.machine_learning"] = true
Setting["machine_learning.tags"] = true
allow(controller).to receive(:current_user).and_return(create(:administrator).user)
end
it "displays machine learning tags when machine learning is enabled" do
render_inline component
expect(page).not_to have_link "user tag"
expect(page).to have_link "machine learning tag"
expect(page).to have_content "Content generated by AI / Machine Learning"
end
it "displays user tags when machine learning is disabled" do
Setting["feature.machine_learning"] = false
render_inline component
expect(page).to have_link "user tag"
expect(page).not_to have_link "machine learning tag"
expect(page).not_to have_content "Content generated by AI / Machine Learning"
end
it "displays user tags when machine learning tags are disabled" do
Setting["machine_learning.tags"] = false
render_inline component
expect(page).to have_link "user tag"
expect(page).not_to have_link "machine learning tag"
expect(page).not_to have_content "Content generated by AI / Machine Learning"
end
it "is not rendered when there are no tags" do
render_inline Shared::TagListComponent.new(Proposal.new, limit: nil)
expect(page.native.inner_html).to be_empty
end
end

14
spec/factories/classifications.rb
View File

@@ -41,6 +41,20 @@ FactoryBot.define do
association :author, factory: :user association :author, factory: :user
association :parent_relationable, factory: [:proposal, :debate].sample association :parent_relationable, factory: [:proposal, :debate].sample
association :child_relationable, factory: [:proposal, :debate].sample association :child_relationable, factory: [:proposal, :debate].sample
trait :proposals do
association :parent_relationable, factory: :proposal
association :child_relationable, factory: :proposal
end
trait :budget_investments do
association :parent_relationable, factory: :budget_investment
association :child_relationable, factory: :budget_investment
end
trait :from_machine_learning do
machine_learning { true }
end
end end
factory :related_content_score do factory :related_content_score do

19
spec/factories/machine_learning.rb Normal file
View File

@@ -0,0 +1,19 @@
FactoryBot.define do
factory :machine_learning_job do
association :user, factory: :user
script { "script.py" }
started_at { Time.current }
finished_at { nil }
error { nil }
end
factory :machine_learning_info do
kind { "tags" }
generated_at { Time.current }
script { "script.py" }
end
factory :ml_summary_comment do
body { "Summary comment generated by Machine Learning" }
end
end

610
spec/models/machine_learning_spec.rb Normal file
View File

@@ -0,0 +1,610 @@
require "rails_helper"
describe MachineLearning do
def full_sanitizer(string)
ActionView::Base.full_sanitizer.sanitize(string)
end
let(:job) { create(:machine_learning_job) }
describe "#cleanup_proposals_tags!" do
it "does not delete other machine learning generated data" do
create(:ml_summary_comment, commentable: create(:proposal))
create(:ml_summary_comment, commentable: create(:budget_investment))
create(:related_content, :proposals, :from_machine_learning)
create(:related_content, :budget_investments, :from_machine_learning)
expect(MlSummaryComment.count).to be 2
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_proposals_tags!)
expect(MlSummaryComment.count).to be 2
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
end
it "deletes proposals tags machine learning generated data" do
proposal = create(:proposal)
investment = create(:budget_investment)
user_tag = create(:tag)
create(:tagging, tag: user_tag, taggable: proposal)
ml_proposal_tag = create(:tag)
create(:tagging, tag: ml_proposal_tag, taggable: proposal, context: "ml_tags")
ml_investment_tag = create(:tag)
create(:tagging, tag: ml_investment_tag, taggable: investment, context: "ml_tags")
common_tag = create(:tag)
create(:tagging, tag: common_tag, taggable: proposal)
create(:tagging, tag: common_tag, taggable: proposal, context: "ml_tags")
create(:tagging, tag: common_tag, taggable: investment, context: "ml_tags")
expect(Tag.count).to be 4
expect(Tagging.count).to be 6
expect(Tagging.where(context: "tags").count).to be 2
expect(Tagging.where(context: "ml_tags", taggable_type: "Proposal").count).to be 2
expect(Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment").count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_proposals_tags!)
expect(Tag.count).to be 3
expect(Tag.all).not_to include ml_proposal_tag
expect(Tagging.count).to be 4
expect(Tagging.where(context: "tags").count).to be 2
expect(Tagging.where(context: "ml_tags", taggable_type: "Proposal")).to be_empty
expect(Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment").count).to be 2
end
end
describe "#cleanup_investments_tags!" do
it "does not delete other machine learning generated data" do
create(:ml_summary_comment, commentable: create(:proposal))
create(:ml_summary_comment, commentable: create(:budget_investment))
create(:related_content, :proposals, :from_machine_learning)
create(:related_content, :budget_investments, :from_machine_learning)
expect(MlSummaryComment.count).to be 2
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_investments_tags!)
expect(MlSummaryComment.count).to be 2
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
end
it "deletes investments tags machine learning generated data" do
proposal = create(:proposal)
investment = create(:budget_investment)
user_tag = create(:tag)
create(:tagging, tag: user_tag, taggable: investment)
ml_investment_tag = create(:tag)
create(:tagging, tag: ml_investment_tag, taggable: investment, context: "ml_tags")
ml_proposal_tag = create(:tag)
create(:tagging, tag: ml_proposal_tag, taggable: proposal, context: "ml_tags")
common_tag = create(:tag)
create(:tagging, tag: common_tag, taggable: investment)
create(:tagging, tag: common_tag, taggable: investment, context: "ml_tags")
create(:tagging, tag: common_tag, taggable: proposal, context: "ml_tags")
expect(Tag.count).to be 4
expect(Tagging.count).to be 6
expect(Tagging.where(context: "tags").count).to be 2
expect(Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment").count).to be 2
expect(Tagging.where(context: "ml_tags", taggable_type: "Proposal").count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_investments_tags!)
expect(Tag.count).to be 3
expect(Tag.all).not_to include ml_investment_tag
expect(Tagging.count).to be 4
expect(Tagging.where(context: "tags").count).to be 2
expect(Tagging.where(context: "ml_tags", taggable_type: "Budget::Investment")).to be_empty
expect(Tagging.where(context: "ml_tags", taggable_type: "Proposal").count).to be 2
end
end
describe "#cleanup_proposals_related_content!" do
it "does not delete other machine learning generated data" do
proposal = create(:proposal)
investment = create(:budget_investment)
create(:ml_summary_comment, commentable: proposal)
create(:ml_summary_comment, commentable: investment)
create(:tagging, tag: create(:tag))
create(:tagging, tag: create(:tag), context: "ml_tags", taggable: proposal)
create(:tagging, tag: create(:tag), context: "ml_tags", taggable: investment)
expect(MlSummaryComment.count).to be 2
expect(Tag.count).to be 3
expect(Tagging.count).to be 3
expect(Tagging.where(context: "tags").count).to be 1
expect(Tagging.where(context: "ml_tags").count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_proposals_related_content!)
expect(MlSummaryComment.count).to be 2
expect(Tag.count).to be 3
expect(Tagging.count).to be 3
expect(Tagging.where(context: "tags").count).to be 1
expect(Tagging.where(context: "ml_tags").count).to be 2
end
it "deletes proposals related content machine learning generated data" do
create(:related_content, :proposals)
create(:related_content, :budget_investments)
create(:related_content, :proposals, :from_machine_learning)
create(:related_content, :budget_investments, :from_machine_learning)
expect(RelatedContent.for_proposals.from_users.count).to be 2
expect(RelatedContent.for_investments.from_users.count).to be 2
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_proposals_related_content!)
expect(RelatedContent.for_proposals.from_users.count).to be 2
expect(RelatedContent.for_investments.from_users.count).to be 2
expect(RelatedContent.for_proposals.from_machine_learning).to be_empty
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
end
end
describe "#cleanup_investments_related_content!" do
it "does not delete other machine learning generated data" do
proposal = create(:proposal)
investment = create(:budget_investment)
create(:ml_summary_comment, commentable: proposal)
create(:ml_summary_comment, commentable: investment)
create(:tagging, tag: create(:tag))
create(:tagging, tag: create(:tag), context: "ml_tags", taggable: proposal)
create(:tagging, tag: create(:tag), context: "ml_tags", taggable: investment)
expect(MlSummaryComment.count).to be 2
expect(Tag.count).to be 3
expect(Tagging.count).to be 3
expect(Tagging.where(context: "tags").count).to be 1
expect(Tagging.where(context: "ml_tags").count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_investments_related_content!)
expect(MlSummaryComment.count).to be 2
expect(Tag.count).to be 3
expect(Tagging.count).to be 3
expect(Tagging.where(context: "tags").count).to be 1
expect(Tagging.where(context: "ml_tags").count).to be 2
end
it "deletes proposals related content machine learning generated data" do
create(:related_content, :proposals)
create(:related_content, :budget_investments)
create(:related_content, :proposals, :from_machine_learning)
create(:related_content, :budget_investments, :from_machine_learning)
expect(RelatedContent.for_proposals.from_users.count).to be 2
expect(RelatedContent.for_investments.from_users.count).to be 2
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_investments_related_content!)
expect(RelatedContent.for_proposals.from_users.count).to be 2
expect(RelatedContent.for_investments.from_users.count).to be 2
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning).to be_empty
end
end
describe "#cleanup_proposals_comments_summary!" do
it "does not delete other machine learning generated data" do
create(:related_content, :proposals, :from_machine_learning)
create(:related_content, :budget_investments, :from_machine_learning)
create(:tagging, tag: create(:tag))
create(:tagging, tag: create(:tag), context: "ml_tags", taggable: create(:proposal))
create(:tagging, tag: create(:tag), context: "ml_tags", taggable: create(:budget_investment))
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
expect(Tag.count).to be 3
expect(Tagging.count).to be 3
expect(Tagging.where(context: "tags").count).to be 1
expect(Tagging.where(context: "ml_tags").count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_proposals_comments_summary!)
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
expect(Tag.count).to be 3
expect(Tagging.count).to be 3
expect(Tagging.where(context: "tags").count).to be 1
expect(Tagging.where(context: "ml_tags").count).to be 2
end
it "deletes proposals comments summary machine learning generated data" do
create(:ml_summary_comment, commentable: create(:proposal))
create(:ml_summary_comment, commentable: create(:budget_investment))
expect(MlSummaryComment.where(commentable_type: "Proposal").count).to be 1
expect(MlSummaryComment.where(commentable_type: "Budget::Investment").count).to be 1
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_proposals_comments_summary!)
expect(MlSummaryComment.where(commentable_type: "Proposal")).to be_empty
expect(MlSummaryComment.where(commentable_type: "Budget::Investment").count).to be 1
end
end
describe "#cleanup_investments_comments_summary!" do
it "does not delete other machine learning generated data" do
create(:related_content, :proposals, :from_machine_learning)
create(:related_content, :budget_investments, :from_machine_learning)
create(:tagging, tag: create(:tag))
create(:tagging, tag: create(:tag), context: "ml_tags", taggable: create(:proposal))
create(:tagging, tag: create(:tag), context: "ml_tags", taggable: create(:budget_investment))
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
expect(Tag.count).to be 3
expect(Tagging.count).to be 3
expect(Tagging.where(context: "tags").count).to be 1
expect(Tagging.where(context: "ml_tags").count).to be 2
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_investments_comments_summary!)
expect(RelatedContent.for_proposals.from_machine_learning.count).to be 2
expect(RelatedContent.for_investments.from_machine_learning.count).to be 2
expect(Tag.count).to be 3
expect(Tagging.count).to be 3
expect(Tagging.where(context: "tags").count).to be 1
expect(Tagging.where(context: "ml_tags").count).to be 2
end
it "deletes budget investments comments summary machine learning generated data" do
create(:ml_summary_comment, commentable: create(:proposal))
create(:ml_summary_comment, commentable: create(:budget_investment))
expect(MlSummaryComment.where(commentable_type: "Proposal").count).to be 1
expect(MlSummaryComment.where(commentable_type: "Budget::Investment").count).to be 1
machine_learning = MachineLearning.new(job)
machine_learning.send(:cleanup_investments_comments_summary!)
expect(MlSummaryComment.where(commentable_type: "Proposal").count).to be 1
expect(MlSummaryComment.where(commentable_type: "Budget::Investment")).to be_empty
end
end
describe "#export_proposals_to_json" do
it "creates a JSON file with all proposals" do
require "fileutils"
FileUtils.mkdir_p Rails.root.join("public", "machine_learning", "data")
first_proposal = create(:proposal)
last_proposal = create(:proposal)
machine_learning = MachineLearning.new(job)
machine_learning.send(:export_proposals_to_json)
json_file = MachineLearning::DATA_FOLDER.join("proposals.json")
json = JSON.parse(File.read(json_file))
expect(json).to be_an Array
expect(json.size).to be 2
expect(json.first["id"]).to eq first_proposal.id
expect(json.first["title"]).to eq first_proposal.title
expect(json.first["summary"]).to eq full_sanitizer(first_proposal.summary)
expect(json.first["description"]).to eq full_sanitizer(first_proposal.description)
expect(json.last["id"]).to eq last_proposal.id
expect(json.last["title"]).to eq last_proposal.title
expect(json.last["summary"]).to eq full_sanitizer(last_proposal.summary)
expect(json.last["description"]).to eq full_sanitizer(last_proposal.description)
end
end
describe "#export_budget_investments_to_json" do
it "creates a JSON file with all budget investments" do
require "fileutils"
FileUtils.mkdir_p Rails.root.join("public", "machine_learning", "data")
first_budget_investment = create(:budget_investment)
last_budget_investment = create(:budget_investment)
machine_learning = MachineLearning.new(job)
machine_learning.send(:export_budget_investments_to_json)
json_file = MachineLearning::DATA_FOLDER.join("budget_investments.json")
json = JSON.parse(File.read(json_file))
expect(json).to be_an Array
expect(json.size).to be 2
expect(json.first["id"]).to eq first_budget_investment.id
expect(json.first["title"]).to eq first_budget_investment.title
expect(json.first["description"]).to eq full_sanitizer(first_budget_investment.description)
expect(json.last["id"]).to eq last_budget_investment.id
expect(json.last["title"]).to eq last_budget_investment.title
expect(json.last["description"]).to eq full_sanitizer(last_budget_investment.description)
end
end
describe "#export_comments_to_json" do
it "creates a JSON file with all comments" do
require "fileutils"
FileUtils.mkdir_p Rails.root.join("public", "machine_learning", "data")
first_comment = create(:comment)
last_comment = create(:comment)
machine_learning = MachineLearning.new(job)
machine_learning.send(:export_comments_to_json)
json_file = MachineLearning::DATA_FOLDER.join("comments.json")
json = JSON.parse(File.read(json_file))
expect(json).to be_an Array
expect(json.size).to be 2
expect(json.first["id"]).to eq first_comment.id
expect(json.first["commentable_id"]).to eq first_comment.commentable_id
expect(json.first["commentable_type"]).to eq first_comment.commentable_type
expect(json.first["body"]).to eq full_sanitizer(first_comment.body)
expect(json.last["id"]).to eq last_comment.id
expect(json.last["commentable_id"]).to eq last_comment.commentable_id
expect(json.last["commentable_type"]).to eq last_comment.commentable_type
expect(json.last["body"]).to eq full_sanitizer(last_comment.body)
end
end
describe "#run_machine_learning_scripts" do
it "returns true if python script executed correctly" do
machine_learning = MachineLearning.new(job)
command = "cd #{MachineLearning::SCRIPTS_FOLDER} && python script.py 2>&1"
expect(machine_learning).to receive(:`).with(command) do
Process.waitpid Process.fork { exit 0 }
end
expect(Mailer).not_to receive(:machine_learning_error)
expect(machine_learning.send(:run_machine_learning_scripts)).to be true
job.reload
expect(job.finished_at).not_to be_present
expect(job.error).not_to be_present
end
it "returns false if python script errored" do
machine_learning = MachineLearning.new(job)
command = "cd #{MachineLearning::SCRIPTS_FOLDER} && python script.py 2>&1"
expect(machine_learning).to receive(:`).with(command) do
Process.waitpid Process.fork { abort "error message" }
end
mailer = double("mailer")
expect(mailer).to receive(:deliver_later)
expect(Mailer).to receive(:machine_learning_error).and_return mailer
expect(machine_learning.send(:run_machine_learning_scripts)).to be false
job.reload
expect(job.finished_at).to be_present
expect(job.error).not_to eq "error message"
end
end
describe "#import_ml_proposals_comments_summary" do
it "feeds the database using content from the JSON file generated by the machine learning script" do
machine_learning = MachineLearning.new(job)
proposal = create(:proposal)
data = [
{ commentable_id: proposal.id,
commentable_type: "Proposal",
body: "Summary comment for proposal with ID #{proposal.id}" }
]
filename = "ml_comments_summaries_proposals.json"
json_file = MachineLearning::DATA_FOLDER.join(filename)
expect(File).to receive(:read).with(json_file).and_return data.to_json
machine_learning.send(:import_ml_proposals_comments_summary)
expect(proposal.summary_comment.body).to eq "Summary comment for proposal with ID #{proposal.id}"
end
end
describe "#import_ml_investments_comments_summary" do
it "feeds the database using content from the JSON file generated by the machine learning script" do
machine_learning = MachineLearning.new(job)
investment = create(:budget_investment)
data = [
{ commentable_id: investment.id,
commentable_type: "Budget::Investment",
body: "Summary comment for investment with ID #{investment.id}" }
]
filename = "ml_comments_summaries_budgets.json"
json_file = MachineLearning::DATA_FOLDER.join(filename)
expect(File).to receive(:read).with(json_file).and_return data.to_json
machine_learning.send(:import_ml_investments_comments_summary)
expect(investment.summary_comment.body).to eq "Summary comment for investment with ID #{investment.id}"
end
end
describe "#import_proposals_related_content" do
it "feeds the database using content from the JSON file generated by the machine learning script" do
machine_learning = MachineLearning.new(job)
proposal = create(:proposal)
related_proposal = create(:proposal)
other_related_proposal = create(:proposal)
data = [
{
"id" => proposal.id,
"related1" => related_proposal.id,
"related2" => other_related_proposal.id
}
]
filename = "ml_related_content_proposals.json"
json_file = MachineLearning::DATA_FOLDER.join(filename)
expect(File).to receive(:read).with(json_file).and_return data.to_json
machine_learning.send(:import_proposals_related_content)
expect(proposal.related_contents.count).to be 2
expect(proposal.related_contents.first.child_relationable).to eq related_proposal
expect(proposal.related_contents.last.child_relationable).to eq other_related_proposal
end
end
describe "#import_budget_investments_related_content" do
it "feeds the database using content from the JSON file generated by the machine learning script" do
machine_learning = MachineLearning.new(job)
investment = create(:budget_investment)
related_investment = create(:budget_investment)
other_related_investment = create(:budget_investment)
data = [
{
"id" => investment.id,
"related1" => related_investment.id,
"related2" => other_related_investment.id
}
]
filename = "ml_related_content_budgets.json"
json_file = MachineLearning::DATA_FOLDER.join(filename)
expect(File).to receive(:read).with(json_file).and_return data.to_json
machine_learning.send(:import_budget_investments_related_content)
expect(investment.related_contents.count).to be 2
expect(investment.related_contents.first.child_relationable).to eq related_investment
expect(investment.related_contents.last.child_relationable).to eq other_related_investment
end
end
describe "#import_ml_proposals_tags" do
it "feeds the database using content from the JSON file generated by the machine learning script" do
create(:tag, name: "Existing tag")
proposal = create(:proposal)
machine_learning = MachineLearning.new(job)
tags_data = [
{ id: 0,
name: "Existing tag" },
{ id: 1,
name: "Machine learning tag" }
]
taggings_data = [
{ tag_id: 0,
taggable_id: proposal.id
},
{ tag_id: 1,
taggable_id: proposal.id
}
]
tags_filename = "ml_tags_proposals.json"
tags_json_file = MachineLearning::DATA_FOLDER.join(tags_filename)
expect(File).to receive(:read).with(tags_json_file).and_return tags_data.to_json
taggings_filename = "ml_taggings_proposals.json"
taggings_json_file = MachineLearning::DATA_FOLDER.join(taggings_filename)
expect(File).to receive(:read).with(taggings_json_file).and_return taggings_data.to_json
machine_learning.send(:import_ml_proposals_tags)
expect(Tag.count).to be 2
expect(Tag.first.name).to eq "Existing tag"
expect(Tag.last.name).to eq "Machine learning tag"
expect(proposal.tags).to be_empty
expect(proposal.ml_tags.count).to be 2
expect(proposal.ml_tags.first.name).to eq "Existing tag"
expect(proposal.ml_tags.last.name).to eq "Machine learning tag"
end
end
describe "#import_ml_investments_tags" do
it "feeds the database using content from the JSON file generated by the machine learning script" do
create(:tag, name: "Existing tag")
investment = create(:budget_investment)
machine_learning = MachineLearning.new(job)
tags_data = [
{ id: 0,
name: "Existing tag" },
{ id: 1,
name: "Machine learning tag" }
]
taggings_data = [
{ tag_id: 0,
taggable_id: investment.id
},
{ tag_id: 1,
taggable_id: investment.id
}
]
tags_filename = "ml_tags_budgets.json"
tags_json_file = MachineLearning::DATA_FOLDER.join(tags_filename)
expect(File).to receive(:read).with(tags_json_file).and_return tags_data.to_json
taggings_filename = "ml_taggings_budgets.json"
taggings_json_file = MachineLearning::DATA_FOLDER.join(taggings_filename)
expect(File).to receive(:read).with(taggings_json_file).and_return taggings_data.to_json
machine_learning.send(:import_ml_investments_tags)
expect(Tag.count).to be 2
expect(Tag.first.name).to eq "Existing tag"
expect(Tag.last.name).to eq "Machine learning tag"
expect(investment.tags).to be_empty
expect(investment.ml_tags.count).to be 2
expect(investment.ml_tags.first.name).to eq "Existing tag"
expect(investment.ml_tags.last.name).to eq "Machine learning tag"
end
end
end

255
spec/system/admin/machine_learning_spec.rb Normal file
View File

@@ -0,0 +1,255 @@
require "rails_helper"
describe "Machine learning" do
let(:admin) { create(:administrator) }
before do
login_as(admin.user)
Setting["feature.machine_learning"] = true
end
scenario "Section does not appear if feature is not enabled" do
Setting["feature.machine_learning"] = false
visit admin_root_path
within "#admin_menu" do
expect(page).not_to have_link "AI / Machine learning"
end
end
scenario "Section appears if feature is enabled" do
visit admin_root_path
within "#admin_menu" do
expect(page).to have_link "AI / Machine learning"
end
click_link "AI / Machine learning"
expect(page).to have_content "AI / Machine learning"
expect(page).to have_content "This functionality is experimental"
expect(page).to have_link "Execute script"
expect(page).to have_link "Settings / Generated content"
expect(page).to have_link "Help"
expect(page).to have_current_path(admin_machine_learning_path)
end
scenario "Show message if feature is disabled" do
Setting["feature.machine_learning"] = false
visit admin_machine_learning_path
expect(page).to have_content "This feature is disabled. To use Machine Learning you can enable it from "\
"the settings page"
expect(page).to have_link "settings page", href: admin_settings_path(anchor: "tab-feature-flags")
end
scenario "Script executed sucessfully" do
allow_any_instance_of(MachineLearning).to receive(:run) do
MachineLearningJob.first.update!(finished_at: Time.current)
end
visit admin_machine_learning_path
select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute"
click_button "Execute script"
expect(page).to have_content "The last script has been executed successfully."
expect(page).to have_content "You will receive an email in #{admin.email} when the script "\
"finishes running."
expect(page).to have_field "Select python script to execute"
expect(page).to have_button "Execute script"
end
scenario "Settings" do
visit admin_machine_learning_path
within "#machine_learning_tabs" do
click_link "Settings / Generated content"
end
expect(page).to have_content "Related content"
expect(page).to have_content "Adds automatically generated related content to proposals and "\
"participatory budget projects"
expect(page).to have_content "Comments summary"
expect(page).to have_content "Displays an automatically generated comment summary on all items that "\
"can be commented on."
expect(page).to have_content "Tags"
expect(page).to have_content "Generates automatic tags on all items that can be tagged on."
expect(page).to have_content "No content generated yet", count: 3
expect(page).not_to have_button "Yes"
expect(page).not_to have_button "No"
end
scenario "Script started but not finished yet" do
allow_any_instance_of(MachineLearning).to receive(:run)
visit admin_machine_learning_path
select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute"
click_button "Execute script"
expect(page).to have_content "The script is running. The administrator who executed it will receive "\
"an email when it is finished."
expect(page).to have_content "Executed by: #{admin.name}"
expect(page).to have_content "Script name: proposals_related_content_and_tags_nmf.py"
expect(page).to have_content "Started at:"
expect(page).not_to have_content "Select python script to execute"
expect(page).not_to have_button "Execute script"
end
scenario "Admin can cancel operation if script is working for too long" do
allow_any_instance_of(MachineLearning).to receive(:run) do
MachineLearningJob.first.update!(started_at: 25.hours.ago)
end
visit admin_machine_learning_path
select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute"
click_button "Execute script"
accept_confirm { click_button "Cancel operation" }
expect(page).to have_content "Generated content has been successfully deleted."
expect(page).to have_field "Select python script to execute"
expect(page).to have_button "Execute script"
end
scenario "Script finished with an error" do
allow_any_instance_of(MachineLearning).to receive(:run) do
MachineLearningJob.first.update!(finished_at: Time.current, error: "Error description")
end
visit admin_machine_learning_path
select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute"
click_button "Execute script"
expect(page).to have_content "An error has occurred. You can see the details below."
expect(page).to have_content "Executed by: #{admin.name}"
expect(page).to have_content "Script name: proposals_related_content_and_tags_nmf.py"
expect(page).to have_content "Error: Error description"
expect(page).to have_content "You will receive an email in #{admin.email} when the script "\
"finishes running."
expect(page).to have_field "Select python script to execute"
expect(page).to have_button "Execute script"
end
scenario "Email content received by the user who execute the script" do
reset_mailer
Mailer.machine_learning_success(admin.user).deliver
email = open_last_email
expect(email).to have_subject "Machine Learning - Content has been generated successfully"
expect(email).to have_content "Machine Learning script"
expect(email).to have_content "Content has been generated successfully."
expect(email).to have_link "Visit Machine Learning panel"
expect(email).to deliver_to(admin.user.email)
reset_mailer
Mailer.machine_learning_error(admin.user).deliver
email = open_last_email
expect(email).to have_subject "Machine Learning - An error has occurred running the script"
expect(email).to have_content "Machine Learning script"
expect(email).to have_content "An error has occurred running the Machine Learning script."
expect(email).to have_link "Visit Machine Learning panel"
expect(email).to deliver_to(admin.user.email)
end
scenario "Machine Learning visualization settings are disabled by default" do
allow_any_instance_of(MachineLearning).to receive(:run) do
MachineLearningJob.first.update!(finished_at: Time.current)
end
visit admin_machine_learning_path
select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute"
click_button "Execute script"
expect(page).to have_content "The last script has been executed successfully."
within "#machine_learning_tabs" do
click_link "Settings / Generated content"
end
expect(page).to have_content "No content generated yet", count: 3
expect(page).not_to have_button "Yes"
expect(page).not_to have_button "No"
end
scenario "Show script descriptions" do
visit admin_machine_learning_path
select "proposals_summary_comments_textrank.py", from: "Select python script to execute"
within "#script_descriptions" do
expect(page).to have_content "Proposals comments summaries - Dummy script"
end
select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute"
within "#script_descriptions" do
expect(page).to have_content "Related Proposals and Tags - Dummy script"
expect(page).not_to have_content "Proposals comments summaries - Dummy script"
end
end
scenario "Show output files info on settins page" do
require "fileutils"
FileUtils.mkdir_p Rails.root.join("public", "machine_learning", "data")
allow_any_instance_of(MachineLearning).to receive(:run) do
MachineLearningJob.first.update!(finished_at: Time.current + 2.minutes)
create(:machine_learning_info,
script: "proposals_summary_comments_textrank.py",
kind: "comments_summary",
updated_at: Time.current + 2.minutes)
comments_file = MachineLearning::DATA_FOLDER.join(MachineLearning.comments_filename)
File.open(comments_file, "w") { |file| file.write([].to_json) }
proposals_comments_summary_file = MachineLearning::DATA_FOLDER.join(MachineLearning.proposals_comments_summary_filename)
File.open(proposals_comments_summary_file, "w") { |file| file.write([].to_json) }
end
visit admin_machine_learning_path
within "#machine_learning_tabs" do
click_link "Settings / Generated content"
end
expect(page).to have_content "No content generated yet.", count: 3
within "#machine_learning_tabs" do
click_link "Execute script"
end
travel_to(Time.zone.local(2019, 10, 20, 14, 57, 25)) do
select "proposals_related_content_and_tags_nmf.py", from: "Select python script to execute"
click_button "Execute script"
expect(page).to have_content "The last script has been executed successfully."
end
within "#machine_learning_tabs" do
click_link "Settings / Generated content"
end
expect(page).to have_content "Last execution\n2019-10-20 14:57 - 2019-10-20 14:59"
expect(page).to have_content "Executed script:"
expect(page).to have_content "proposals_summary_comments_textrank.py"
expect(page).to have_content "Output files:"
expect(page).to have_link "ml_comments_summaries_proposals.json",
href: "/machine_learning/data/ml_comments_summaries_proposals.json"
end
end

75
spec/system/machine_learning_spec.rb Normal file
View File

@@ -0,0 +1,75 @@
require "rails_helper"
describe "Machine learning" do
let(:user_tag) { create(:tag, name: "user tag") }
let(:ml_proposal_tag) { create(:tag, name: "machine learning proposal tag") }
let(:ml_investment_tag) { create(:tag, name: "machine learning investment tag") }
let(:proposal) { create(:proposal) }
let(:related_proposal) { create(:proposal) }
let(:investment) { create(:budget_investment) }
let(:related_investment) { create(:budget_investment) }
before do
Setting["feature.machine_learning"] = true
Setting["machine_learning.comments_summary"] = true
Setting["machine_learning.related_content"] = true
Setting["machine_learning.tags"] = true
proposal.update!(tag_list: [user_tag])
proposal.update!(ml_tag_list: [ml_proposal_tag])
investment.update!(tag_list: [user_tag])
investment.update!(ml_tag_list: [ml_investment_tag])
end
scenario "proposal view" do
create(:ml_summary_comment, commentable: proposal, body: "Life is wonderful")
create(:related_content, parent_relationable: proposal,
child_relationable: related_proposal,
machine_learning: true)
visit proposal_path(proposal)
within "#tags_proposal_#{proposal.id}" do
expect(page).not_to have_link "user tag"
expect(page).to have_link "machine learning proposal tag"
expect(page).not_to have_link "machine learning investment tag"
end
within ".related-content" do
expect(page).to have_content "Related content (1)"
expect(page).to have_css ".related-content-title"
expect(page).to have_content related_proposal.title
end
within "#comments" do
expect(page).to have_content "Comments summary"
expect(page).to have_content "Life is wonderful"
end
end
scenario "investment view" do
create(:ml_summary_comment, commentable: investment, body: "Build in the main square")
create(:related_content, parent_relationable: investment,
child_relationable: related_investment,
machine_learning: true)
visit budget_investment_path(investment.budget, investment)
within "#tags_budget_investment_#{investment.id}" do
expect(page).not_to have_link "user tag"
expect(page).not_to have_link "machine learning proposal tag"
expect(page).to have_link "machine learning investment tag"
end
within ".related-content" do
expect(page).to have_content "Related content (1)"
expect(page).to have_css ".related-content-title", count: 1
expect(page).to have_content related_investment.title
end
within "#tab-comments" do
expect(page).to have_content "Comments summary"
expect(page).to have_content "Build in the main square"
end
end
end