From 48124da4ffb2ce75afc7b113f3ab900143eceb47 Mon Sep 17 00:00:00 2001 From: Maxim Colls Date: Fri, 4 Oct 2024 11:58:11 +0200 Subject: [PATCH 1/9] Add verification data export page in stats module --- config/routes.rb | 1 + .../stats/authorization_exports_controller.rb | 38 ++++++++++++++++++ .../stats/authorization_exports_form.rb | 17 ++++++++ .../stats/authorization_exports_job.rb | 39 +++++++++++++++++++ .../decidim/stats/authorization_serializer.rb | 31 +++++++++++++++ .../authorization_exports/index.html.erb | 39 +++++++++++++++++++ decidim-stats/config/locales/ca.yml | 10 +++++ decidim-stats/config/locales/en.yml | 10 +++++ decidim-stats/config/locales/es.yml | 10 +++++ decidim-stats/lib/decidim/stats/engine.rb | 13 +++++++ 10 files changed, 208 insertions(+) create mode 100644 decidim-stats/app/controllers/decidim/stats/authorization_exports_controller.rb create mode 100644 decidim-stats/app/form/decidim/stats/authorization_exports_form.rb create mode 100644 decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb create mode 100644 decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb create mode 100644 decidim-stats/app/views/decidim/stats/authorization_exports/index.html.erb create mode 100644 decidim-stats/config/locales/ca.yml create mode 100644 decidim-stats/config/locales/en.yml create mode 100644 decidim-stats/config/locales/es.yml diff --git a/config/routes.rb b/config/routes.rb index 7427c3e8c1..ba864d5d27 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -51,6 +51,7 @@ end mount Decidim::Core::Engine => "/" + mount Decidim::Stats::Engine, at: "/stats", as: "decidim_stats" mount Decidim::EphemeralParticipation::Engine, at: "/", as: "decidim_ephemeral_participation" mount LetterOpenerWeb::Engine, at: "/letter_opener" if Rails.env.development? authenticate :user, ->(u) { u.admin? } do diff --git a/decidim-stats/app/controllers/decidim/stats/authorization_exports_controller.rb b/decidim-stats/app/controllers/decidim/stats/authorization_exports_controller.rb new file mode 100644 index 0000000000..d643c1f7cd --- /dev/null +++ b/decidim-stats/app/controllers/decidim/stats/authorization_exports_controller.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Decidim + module Stats + class AuthorizationExportsController < Decidim::Admin::ApplicationController + layout "decidim/admin/users" + + def index + enforce_permission_to :index, :authorization_workflow + + @workflows = Decidim::Verifications.workflows.select do |manifest| + current_organization.available_authorizations.include?(manifest.name.to_s) + end + + @form = form(AuthorizationExportsForm).instance + end + + def create + AuthorizationExportsJob.perform_later( + current_user, + authorization_params[:authorization_handler_name], + authorization_params[:start_date], + authorization_params[:end_date] + ) + + flash[:notice] = t("decidim.admin.exports.notice") + + redirect_to authorization_exports_path + end + + private + + def authorization_params + params.require(:authorization_exports).permit(:authorization_handler_name, :start_date, :end_date) + end + end + end +end diff --git a/decidim-stats/app/form/decidim/stats/authorization_exports_form.rb b/decidim-stats/app/form/decidim/stats/authorization_exports_form.rb new file mode 100644 index 0000000000..22aa5ead5e --- /dev/null +++ b/decidim-stats/app/form/decidim/stats/authorization_exports_form.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Decidim + module Stats + # A form to validate the date range for authorization exports + class AuthorizationExportsForm < Form + include TranslatableAttributes + + attribute :start_date, Decidim::Attributes::LocalizedDate + attribute :end_date, Decidim::Attributes::LocalizedDate + attribute :authorization_handler_name, String + + validates :start_date, presence: true + validates :end_date, presence: true + end + end +end diff --git a/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb b/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb new file mode 100644 index 0000000000..bf0c1c2e17 --- /dev/null +++ b/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Decidim + module Stats + class AuthorizationExportsJob < ApplicationJob + queue_as :default + + def perform(user, authorization_handler_name, start_date, end_date) + ExportMailer.export( + user, + export_file_name, + export_data(authorization_handler_name, start_date, end_date) + ).deliver_now + end + + def export_data(authorization_handler_name, start_date, end_date) + Decidim::Exporters::CSV.new( + collection(authorization_handler_name, start_date, end_date), + serializer + ).export + end + + def export_file_name + "authorizations_export" + end + + def collection(authorization_handler_name, start_date, end_date) + Decidim::Authorization.where( + granted_at: start_date..end_date, + name: authorization_handler_name + ) + end + + def serializer + Decidim::Stats::AuthorizationSerializer + end + end + end +end diff --git a/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb b/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb new file mode 100644 index 0000000000..227a80f4c7 --- /dev/null +++ b/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Decidim + module Stats + # This class serializes a Authorization so can be exported to CSV + class AuthorizationSerializer < Decidim::Exporters::Serializer + attr_reader :authorization + + def initialize(authorization) + @authorization = authorization + end + + def serialize + { + id: authorization.id, + name: authorization.name, + granted_at: authorization.granted_at, + postal_code: metadata["postal_code"], + date_of_birth: metadata["date_of_birth"], + gender: metadata["gender"] + } + end + + private + + def metadata + authorization.metadata || {} + end + end + end +end diff --git a/decidim-stats/app/views/decidim/stats/authorization_exports/index.html.erb b/decidim-stats/app/views/decidim/stats/authorization_exports/index.html.erb new file mode 100644 index 0000000000..17ed76d9a0 --- /dev/null +++ b/decidim-stats/app/views/decidim/stats/authorization_exports/index.html.erb @@ -0,0 +1,39 @@ +<% add_decidim_page_title(t("title", scope: "decidim.stats.authorization_exports")) %> + +
+

+ <%= t("title", scope: "decidim.stats.authorization_exports") %> +

+
+ +
+
+
+
+
+
+ <%= decidim_form_for @form, url: authorization_exports_path, multipart: true, html: { class: "form form-defaults" } do |form| %> +
+ <%= form.date_field :start_date, label: t("start_date", scope: "decidim.stats.authorization_exports") %> +
+ +
+ <%= form.date_field :end_date, label: t("end_date", scope: "decidim.stats.authorization_exports") %> +
+ +
+ <%= form.select :authorization_handler_name, @workflows.map { |workflow| [workflow.fullname, workflow.name] }, label: t("authorization_handler_name", scope: "decidim.stats.authorization_exports") %> +
+ +
+
+ <%= submit_tag t("submit", scope: "decidim.stats.authorization_exports"), class: "button button__sm button__secondary" %> +
+
+ <% end %> +
+
+
+
+
+
diff --git a/decidim-stats/config/locales/ca.yml b/decidim-stats/config/locales/ca.yml new file mode 100644 index 0000000000..5ec6b5fbed --- /dev/null +++ b/decidim-stats/config/locales/ca.yml @@ -0,0 +1,10 @@ +ca: + decidim: + stats: + authorization_exports: + title: Export verification data + authorization_handler_name: Verification method + end_date: End date + start_date: Start date + submit: Export verification data + menu: Export data diff --git a/decidim-stats/config/locales/en.yml b/decidim-stats/config/locales/en.yml new file mode 100644 index 0000000000..904335942c --- /dev/null +++ b/decidim-stats/config/locales/en.yml @@ -0,0 +1,10 @@ +en: + decidim: + stats: + authorization_exports: + title: Export verification data + authorization_handler_name: Verification method + end_date: End date + start_date: Start date + submit: Export verification data + menu: Export data diff --git a/decidim-stats/config/locales/es.yml b/decidim-stats/config/locales/es.yml new file mode 100644 index 0000000000..222f6e2de6 --- /dev/null +++ b/decidim-stats/config/locales/es.yml @@ -0,0 +1,10 @@ +es: + decidim: + stats: + authorization_exports: + title: Export verification data + authorization_handler_name: Verification method + end_date: End date + start_date: Start date + submit: Export verification data + menu: Export data diff --git a/decidim-stats/lib/decidim/stats/engine.rb b/decidim-stats/lib/decidim/stats/engine.rb index ddd3554c74..398d9a1cc2 100644 --- a/decidim-stats/lib/decidim/stats/engine.rb +++ b/decidim-stats/lib/decidim/stats/engine.rb @@ -4,6 +4,19 @@ module Decidim module Stats class Engine < ::Rails::Engine isolate_namespace Decidim::Stats + + routes do + resources :authorization_exports, only: [:index, :create] + end + + initializer "decidim_stats.admin_menus" do + Decidim.menu :admin_user_menu do |menu| + menu.add_item :authorization_exports, + I18n.t("authorization_exports.menu", scope: "decidim.stats"), + decidim_stats.authorization_exports_path, + active: is_active_link?(decidim_stats.authorization_exports_path) + end + end end end end From 537c6ad3e9640d00a85364f81462d7117878bcba Mon Sep 17 00:00:00 2001 From: Maxim Colls Date: Fri, 4 Oct 2024 13:06:12 +0200 Subject: [PATCH 2/9] Add filtering by current organization --- .../stats/authorization_exports_controller.rb | 7 ++--- .../stats/authorization_exports_job.rb | 26 ++++++++++++------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/decidim-stats/app/controllers/decidim/stats/authorization_exports_controller.rb b/decidim-stats/app/controllers/decidim/stats/authorization_exports_controller.rb index d643c1f7cd..49d05ebda4 100644 --- a/decidim-stats/app/controllers/decidim/stats/authorization_exports_controller.rb +++ b/decidim-stats/app/controllers/decidim/stats/authorization_exports_controller.rb @@ -18,9 +18,10 @@ def index def create AuthorizationExportsJob.perform_later( current_user, - authorization_params[:authorization_handler_name], - authorization_params[:start_date], - authorization_params[:end_date] + current_organization, + name: authorization_params[:authorization_handler_name], + start_date: authorization_params[:start_date], + end_date: authorization_params[:end_date] ) flash[:notice] = t("decidim.admin.exports.notice") diff --git a/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb b/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb index bf0c1c2e17..ea8c232bae 100644 --- a/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb +++ b/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb @@ -5,17 +5,18 @@ module Stats class AuthorizationExportsJob < ApplicationJob queue_as :default - def perform(user, authorization_handler_name, start_date, end_date) + def perform(user, organization, filters) + byebug ExportMailer.export( user, export_file_name, - export_data(authorization_handler_name, start_date, end_date) + export_data(organization, filters) ).deliver_now end - def export_data(authorization_handler_name, start_date, end_date) + def export_data(organization, filters) Decidim::Exporters::CSV.new( - collection(authorization_handler_name, start_date, end_date), + collection(organization, filters), serializer ).export end @@ -24,11 +25,18 @@ def export_file_name "authorizations_export" end - def collection(authorization_handler_name, start_date, end_date) - Decidim::Authorization.where( - granted_at: start_date..end_date, - name: authorization_handler_name - ) + def collection(organization, filters) + Decidim::Authorization.joins(:user) + .where( + granted_at: filters[:start_date]..filters[:end_date], + name: filters[:name] + ) + Decidim::Authorization.joins(:user) + .where(decidim_users: { decidim_organization_id: organization.id }) + .where( + granted_at: filters[:start_date]..filters[:end_date], + name: filters[:name] + ) end def serializer From 440d49c2c52c873eae5e7a5ee6db4a4a326792f5 Mon Sep 17 00:00:00 2001 From: Maxim Colls Date: Mon, 7 Oct 2024 12:05:46 +0200 Subject: [PATCH 3/9] Remove byebug --- .../app/jobs/decidim/stats/authorization_exports_job.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb b/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb index ea8c232bae..996e1dab23 100644 --- a/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb +++ b/decidim-stats/app/jobs/decidim/stats/authorization_exports_job.rb @@ -6,7 +6,6 @@ class AuthorizationExportsJob < ApplicationJob queue_as :default def perform(user, organization, filters) - byebug ExportMailer.export( user, export_file_name, From b0d032d9874990c296bce02f841554ece9028ddd Mon Sep 17 00:00:00 2001 From: Maxim Colls Date: Tue, 8 Oct 2024 18:39:14 +0200 Subject: [PATCH 4/9] Fix export fields --- .../decidim/stats/authorization_serializer.rb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb b/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb index 227a80f4c7..d932ba1780 100644 --- a/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb +++ b/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb @@ -13,11 +13,13 @@ def initialize(authorization) def serialize { id: authorization.id, - name: authorization.name, - granted_at: authorization.granted_at, - postal_code: metadata["postal_code"], date_of_birth: metadata["date_of_birth"], - gender: metadata["gender"] + postal_code: metadata["postal_code"], + scope_name: metadata["scope_name"], + scope_id: metadata["scope_id"], + scope_code: metadata["scope_code"], + gender: metadata_extras["gender"], + granted_at: authorization.granted_at } end @@ -26,6 +28,10 @@ def serialize def metadata authorization.metadata || {} end + + def metadata_extras + metadata["extras"] || {} + end end end end From 350facd62b5a13f48365e471883d602a138213f9 Mon Sep 17 00:00:00 2001 From: Maxim Colls Date: Thu, 10 Oct 2024 19:14:59 +0200 Subject: [PATCH 5/9] Translate into ca and es --- decidim-stats/config/locales/ca.yml | 12 ++++++------ decidim-stats/config/locales/es.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/decidim-stats/config/locales/ca.yml b/decidim-stats/config/locales/ca.yml index 5ec6b5fbed..193d64cd8d 100644 --- a/decidim-stats/config/locales/ca.yml +++ b/decidim-stats/config/locales/ca.yml @@ -2,9 +2,9 @@ ca: decidim: stats: authorization_exports: - title: Export verification data - authorization_handler_name: Verification method - end_date: End date - start_date: Start date - submit: Export verification data - menu: Export data + title: Exportar dades de verificació + authorization_handler_name: Mètode de verificació + end_date: Data fi + start_date: Data inici + submit: Exportar dades + menu: Exportar dades diff --git a/decidim-stats/config/locales/es.yml b/decidim-stats/config/locales/es.yml index 222f6e2de6..6381d8ffde 100644 --- a/decidim-stats/config/locales/es.yml +++ b/decidim-stats/config/locales/es.yml @@ -2,9 +2,9 @@ es: decidim: stats: authorization_exports: - title: Export verification data - authorization_handler_name: Verification method - end_date: End date - start_date: Start date - submit: Export verification data - menu: Export data + title: Exportar datos de verificación + authorization_handler_name: Método de verificación + end_date: Fecha fin + start_date: Fecha inicio + submit: Exportar datos + menu: Exportar datos From 778569cd70ec6b85b8b546e08efe027eb43492bd Mon Sep 17 00:00:00 2001 From: Maxim Colls Date: Wed, 16 Oct 2024 10:15:47 +0200 Subject: [PATCH 6/9] Add icon to export data menu item --- decidim-stats/lib/decidim/stats/engine.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/decidim-stats/lib/decidim/stats/engine.rb b/decidim-stats/lib/decidim/stats/engine.rb index 398d9a1cc2..282cd70c25 100644 --- a/decidim-stats/lib/decidim/stats/engine.rb +++ b/decidim-stats/lib/decidim/stats/engine.rb @@ -14,7 +14,8 @@ class Engine < ::Rails::Engine menu.add_item :authorization_exports, I18n.t("authorization_exports.menu", scope: "decidim.stats"), decidim_stats.authorization_exports_path, - active: is_active_link?(decidim_stats.authorization_exports_path) + active: is_active_link?(decidim_stats.authorization_exports_path), + icon_name: "download-line" end end end From 3c8116ba277689c3be6ae1d6cb9e4ac73a07262d Mon Sep 17 00:00:00 2001 From: Maxim Colls Date: Wed, 16 Oct 2024 10:22:51 +0200 Subject: [PATCH 7/9] Add user hash column in authorization serializer --- .../decidim/stats/authorization_serializer.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb b/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb index d932ba1780..7e23cf5608 100644 --- a/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb +++ b/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb @@ -12,7 +12,7 @@ def initialize(authorization) def serialize { - id: authorization.id, + user_hash:, date_of_birth: metadata["date_of_birth"], postal_code: metadata["postal_code"], scope_name: metadata["scope_name"], @@ -32,6 +32,16 @@ def metadata def metadata_extras metadata["extras"] || {} end + + def user_hash + return "" unless user + + Digest::SHA256.hexdigest(user.id.to_s).last(8) + end + + def user + authorization.user + end end end end From 0f5ccb88a2855c54645592a0cda3b1085e388f63 Mon Sep 17 00:00:00 2001 From: Maxim Colls Date: Wed, 6 Nov 2024 10:12:40 +0100 Subject: [PATCH 8/9] Fix scope name serialization key --- .../app/serializers/decidim/stats/authorization_serializer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb b/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb index 7e23cf5608..62121834ab 100644 --- a/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb +++ b/decidim-stats/app/serializers/decidim/stats/authorization_serializer.rb @@ -15,7 +15,7 @@ def serialize user_hash:, date_of_birth: metadata["date_of_birth"], postal_code: metadata["postal_code"], - scope_name: metadata["scope_name"], + scope_name: metadata["scope"], scope_id: metadata["scope_id"], scope_code: metadata["scope_code"], gender: metadata_extras["gender"], From e086908cb447203db25233a6cf50d6b06b6a7295 Mon Sep 17 00:00:00 2001 From: Maxim Colls Date: Fri, 8 Nov 2024 15:42:20 +0100 Subject: [PATCH 9/9] Add documentation to Readme --- decidim-stats/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/decidim-stats/README.md b/decidim-stats/README.md index 76bb7af3fc..225edaeef1 100644 --- a/decidim-stats/README.md +++ b/decidim-stats/README.md @@ -38,6 +38,13 @@ participatory_space_type,participatory_space_id,component_id,action,metric_type, participatory_processes,2,4,comment,age_group,20-24,6 ``` +## Authorization data export page + +This module adds a new admin page, “Export verification data” accessible under the Admin > Participants menu as “Export Data.” + +On this page, admins can fill out a form to specify a Start Date, End Date, and Verification Method. Using these parameters, the form allows the export of census data gathered during the user verification process. Internally, it exports selected metadata fields from the Authorization model. + + ## Development guide The module has different pieces: