From db5029f2f827916dcf27d489579211f759fe4d1d Mon Sep 17 00:00:00 2001 From: louispt1 Date: Tue, 15 Oct 2024 09:54:50 +0200 Subject: [PATCH 1/2] Download inputs as a csv with min, max, default, unit and user value --- app/controllers/api/v3/inputs_controller.rb | 69 ++++- config/routes.rb | 2 +- .../api/v3/inputs_controller_spec.rb | 277 +++++++++++++----- 3 files changed, 269 insertions(+), 79 deletions(-) diff --git a/app/controllers/api/v3/inputs_controller.rb b/app/controllers/api/v3/inputs_controller.rb index fb7b81ac4..a30d5e177 100644 --- a/app/controllers/api/v3/inputs_controller.rb +++ b/app/controllers/api/v3/inputs_controller.rb @@ -1,27 +1,31 @@ # frozen_string_literal: true +require 'csv' module Api module V3 class InputsController < ::Api::V3::BaseController before_action do @scenario = Scenario.find(params[:scenario_id]) - authorize!(:read, @scenario) + if user_signed_in? && current_user.can?(:read, @scenario) + authorize!(:read, @scenario) + end end # GET /api/v3/inputs # GET /api/v3/scenarios/:scenario_id/inputs + # GET /api/v3/scenarios/:scenario_id/inputs.csv # - # Returns the details for all available inputs. If the scenario_id isn't - # passed then the action will use the latest scenario. - # + # Returns input details in JSON or CSV format. Uses the latest scenario if + # scenario_id is not provided. + def index extras = ActiveModel::Type::Boolean.new.cast(params[:include_extras]) + inputs = serialized_inputs(extras) - render json: InputSerializer.collection( - Input.all, - @scenario, - **serializer_args(extra_attributes: extras) - ) + respond_to do |format| + format.json { render json: inputs } + format.csv { send_csv_data(inputs) } + end end # GET /api/v3/inputs/:id @@ -77,6 +81,53 @@ def serializer_args(extra_attributes:) extra_attributes: } end + + def serialized_inputs(extras) + InputSerializer.collection( + Input.all, + @scenario, + **serializer_args(extra_attributes: extras) + ) + end + + def send_csv_data(inputs) + csv_data = generate_csv(inputs) + send_data csv_data, filename: "inputs_#{@scenario.id}.csv" + end + + def generate_csv(inputs) + CSV.generate(headers: true) do |csv| + csv << csv_headers + cached_values = Input.cache(@scenario.parent) + user_values = @scenario.user_values + + inputs.each do |key, input| + add_csv_row(csv, key, input, cached_values, user_values) + end + end + end + + def csv_headers + ["Key", "Min", "Max", "Default", "User Value", "Unit", "Share Group"] + end + + def add_csv_row(csv, key, input, cached_values, user_values) + input_data = input.instance_variable_get(:@input) + return if input_data.nil? + + values = cached_values.read(@scenario.parent, input_data) + default_value = input.instance_variable_get(:@default_values_from).call(values) + + csv << [ + key, + input_data.min_value, + input_data.max_value, + default_value, + user_values[input_data.key] || "", + input_data.unit, + input_data.share_group + ] + end end end end diff --git a/config/routes.rb b/config/routes.rb index 77039dcb3..7ef9004a4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -85,7 +85,7 @@ get 'converters', to: redirect('/api/v3/scenarios/%{scenario_id}/nodes') get 'converters/:id', to: redirect('/api/v3/scenarios/%{scenario_id}/nodes/%{id}') - resources :inputs, :only => [:index, :show] + resources :inputs, :only => [:index, :show], defaults: { format: :json} resource :version, :only => [:create, :show, :update], controller: 'scenario_version_tags' diff --git a/spec/controllers/api/v3/inputs_controller_spec.rb b/spec/controllers/api/v3/inputs_controller_spec.rb index 96ba621e1..056cd909d 100644 --- a/spec/controllers/api/v3/inputs_controller_spec.rb +++ b/spec/controllers/api/v3/inputs_controller_spec.rb @@ -21,13 +21,13 @@ before do NastyCache.instance.expire! - allow(Input).to receive(:all).and_return([ static_input, gql_input ]) + allow(Input).to receive(:all).and_return([static_input, gql_input]) end # -------------------------------------------------------------------------- - describe 'GET /api/v3/scenarios/:scenario_id/inputs' do - let(:json) { JSON.parse(get(:index, params: { scenario_id: scenario.id }).body) } + describe 'GET /api/v3/scenarios/:scenario_id/inputs (JSON format)' do + let(:json) { JSON.parse(get(:index, params: { scenario_id: scenario.id, format: :json }).body) } it 'is successful' do json @@ -174,7 +174,8 @@ end let(:json) do - JSON.parse(get(:index, params: { scenario_id: scenario.id, defaults: 'parent' }).body) + response = get(:index, params: { scenario_id: scenario.id, defaults: 'parent', format: :json }) + JSON.parse(response.body) end it 'has a "default" attribute for each input based on the parent' do @@ -192,7 +193,7 @@ end let(:json) do - JSON.parse(get(:index, params: { scenario_id: scenario.id, defaults: 'original' }).body) + JSON.parse(get(:index, params: { scenario_id: scenario.id, defaults: 'original', format: :json }).body) end it 'has a "default" attribute for each input based on the dataset' do @@ -211,7 +212,7 @@ gql_input.key => gql_input }) - get(:show, params: { scenario_id: scenario.id, id: static_input.key }) + get(:show, params: { scenario_id: scenario.id, id: static_input.key, format: :json }) JSON.parse(response.body) end @@ -231,77 +232,215 @@ expect(json['default']).to eql(10) end - context '"disabled" attribute' do - it 'is present when an input is disabled' do - expect(static_input).to receive(:disabled_in_current_area?) { true } - expect(json['disabled']).to be_truthy - end - - it 'is false when an input is not disabled' do - expect(static_input).to receive(:disabled_in_current_area?) { false } - expect(json).to include('disabled' => false) - end - end # "disabled" attribute - - context '"label" attribute' do - it 'is present when an input has a label' do - static_input.label_query = 'present:2.0 * 16' - static_input.label = 'g' - - expect(json['label']).to eql('value' => 32.0, 'suffix' => 'g') - end - - it 'is present when an input is not disabled' do - static_input.label_query = nil - expect(json).not_to have_key('label') - end - end # "label" attribute - - context '"user" attribute' do - it 'is present when an input has a user value' do - scenario.update(user_values: { static_input.key => 42.0 }) - expect(json['user']).to eql(42.0) - end - - it 'is not present when an input does not have a user value' do - expect(json).not_to have_key('user') - end - end # "user" attribute - end # GET /api/v3/scenarios/:scenario_id/inputs + context '"disabled" attribute' do + it 'is present when an input is disabled' do + expect(static_input).to receive(:disabled_in_current_area?) { true } + expect(json['disabled']).to be_truthy + end - # --------------------------------------------------------------------------- + it 'is false when an input is not disabled' do + expect(static_input).to receive(:disabled_in_current_area?) { false } + expect(json).to include('disabled' => false) + end + end # "disabled" attribute - describe 'GET /api/v3/scenarios/:scenario_id/inputs/:id,:id,...' do - let(:third_input) { FactoryBot.build(:input) } + context '"label" attribute' do + it 'is present when an input has a label' do + static_input.label_query = 'present:2.0 * 16' + static_input.label = 'g' - let(:json) do - allow(Input).to receive(:records).and_return({ - static_input.key => static_input, - gql_input.key => gql_input, - third_input.key => third_input - }) + expect(json['label']).to eql('value' => 32.0, 'suffix' => 'g') + end - allow(Input).to receive(:all).and_return(Input.records.values) + it 'is present when an input is not disabled' do + static_input.label_query = nil + expect(json).not_to have_key('label') + end + end # "label" attribute - keys = "#{ static_input.key },#{ third_input.key }" - get(:show, params: { scenario_id: scenario.id, id: keys }) - JSON.parse(response.body) - end + context '"user" attribute' do + it 'is present when an input has a user value' do + scenario.update(user_values: { static_input.key => 42.0 }) + expect(json['user']).to eql(42.0) + end - it 'returns an array' do - expect(json).to be_kind_of(Array) - end + it 'is not present when an input does not have a user value' do + expect(json).not_to have_key('user') + end + end # "user" attribute + end # GET /api/v3/scenarios/:scenario_id/inputs - it 'includes the requested inputs' do - expect(json.size).to eq(2) + # --------------------------------------------------------------------------- - expect(json.any? { |v| v['code'] == static_input.key }).to be_truthy - expect(json.any? { |v| v['code'] == third_input.key }).to be_truthy - end + describe 'GET /api/v3/scenarios/:scenario_id/inputs/:id,:id,...' do + let(:third_input) { FactoryBot.build(:input) } - it 'does not include unrequested inputs' do - expect(json.any? { |v| v['code'] == gql_input.key }).to be_falsey - end - end # GET /api/v3/scenarios/:scenario_id/inputs + let(:json) do + allow(Input).to receive(:records).and_return({ + static_input.key => static_input, + gql_input.key => gql_input, + third_input.key => third_input + }) + + allow(Input).to receive(:all).and_return(Input.records.values) + keys = "#{ static_input.key },#{ third_input.key }" + get(:show, params: { scenario_id: scenario.id, id: keys }) + JSON.parse(response.body) + end + + it 'returns an array' do + expect(json).to be_kind_of(Array) + end + + it 'includes the requested inputs' do + expect(json.size).to eq(2) + + expect(json.any? { |v| v['code'] == static_input.key }).to be_truthy + expect(json.any? { |v| v['code'] == third_input.key }).to be_truthy + end + + it 'does not include unrequested inputs' do + expect(json.any? { |v| v['code'] == gql_input.key }).to be_falsey + end + end # GET /api/v3/scenarios/:scenario_id/inputs + + describe 'GET /api/v3/scenarios/:scenario_id/inputs.csv' do + let(:csv) { JSON.parse(get(:index, params: { scenario_id: scenario.id, format: :json }).body) } + + it 'is successful' do + csv + expect(response).to be_ok + end + + it 'should contain each input' do + expect(csv).to have_key(static_input.key) + expect(csv).to have_key(gql_input.key) + end + + it 'does not have a "code" attribute for each input' do + expect(csv[static_input.key]).not_to have_key('code') + expect(csv[gql_input.key]).not_to have_key('code') + end + + it 'should have a "min" attribute for each input' do + expect(csv[static_input.key]).to include('min' => 5) + expect(csv[gql_input.key]).to include('min' => 4) + end + + it 'should have a "max" attribute for each input' do + expect(csv[static_input.key]).to include('max' => 15) + expect(csv[gql_input.key]).to include('max' => 16) + end + + it 'should have a "default" attribute for each input' do + expect(csv[static_input.key]).to include('default' => 10) + expect(csv[gql_input.key]).to include('default' => 8) + end + + context '"disabled" attribute' do + before do + expect(static_input).to receive(:disabled_in_current_area?) { true } + expect(gql_input).to receive(:disabled_in_current_area?) { false } + end + + it 'should be true when an input is disabled' do + expect(csv[static_input.key]).to include('disabled' => true) + end + + it 'should be false when an input is not disabled' do + expect(csv[gql_input.key]).to include('disabled' => false) + end + end + + context '"label" attribute' do + before do + static_input.label_query = 'present:2 * 16' + static_input.label = 'g' + gql_input.label_query = nil + end + + it 'should be present when an input has a label' do + expect(csv[static_input.key]).to \ + include('label' => { 'value' => 32.0, 'suffix' => 'g'}) + end + + it 'should not be present when an input is not disabled' do + expect(csv[gql_input.key]).not_to have_key('label') + end + end # "label" attribute + + context '"user" attribute' do + before do + scenario.update(user_values: { gql_input.key => 42.0 }) + end + + it 'should be present when an input has a user value' do + expect(csv[gql_input.key]).to include('user' => 42.0) + end + + it 'should not be present when an input does not have a user value' do + expect(csv[static_input.key]).not_to have_key('user') + end + end # "user" attribute + + context 'with a scaled scenario' do + before do + ScenarioScaling.create( + scenario: scenario, + area_attribute: 'present_number_of_residences', + value: 1_000_000) + end + + let(:divisor) do + Atlas::Dataset.find(:nl).present_number_of_residences / 1_000_000 + end + + it 'scales static input values' do + expect(csv[static_input.key]).to include( + 'min' => 5 / divisor, + 'max' => 15 / divisor, + 'default' => 10 / divisor + ) + end + + it 'does not scale GQL-based input values' do + # GQL inputs are not scaled, since they use the local graph to compute + # their values. + expect(csv[gql_input.key]).to include( + 'min' => 4, + 'max' => 16, + 'default' => 8 + ) + end + end # with a scaled scenario + + context 'with an enum input' do + let(:gql_input) do + FactoryBot.build(:input, { + start_value_gql: 'present:1 + 1', + unit: 'enum', + min_value_gql: 'present:[1, 2, 3]' + }) + end + + it 'omits the min value' do + expect(csv[gql_input.key]).not_to have_key('min') + end + + it 'omits the max value' do + expect(csv[gql_input.key]).not_to have_key('max') + end + + it 'omits the step value' do + expect(csv[gql_input.key]).not_to have_key('step') + end + + it 'includes the permitted values' do + expect(csv[gql_input.key]).to include( + 'permitted_values' => %w[1 2 3] + ) + end + end + end end From 452ccc9523f27adab2d2fb5b83b8986655c29a62 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Mon, 28 Oct 2024 09:04:04 +0100 Subject: [PATCH 2/2] WIP default values for empty scenarios not working --- app/controllers/api/v3/inputs_controller.rb | 20 +++++++++++--------- config/routes.rb | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/v3/inputs_controller.rb b/app/controllers/api/v3/inputs_controller.rb index a30d5e177..6786df728 100644 --- a/app/controllers/api/v3/inputs_controller.rb +++ b/app/controllers/api/v3/inputs_controller.rb @@ -100,7 +100,6 @@ def generate_csv(inputs) csv << csv_headers cached_values = Input.cache(@scenario.parent) user_values = @scenario.user_values - inputs.each do |key, input| add_csv_row(csv, key, input, cached_values, user_values) end @@ -108,24 +107,27 @@ def generate_csv(inputs) end def csv_headers - ["Key", "Min", "Max", "Default", "User Value", "Unit", "Share Group"] + ["Key", "Unit", "Start Year Value", "Scenario Value", "Min", "Max", "Share Group"] end def add_csv_row(csv, key, input, cached_values, user_values) input_data = input.instance_variable_get(:@input) return if input_data.nil? - values = cached_values.read(@scenario.parent, input_data) - default_value = input.instance_variable_get(:@default_values_from).call(values) + puts(input.data.inspect) + + # Provide a safe fallback for missing values + values = cached_values&.read(@scenario.parent, input_data) || {} + default_value = input.instance_variable_get(:@default_values_from)&.call(values) || "" csv << [ key, - input_data.min_value, - input_data.max_value, - default_value, + input_data.unit || "", + default_value || "", user_values[input_data.key] || "", - input_data.unit, - input_data.share_group + input_data.min_value || "", + input_data.max_value || "", + input_data.share_group || "" ] end end diff --git a/config/routes.rb b/config/routes.rb index 7ef9004a4..a6c6e6ce7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -85,7 +85,7 @@ get 'converters', to: redirect('/api/v3/scenarios/%{scenario_id}/nodes') get 'converters/:id', to: redirect('/api/v3/scenarios/%{scenario_id}/nodes/%{id}') - resources :inputs, :only => [:index, :show], defaults: { format: :json} + resources :inputs, only: [:index, :show], defaults: { format: :json }, constraints: { format: /(json|csv)/ } resource :version, :only => [:create, :show, :update], controller: 'scenario_version_tags'