Merge pull request #560 from quintel/eu-datasets-building-stock

Update building stock data and technology split heat pumps of EU datasets

db/migrate/20241106112225_eu_datasets_building_stock_update.rb

@@ -0,0 +1,65 @@
+class EuDatasetsBuildingStockUpdate < ActiveRecord::Migration[7.0]
+  def self.up
+    directory    = Rails.root.join('db/migrate/20241106112225_eu_datasets_building_stock_update')
+    data_path    = directory.join('data.csv')
+    commits_path = directory.join('commits.yml')
+    datasets     = []
+
+    # By default, CSVImporter only allows updating existing datasets. If the
+    # migration is adding a new dataset, add the `create_missing_datasets`
+    # keyword argument. For example:
+    #
+    #   CSVImporter.run(data_path, commits_path, create_missing_datasets: true) do |row, runner|
+    #     # ...
+    #   end
+    #
+    CSVImporter.run(data_path, commits_path) do |row, runner|
+      print("Updating #{row['geo_id']}... ")
+      commits = runner.call
+
+      if commits.any?
+        datasets.push(find_dataset(commits))
+        puts 'done!'
+      else
+        puts 'nothing to change!'
+      end
+    end
+
+    sleep(1)
+    puts
+    puts "Updated #{datasets.length} datasets with the following IDs:"
+    puts "  #{datasets.map(&:id).join(',')}"
+  rescue ActiveRecord::RecordInvalid => e
+    if e.record.is_a?(Commit) && e.record.errors['dataset_edits.value'].any?
+      warn('')
+      warn('-' * 80)
+      warn('The following errors occurred while processing CSV rows:')
+      warn('')
+
+      # Grab all the errors from individual datasets to show those instead. This is typically
+      # the case when a CSV specifies a value that is not allowed to be edited.
+      e.record
+        .dataset_edits
+        .reject(&:valid?)
+        .each do |edit|
+          edit.errors.each do |field, msg|
+            warn("* #{edit.commit.dataset.geo_id}: #{edit.key}: #{field} #{msg}")
+          end
+        end
+
+      warn('-' * 80)
+    end
+
+    raise e
+  end
+
+  # def self.down
+  #   raise ActiveRecord::IrreversibleMigration
+  # end
+
+  def find_dataset(commits)
+    commits.each do |commit|
+      return commit.dataset if commit&.dataset
+    end
+  end
+end

db/migrate/20241106112225_eu_datasets_building_stock_update/commits.yml

@@ -0,0 +1,65 @@
+---
+- fields:
+  - present_number_of_apartments_before_1945
+  - present_number_of_apartments_1945_1964
+  - present_number_of_apartments_1965_1984
+  - present_number_of_apartments_1985_2004
+  - present_number_of_apartments_2005_present
+  - present_number_of_detached_houses_before_1945
+  - present_number_of_detached_houses_1945_1964
+  - present_number_of_detached_houses_1965_1984
+  - present_number_of_detached_houses_1985_2004
+  - present_number_of_detached_houses_2005_present
+  - present_number_of_semi_detached_houses_before_1945
+  - present_number_of_semi_detached_houses_1945_1964
+  - present_number_of_semi_detached_houses_1965_1984
+  - present_number_of_semi_detached_houses_1985_2004
+  - present_number_of_semi_detached_houses_2005_present
+  - present_number_of_terraced_houses_before_1945
+  - present_number_of_terraced_houses_1945_1964
+  - present_number_of_terraced_houses_1965_1984
+  - present_number_of_terraced_houses_1985_2004
+  - present_number_of_terraced_houses_2005_present
+  message:
+    Orginates from the EU Building Stock Observatory, BSO database obtained from
+    <a href="https://energy.ec.europa.eu/topics/energy-efficiency/energy-efficient-buildings/eu-building-stock-observatory_en"> this link</a>.</br></br>
+    For details on transformation of this source data to the final input values, see
+    <a href="https://github.com/quintel/etdataset-public/tree/master/source_analyses/EU27_european_union/2019/eu_datasets/additional_data/Sources/Input_EU_building_stock_observatory.xlsx">this file</a>.
+- fields:
+  - typical_useful_demand_for_space_heating_semi_detached_houses_future
+  - typical_useful_demand_for_space_heating_semi_detached_houses_2005_present
+  - typical_useful_demand_for_space_heating_semi_detached_houses_1985_2004
+  - typical_useful_demand_for_space_heating_semi_detached_houses_1965_1984
+  - typical_useful_demand_for_space_heating_semi_detached_houses_1945_1964
+  - typical_useful_demand_for_space_heating_semi_detached_houses_before_1945
+  message:
+    Derived from the Hotmaps project (2018), overview of building age and energy demand of all EU countries,
+    see <a href="https://gitlab.com/hotmaps/building-stock/-/blob/master/README.md?ref_type=heads"> this link</a>. </br></br>
+    Values for semi-detached houses assumed similar as for detached houses. For details on transformation of
+    the source data to the final input values, see
+    <a href="https://github.com/quintel/etdataset-public/tree/master/source_analyses/EU27_european_union/2019/eu_datasets/additional_data/Sources/Input_building_stock_incl_semi_detached.xlsx">this file</a>.
+- fields:
+  - present_share_of_apartments_2005_present_in_useful_demand_for_space_heating
+  - present_share_of_apartments_1985_2004_in_useful_demand_for_space_heating
+  - present_share_of_apartments_1965_1984_in_useful_demand_for_space_heating
+  - present_share_of_apartments_1945_1964_in_useful_demand_for_space_heating
+  - present_share_of_apartments_before_1945_in_useful_demand_for_space_heating
+  - present_share_of_detached_houses_2005_present_in_useful_demand_for_space_heating
+  - present_share_of_detached_houses_1985_2004_in_useful_demand_for_space_heating
+  - present_share_of_detached_houses_1965_1984_in_useful_demand_for_space_heating
+  - present_share_of_detached_houses_1945_1964_in_useful_demand_for_space_heating
+  - present_share_of_detached_houses_before_1945_in_useful_demand_for_space_heating
+  - present_share_of_semi_detached_houses_2005_present_in_useful_demand_for_space_heating
+  - present_share_of_semi_detached_houses_1985_2004_in_useful_demand_for_space_heating
+  - present_share_of_semi_detached_houses_1965_1984_in_useful_demand_for_space_heating
+  - present_share_of_semi_detached_houses_1945_1964_in_useful_demand_for_space_heating
+  - present_share_of_semi_detached_houses_before_1945_in_useful_demand_for_space_heating
+  - present_share_of_terraced_houses_2005_present_in_useful_demand_for_space_heating
+  - present_share_of_terraced_houses_1985_2004_in_useful_demand_for_space_heating
+  - present_share_of_terraced_houses_1965_1984_in_useful_demand_for_space_heating
+  - present_share_of_terraced_houses_1945_1964_in_useful_demand_for_space_heating
+  - present_share_of_terraced_houses_before_1945_in_useful_demand_for_space_heating
+  message:
+    Derived from number of residences and typical heat demand per residence type and construction period.For details on transformation of
+    the source data to the final input values, see
+    <a href="https://github.com/quintel/etdataset-public/tree/master/source_analyses/EU27_european_union/2019/eu_datasets/additional_data/Sources/Input_building_stock_incl_semi_detached.xlsx">this file</a>.

db/migrate/20241106112225_eu_datasets_building_stock_update/data.csv

@@ -0,0 +1,28 @@
+geo_id,present_number_of_apartments_before_1945,present_number_of_apartments_1945_1964,present_number_of_apartments_1965_1984,present_number_of_apartments_1985_2004,present_number_of_apartments_2005_present,present_number_of_detached_houses_before_1945,present_number_of_detached_houses_1945_1964,present_number_of_detached_houses_1965_1984,present_number_of_detached_houses_1985_2004,present_number_of_detached_houses_2005_present,present_number_of_semi_detached_houses_before_1945,present_number_of_semi_detached_houses_1945_1964,present_number_of_semi_detached_houses_1965_1984,present_number_of_semi_detached_houses_1985_2004,present_number_of_semi_detached_houses_2005_present,present_number_of_terraced_houses_before_1945,present_number_of_terraced_houses_1945_1964,present_number_of_terraced_houses_1965_1984,present_number_of_terraced_houses_1985_2004,present_number_of_terraced_houses_2005_present,typical_useful_demand_for_space_heating_semi_detached_houses_future,typical_useful_demand_for_space_heating_semi_detached_houses_2005_present,typical_useful_demand_for_space_heating_semi_detached_houses_1985_2004,typical_useful_demand_for_space_heating_semi_detached_houses_1965_1984,typical_useful_demand_for_space_heating_semi_detached_houses_1945_1964,typical_useful_demand_for_space_heating_semi_detached_houses_before_1945,present_share_of_apartments_before_1945_in_useful_demand_for_space_heating,present_share_of_apartments_1945_1964_in_useful_demand_for_space_heating,present_share_of_apartments_1965_1984_in_useful_demand_for_space_heating,present_share_of_apartments_1985_2004_in_useful_demand_for_space_heating,present_share_of_apartments_2005_present_in_useful_demand_for_space_heating,present_share_of_detached_houses_before_1945_in_useful_demand_for_space_heating,present_share_of_detached_houses_1945_1964_in_useful_demand_for_space_heating,present_share_of_detached_houses_1965_1984_in_useful_demand_for_space_heating,present_share_of_detached_houses_1985_2004_in_useful_demand_for_space_heating,present_share_of_detached_houses_2005_present_in_useful_demand_for_space_heating,present_share_of_semi_detached_houses_before_1945_in_useful_demand_for_space_heating,present_share_of_semi_detached_houses_1945_1964_in_useful_demand_for_space_heating,present_share_of_semi_detached_houses_1965_1984_in_useful_demand_for_space_heating,present_share_of_semi_detached_houses_1985_2004_in_useful_demand_for_space_heating,present_share_of_semi_detached_houses_2005_present_in_useful_demand_for_space_heating,present_share_of_terraced_houses_before_1945_in_useful_demand_for_space_heating,present_share_of_terraced_houses_1945_1964_in_useful_demand_for_space_heating,present_share_of_terraced_houses_1965_1984_in_useful_demand_for_space_heating,present_share_of_terraced_houses_1985_2004_in_useful_demand_for_space_heating,present_share_of_terraced_houses_2005_present_in_useful_demand_for_space_heating
+AT,297239,226937,225686,153258,68328,280325,288397,384554,317623,67480,280325,288397,384554,317623,67480,200653,137091,131673,161827,50649,97.54,97.54,119.13,158.39,175.78,173.25,0.02895,0.02291,0.02062,0.01075,0.00399,0.11677,0.12188,0.14644,0.09097,0.01582,0.077,0.08037,0.09657,0.05999,0.01044,0.02909,0.02321,0.02019,0.01902,0.00502
+BE,20051,108528,298732,284230,178259,82037,31726,30411,83960,99982,82037,31726,30411,83960,99982,371233,450456,1058591,1083595,628918,129.31,129.31,169.15,333.05,410.9,410.9,0.00297,0.01601,0.04107,0.02495,0.01134,0.0678,0.02622,0.02037,0.02856,0.026,0.04471,0.01729,0.01343,0.01884,0.01715,0.08494,0.10306,0.22569,0.14753,0.06207
+BG,84194,85121,138101,134536,74648,280995,218162,273105,234372,119282,280995,218162,273105,234372,119282,16108,31568,90475,91659,49047,47.29,47.29,56.32,67.56,78.78,75.99,0.01233,0.01223,0.0182,0.01409,0.00594,0.15053,0.12116,0.13007,0.09306,0.03977,0.09926,0.0799,0.08577,0.06136,0.02622,0.00365,0.00703,0.01849,0.01489,0.00605
+HR,36670,37879,79971,80874,52039,95715,99741,218528,203642,106951,95715,99741,218528,203642,106951,23961,28060,60549,53756,64681,49.93,49.93,87.25,173.22,204.21,190.77,0.01191,0.01034,0.01697,0.01249,0.00462,0.09701,0.10821,0.20111,0.0944,0.02837,0.06397,0.07136,0.13262,0.06225,0.01871,0.01206,0.01188,0.01993,0.01288,0.00891
+CY,759,13579,13631,7318,11013,12197,45324,46384,32008,35937,12197,45324,46384,32008,35937,5691,8103,12126,10587,3002,56.39,56.39,56.72,57.34,57.07,56.56,0.00074,0.01216,0.01163,0.00601,0.00814,0.03863,0.14485,0.14894,0.10167,0.11348,0.02547,0.09552,0.09821,0.06704,0.07483,0.00846,0.01125,0.01604,0.01349,0.00344
+CZ,167134,201520,314918,108409,46305,348772,324807,574148,260253,145223,348772,324807,574148,260253,145223,9597,11896,77360,308356,248789,112.18,112.18,153.22,167.85,186.27,190.27,0.01456,0.01715,0.02478,0.00688,0.00267,0.12439,0.11341,0.18064,0.07475,0.03054,0.08202,0.07478,0.11912,0.04929,0.02014,0.0013,0.00157,0.00944,0.03034,0.02223
+DK,77284,43341,76710,40513,33608,158622,97849,177377,35956,33206,158622,97849,177377,35956,33206,172046,150770,327105,412493,302583,59.72,59.72,109.42,172.18,202.58,226.35,0.01885,0.01006,0.01223,0.00434,0.00247,0.15229,0.08408,0.12954,0.01669,0.00841,0.10042,0.05544,0.08542,0.011,0.00555,0.06509,0.05429,0.08089,0.06847,0.03447
+EE,34042,36214,36013,33988,15698,81995,42686,22460,8947,10983,81995,42686,22460,8947,10983,13276,5196,44006,96607,43751,203.98,203.98,191.33,229.48,236.43,291.75,0.02824,0.02036,0.01831,0.01203,0.00557,0.26145,0.1103,0.05633,0.01871,0.02449,0.17241,0.07274,0.03715,0.01234,0.01615,0.01709,0.00453,0.03469,0.05304,0.02407
+EU27,4984535,6464071,11042214,8374951,4376323,11053175,9252972,14200329,12496438,6419872,11053175,9252972,14200329,12496438,6419872,8855985,9311584,16513097,18567684,13538392,72.67,72.67,107.85,163.86,189.77,209.7,0.01176,0.01394,0.02083,0.01106,0.00402,0.12243,0.09275,0.12291,0.07119,0.02464,0.08074,0.06116,0.08105,0.04694,0.01625,0.04213,0.04016,0.0631,0.04786,0.02508
+FI,10351,10170,29463,32457,60211,105820,125904,204614,205773,84573,105820,125904,204614,205773,84573,201289,241638,379767,379090,108934,190.48,190.48,205.87,234.43,233.05,254.1,0.00158,0.0013,0.00387,0.00387,0.00292,0.07328,0.07997,0.13073,0.11545,0.0439,0.04832,0.05273,0.0862,0.07613,0.02895,0.04724,0.0479,0.07738,0.07007,0.00821
+FR,339761,854255,1634317,1254881,681431,1793424,587206,1528743,880896,777747,1793424,587206,1528743,880896,777747,1910563,2233673,4737245,5668791,4404256,80.25,80.25,152.04,210.76,197.91,243.89,0.0069,0.01365,0.02452,0.01217,0.00465,0.14269,0.03791,0.10511,0.04369,0.02036,0.09409,0.025,0.06931,0.02881,0.01343,0.06025,0.05536,0.11022,0.08528,0.0466
+DE,281971,1804893,2691415,1199460,334852,2377845,2473457,2941305,2376772,963897,2377845,2473457,2941305,2376772,963897,2379682,3154370,3004374,1789092,756414,79.41,79.41,121.3,204.46,232.4,261.92,0.00332,0.0195,0.02324,0.0072,0.00129,0.12682,0.11705,0.12246,0.05871,0.01559,0.08363,0.07719,0.08075,0.03871,0.01028,0.05903,0.07428,0.05521,0.02028,0.00546
+EL,49430,223846,428819,371145,10109,118357,328026,608470,538729,70330,118357,328026,608470,538729,70330,140600,158204,345104,294590,237695,156.8,156.8,116.8,106.07,109.22,105.78,0.00353,0.0165,0.03173,0.02535,0.0002,0.03275,0.09373,0.16885,0.16462,0.02885,0.0216,0.06181,0.11134,0.10855,0.01902,0.01549,0.01809,0.03961,0.03121,0.00717
+HU,77091,66698,83851,137462,93059,364806,284856,422872,346410,149675,364806,284856,422872,346410,149675,6257,5504,117463,203813,111600,124.38,124.38,141.72,178.5,204.07,243.06,0.00487,0.00595,0.00539,0.00437,0.00142,0.17546,0.11503,0.14937,0.09715,0.03684,0.1157,0.07585,0.09849,0.06406,0.02429,0.00061,0.00076,0.01171,0.01005,0.00263
+IE,18353,15342,25008,28129,26463,151886,120660,213037,261141,167255,151886,120660,213037,261141,167255,11949,13203,41421,38019,13692,70.15,70.15,85.43,112.81,118.44,131.12,0.00651,0.00563,0.00721,0.0038,0.00124,0.12147,0.08717,0.14658,0.13607,0.07156,0.0801,0.05748,0.09666,0.08973,0.04719,0.00658,0.00751,0.01854,0.00797,0.001
+IT,1642075,1005062,1929623,1192868,665813,785454,484582,1003385,638652,342482,785454,484582,1003385,638652,342482,2038169,1729046,3710831,4606238,3289974,111.52,111.52,135.83,185.9,206,147.35,0.03633,0.02277,0.04225,0.02063,0.00915,0.05538,0.04777,0.08926,0.04151,0.01828,0.03652,0.0315,0.05886,0.02737,0.01205,0.06996,0.06076,0.126,0.12354,0.07011
+LV,7277,20498,29859,40184,17335,61671,6536,12216,18784,6046,61671,6536,12216,18784,6046,75989,75178,160945,194382,100129,118.46,118.46,205.52,288.76,289.82,308.69,0.00485,0.01218,0.01709,0.01776,0.0063,0.18235,0.01814,0.03379,0.03698,0.00686,0.12024,0.01196,0.02228,0.02438,0.00452,0.07843,0.06928,0.14287,0.13327,0.05647
+LT,32693,47982,84025,58845,24876,84086,77511,119408,100648,55491,84086,77511,119408,100648,55491,10141,14618,27593,49772,43298,72.51,72.51,111,192.78,264.86,299.05,0.01791,0.02033,0.02365,0.01074,0.00315,0.1574,0.1285,0.14409,0.06993,0.02519,0.10379,0.08474,0.09501,0.04611,0.01661,0.00862,0.0096,0.01205,0.01408,0.0085
+LU,10745,8041,5828,5308,4593,22789,12347,15963,11847,12822,22789,12347,15963,11847,12822,4045,3691,13373,16442,5182,63.15,63.15,132.29,231.5,337.41,344.36,0.02275,0.01771,0.00873,0.00406,0.00077,0.22702,0.12051,0.1069,0.04534,0.02342,0.1497,0.07947,0.07049,0.0299,0.01545,0.01328,0.01261,0.03106,0.01949,0.00134
+MT,4954,3862,6043,6401,2202,24504,17384,27356,27686,7645,24504,17384,27356,27686,7645,565,682,773,2384,5588,17.85,17.85,24.31,30.02,34.33,34.55,0.00497,0.00421,0.00535,0.00478,0.00142,0.16062,0.11323,0.15581,0.12769,0.02589,0.10592,0.07466,0.10274,0.0842,0.01707,0.00088,0.00115,0.00106,0.00276,0.00559
+PL,392733,362655,550747,685267,456017,840585,865498,1260882,1477868,845316,840585,865498,1260882,1477868,845316,237980,56196,204776,314698,811717,121.9,121.9,153.47,204.46,253.77,312.35,0.01303,0.007,0.00846,0.00893,0.00524,0.13572,0.11353,0.13326,0.11724,0.05327,0.0895,0.07487,0.08788,0.07731,0.03512,0.01226,0.00168,0.00488,0.00636,0.01446
+PT,103200,85030,228873,279820,127189,304310,217196,557290,696728,311806,304310,217196,557290,696728,311806,36467,24832,74592,73340,44517,76.88,76.88,90.37,114.12,116.79,136.16,0.00998,0.00638,0.01677,0.01952,0.00477,0.10515,0.06437,0.16139,0.15978,0.06083,0.06934,0.04245,0.10643,0.10536,0.04011,0.00547,0.00289,0.00848,0.00794,0.00259
+RO,146273,306643,489495,365759,269967,437509,773862,759785,724163,536919,437509,773862,759785,724163,536919,47897,201831,324709,150744,215225,73.48,73.48,110.89,165.79,172.51,181.38,0.00733,0.01098,0.01661,0.00827,0.00464,0.09481,0.15949,0.15049,0.09594,0.04713,0.06252,0.10517,0.09924,0.06326,0.03108,0.00373,0.0112,0.01709,0.00529,0.00573
+SK,73624,68237,79796,71044,26985,193675,126005,168544,170258,57203,193675,126005,168544,170258,57203,4300,19734,73236,114824,46646,136.67,136.67,150.24,168.07,182.7,140.94,0.01457,0.01353,0.01412,0.01097,0.00344,0.12907,0.10886,0.13395,0.12095,0.03697,0.08511,0.07178,0.08833,0.07976,0.02438,0.00132,0.00607,0.0201,0.0275,0.00922
+SI,7522,13010,10757,8595,7983,60016,97686,61775,40701,40024,60016,97686,61775,40701,40024,13647,11818,31748,43238,19796,88.33,88.33,130.61,175.54,198.03,201.03,0.00392,0.00605,0.00379,0.002,0.00133,0.13164,0.21107,0.11832,0.058,0.03857,0.0868,0.13918,0.07802,0.03825,0.02544,0.01108,0.00853,0.01733,0.01558,0.0051
+ES,672013,674356,1329996,1618556,1039154,1058663,872103,1657375,2000678,1070162,1058663,872103,1657375,2000678,1070162,534762,164470,594692,1346018,1152776,35.58,35.58,46.23,70.02,80.3,75.9,0.01803,0.01704,0.02954,0.02565,0.01214,0.0969,0.08445,0.13994,0.11153,0.04592,0.0639,0.05569,0.09228,0.07355,0.03028,0.02225,0.00644,0.02049,0.03309,0.02089
+SE,292809,62630,138845,111752,30561,198270,88924,173492,95142,39054,198270,88924,173492,95142,39054,320766,320742,682090,821486,578536,149.56,149.56,157.35,169.67,216.63,263.48,0.05197,0.00816,0.01278,0.00795,0.00181,0.13522,0.04986,0.07619,0.03875,0.01512,0.08916,0.03288,0.05024,0.02555,0.00997,0.08829,0.06481,0.09735,0.0907,0.05324