diff --git a/demeter/change/expansion.py b/demeter/change/expansion.py index 9118e4d..4a4af82 100644 --- a/demeter/change/expansion.py +++ b/demeter/change/expansion.py @@ -109,13 +109,13 @@ def _expansion(diagnostic, diag_file, spat_ludataharm_sub, kernel_vector_sub, co trans_mat = np.zeros((l_shs, l_ord, l_ord)) # process PFTs in order - for pft_ord in sorted(order_rules): + for pft_ord in np.unique(order_rules): # create a copy of the cons_data_sub array cons_data_sub = cons_data_sub_o.copy() - # lookup PFT and land class name - pft = order_rules.index(pft_ord) + # lookup PFT and final land class + pft = np.where(order_rules == pft_ord)[0][0] fcs = final_landclasses[pft] # define intensification targets @@ -237,22 +237,23 @@ def apply_expansion(log, c, allregnumber, allregmet, spat_ludataharm, spat_regio # get region and metric from index regnumber = allregnumber[reg_idx] - metnumber = np.unique(spat_met)[met_idx] - met_ref = met_idx + metnumber = allregmet[reg_idx][met_idx] + metnum_idx = metnumber - 1 # create data subset - spat_ludataharm_sub = spat_ludataharm[(spat_region == regnumber) & (spat_met == metnumber)] - kernel_vector_sub = kernel_vector[(spat_region == regnumber) & (spat_met == metnumber)] - cons_data_sub = cons_data[(spat_region == regnumber) & (spat_met == metnumber)] + reg_met_mask = (spat_region == regnumber) & (spat_met == metnumber) + spat_ludataharm_sub = spat_ludataharm[reg_met_mask] + kernel_vector_sub = kernel_vector[reg_met_mask] + cons_data_sub = cons_data[reg_met_mask] # calculate expansion for each PFT exp = _expansion(c.diagnostic, diag_file, spat_ludataharm_sub, kernel_vector_sub, cons_data_sub, reg_idx, - met_ref, order_rules, final_landclasses, c.errortol, constrain_rules, transition_rules, + metnum_idx, order_rules, final_landclasses, c.errortol, constrain_rules, transition_rules, c.stochastic_expansion, c.selection_threshold, land_mismatch, target_change) # apply expansion and update transitions - spat_ludataharm[(spat_region == regnumber) & (spat_met == metnumber)], target_change, trans_mat = exp - transitions[(spat_region == regnumber) & (spat_met == metnumber), :, :] += trans_mat + spat_ludataharm[reg_met_mask], target_change, trans_mat = exp + transitions[reg_met_mask, :, :] += trans_mat # calculate non-achieved change non_chg = np.sum(abs(target_change[:, :, :])) / 2. diff --git a/demeter/change/intensification.py b/demeter/change/intensification.py index c75db12..9b1b469 100644 --- a/demeter/change/intensification.py +++ b/demeter/change/intensification.py @@ -92,11 +92,12 @@ def _convert_pft(notdone, int_target, metnumber, pft_toconv, spat_ludataharm_sub spat_ludataharm_sub[exist_cells, pft_toconv] -= actexpansion # update the target change values - target_change[reg, metnumber - 1, pft] -= np.sum(actexpansion) - int_target -= np.sum(actexpansion) - target_intensification[metnumber - 1, pft] -= np.sum(actexpansion) - target_change[reg, metnumber - 1, pft_toconv] += np.sum(actexpansion) - target_intensification[metnumber - 1, pft_toconv] += np.sum(actexpansion) + actual_expansion_sum = np.sum(actexpansion) + target_change[reg, metnumber - 1, pft] -= actual_expansion_sum + int_target -= actual_expansion_sum + target_intensification[metnumber - 1, pft] -= actual_expansion_sum + target_change[reg, metnumber - 1, pft_toconv] += actual_expansion_sum + target_intensification[metnumber - 1, pft_toconv] += actual_expansion_sum trans_mat[exist_cells, pft, pft_toconv] += actexpansion # account for target change minuscule values when evaluating notdone @@ -312,9 +313,10 @@ def apply_intensification(log, pass_number, c, spat_region, order_rules, allregn metnumber = allregmet[reg_idx][met_idx] # create data subset - spat_ludataharm_sub = spat_ludataharm[(spat_region == regnumber) & (spat_met == metnumber)] - kernel_vector_sub = kernel_vector[(spat_region == regnumber) & (spat_met == metnumber)] - cons_data_sub = cons_data[(spat_region == regnumber) & (spat_met == metnumber)] + reg_met_mask = (spat_region == regnumber) & (spat_met == metnumber) + spat_ludataharm_sub = spat_ludataharm[reg_met_mask] + kernel_vector_sub = kernel_vector[reg_met_mask] + cons_data_sub = cons_data[reg_met_mask] # calculate intensification citz = _intensification(c.diagnostic, diag_file, spat_ludataharm_sub, target_intensification, kernel_vector_sub, @@ -322,10 +324,10 @@ def apply_intensification(log, pass_number, c, spat_region, order_rules, allregn constrain_rules, target_change, transition_rules, land_mismatch) # apply intensification - spat_ludataharm[(spat_region == regnumber) & (spat_met == metnumber)], trans_mat, target_change, target_intensification = citz + spat_ludataharm[reg_met_mask], trans_mat, target_change, target_intensification = citz # log transition - transitions[(spat_region == regnumber) & (spat_met == metnumber), :, :] += trans_mat + transitions[reg_met_mask, :, :] += trans_mat # calculate non-achieved change non_chg = np.sum(abs(target_change[:, :, :])) / 2. diff --git a/demeter/config_reader.py b/demeter/config_reader.py index 9006c57..7838072 100644 --- a/demeter/config_reader.py +++ b/demeter/config_reader.py @@ -16,6 +16,11 @@ from configobj import ConfigObj +class ValidationException(Exception): + def __init__(self,*args,**kwargs): + Exception.__init__(self,*args,**kwargs) + + class ReadConfig: def __init__(self, config_file): @@ -191,14 +196,14 @@ def ck_ts(t, st_y, ed_y): ts = int(t) if (rng == 0) and (ts != 1): - raise RuntimeError('Parameter "timestep" value must be 1 if only running one year. Your start year and end year are the same in your config file. Exiting...') + raise ValidationException('Parameter "timestep" value must be 1 if only running one year. Your start year and end year are the same in your config file. Exiting...') elif (rng == 0) and (ts == 1): return ts ck = rng / ts if ck == 0: - raise RuntimeError('Parameter "timestep" value "{0}" is too large for start year of "{1}" and end year of "{2}". Max time step available based on year range is "{3}". Exiting...'.format(t, st_y, ed_y, ed_y - st_y)) + raise ValidationException('Parameter "timestep" value "{0}" is too large for start year of "{1}" and end year of "{2}". Max time step available based on year range is "{3}". Exiting...'.format(t, st_y, ed_y, ed_y - st_y)) else: return ts @@ -212,7 +217,7 @@ def ck_yr(y, p): :return: int """ if len(y) != 4: - raise RuntimeError('Year must be in four digit format (e.g., 2005) for parameter "{}". Value entered was "{}". Exiting...'.format(p, y)) + raise ValidationException('Year must be in four digit format (e.g., 2005) for parameter "{}". Value entered was "{}". Exiting...'.format(p, y)) else: return int(y) @@ -227,7 +232,7 @@ def ck_len(s, p, l=30): :return: string """ if len(s) > l: - raise RuntimeError('Length of "{}" exceeds the max length of 20. Please revise. Exiting...'.format(p)) + raise ValidationException('Length of "{}" exceeds the max length of 20. Please revise. Exiting...'.format(p)) else: return s @@ -244,7 +249,7 @@ def ck_vals(v, p, l): if v in l: return v else: - raise RuntimeError('Value "{0}" not in acceptable values for parameter "{1}". Acceptable values are: {2}. Exiting...'.format(v, p, l)) + raise ValidationException('Value "{0}" not in acceptable values for parameter "{1}". Acceptable values are: {2}. Exiting...'.format(v, p, l)) @staticmethod def ck_limit(v, p, l): @@ -259,7 +264,7 @@ def ck_limit(v, p, l): if (v >= l[0]) and (v <= l[1]): return v else: - raise RuntimeError('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1])) + raise ValidationException('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1])) @staticmethod def check_exist(f, kind, log): @@ -296,7 +301,7 @@ def create_dir(d, log): except Exception as e: log.error(e) log.error("ERROR: Failed to create directory.") - sys.exit(1) + raise @staticmethod def ck_agg(a, log): @@ -307,11 +312,11 @@ def ck_agg(a, log): agg = int(a) except TypeError: log.error('"agg_level" parameter must be either 1 or 2. Exiting...') - sys.exit(1) + raise if agg < 1 or agg > 2: log.error('"agg_level" parameter must be either 1 or 2. Exiting...') - sys.exit(1) + raise ValidationException else: return agg @@ -433,7 +438,7 @@ def ck_limit(v, p, l): if (v >= l[0]) and (v <= l[1]): return v else: - raise RuntimeError('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1])) + raise ValidationException('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1])) @staticmethod def ck_len(s, p, l=20): @@ -446,7 +451,7 @@ def ck_len(s, p, l=20): :return: string """ if len(s) > l: - raise RuntimeError('Length of "{}" exceeds the max length of 20. Please revise. Exiting...'.format(p)) + raise ValidationException('Length of "{}" exceeds the max length of 20. Please revise. Exiting...'.format(p)) else: return s @@ -696,14 +701,14 @@ def ck_ts(t, st_y, ed_y): ts = int(t) if (rng == 0) and (ts != 1): - raise RuntimeError('Parameter "timestep" value must be 1 if only running one year. Your start year and end year are the same in your config file. Exiting...') + raise ValidationException('Parameter "timestep" value must be 1 if only running one year. Your start year and end year are the same in your config file. Exiting...') elif (rng == 0) and (ts == 1): return ts ck = rng / ts if ck == 0: - raise RuntimeError('Parameter "timestep" value "{0}" is too large for start year of "{1}" and end year of "{2}". Max time step available based on year range is "{3}". Exiting...'.format(t, st_y, ed_y, ed_y - st_y)) + raise ValidationException('Parameter "timestep" value "{0}" is too large for start year of "{1}" and end year of "{2}". Max time step available based on year range is "{3}". Exiting...'.format(t, st_y, ed_y, ed_y - st_y)) else: return ts @@ -717,7 +722,7 @@ def ck_yr(y, p): :return: int """ if len(y) != 4: - raise RuntimeError('Year must be in four digit format (e.g., 2005) for parameter "{}". Value entered was "{}". Exiting...'.format(p, y)) + raise ValidationException('Year must be in four digit format (e.g., 2005) for parameter "{}". Value entered was "{}". Exiting...'.format(p, y)) else: return int(y) @@ -732,7 +737,7 @@ def ck_len(s, p, l=20): :return: string """ if len(s) > l: - raise RuntimeError('Length of "{}" exceeds the max length of 20. Please revise. Exiting...'.format(p)) + raise ValidationException('Length of "{}" exceeds the max length of 20. Please revise. Exiting...'.format(p)) else: return s @@ -749,7 +754,7 @@ def ck_vals(v, p, l): if v in l: return v else: - raise RuntimeError('Value "{0}" not in acceptable values for parameter "{1}". Acceptable values are: {2}. Exiting...'.format(v, p, l)) + raise ValidationException('Value "{0}" not in acceptable values for parameter "{1}". Acceptable values are: {2}. Exiting...'.format(v, p, l)) @staticmethod def ck_limit(v, p, l): @@ -764,7 +769,7 @@ def ck_limit(v, p, l): if (v >= l[0]) and (v <= l[1]): return v else: - raise RuntimeError('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1])) + raise ValidationException('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1])) @staticmethod @@ -779,11 +784,11 @@ def check_exist(f, kind, log): if kind == 'file' and os.path.isfile(f) is False: log.error("File not found: {0}".format(f)) log.error("Confirm path and retry.") - sys.exit() + raise IOError elif kind == 'dir' and os.path.isdir(f) is False: log.error("Directory not found: {0}".format(f)) log.error("Confirm path and retry.") - sys.exit() + raise IOError else: return f @@ -802,7 +807,7 @@ def create_dir(d, log): except Exception as e: log.error(e) log.error("ERROR: Failed to create directory.") - sys.exit() + raise ValidationException @staticmethod def ck_agg(a, log): @@ -813,11 +818,11 @@ def ck_agg(a, log): agg = int(a) except TypeError: log.error('"agg_level" parameter must be either 1 or 2. Exiting...') - sys.exit(1) + raise ValidationException if agg < 1 or agg > 2: log.error('"agg_level" parameter must be either 1 or 2. Exiting...') - sys.exit(1) + raise ValidationException else: return agg @@ -870,10 +875,4 @@ def get_constraints(self): return l else: - return list() - -if __name__ == "__main__": - - ini = '/users/ladmin/repos/github/demeter/example/config.ini' - - ReadConfigShuffle(ini, '/users/ladmin/Desktop') \ No newline at end of file + return list() \ No newline at end of file diff --git a/demeter/constraints.py b/demeter/constraints.py index f3bc6b0..be8f248 100644 --- a/demeter/constraints.py +++ b/demeter/constraints.py @@ -9,11 +9,15 @@ """ import os import numpy as np -import sys import demeter.demeter_io.reader as rdr +class ValidationException(Exception): + def __init__(self,*args,**kwargs): + Exception.__init__(self,*args,**kwargs) + + class ApplyConstraints: def __init__(self, allreg, allaez, final_landclasses, user_years, ixr_ixm, allregaez, spat_region, allregnumber, @@ -97,7 +101,7 @@ def apply_spat_constraints(self): print("\nERROR: Aggregation numbers for PFT {0} in spatial allocation file sum up to more than 1.".format(i)) print("Please correct and try again.") print("Exiting...\n") - sys.exit() + raise ValidationException # if individual values sum to greater than 1 if np.sum(t > 0) > 1: @@ -193,7 +197,7 @@ def apply_gcam_constraints(self, yr_idx, gcam_landmatrix, spat_landmatrix, ixr_i print("\nERROR: No aggregation class defined for PFT {0} in the GCAM allocation file".format(self.gcam_landclasses(gix))) print("Please correct and try again.") print("Exiting...\n") - sys.exit() + raise ValidationException # Examine the case of one-to-many recombination (e.g., rockicedesert to snow and sparse). Data is split into # the new categories following their share in the base land use layer within the considered region, diff --git a/demeter/logger.py b/demeter/logger.py index aecc36d..aeada48 100644 --- a/demeter/logger.py +++ b/demeter/logger.py @@ -86,5 +86,6 @@ def close_logger(self, log): """ handlers = log.handlers[:] for h in handlers: + h.flush() h.close() log.removeHandler(h) diff --git a/demeter/model.py b/demeter/model.py index e8d6002..8c9382f 100644 --- a/demeter/model.py +++ b/demeter/model.py @@ -22,6 +22,11 @@ from demeter.weight.kernel_density import KernelDensity +class ValidationException(Exception): + def __init__(self,*args,**kwargs): + Exception.__init__(self,*args,**kwargs) + + class Demeter(Logger): def __init__(self, @@ -301,7 +306,7 @@ def _shuffle(dir, i, oc): 2) the type of run you wish to conduct "standard" or "ensemble" """) print('Exiting...') - sys.exit(1) + raise ValidationException # explode args ini = args[0] @@ -313,12 +318,12 @@ def _shuffle(dir, i, oc): print('ERROR: Config file not found.') print('You entered: {0}'.format(ini)) print('Please enter a full path file name with extension to config file and retry.') - sys.exit(1) + raise ValidationException if mode.lower() not in ('standard', 'ensemble'): print("""ERROR: Run mode passed '{0}' not a valid option. Either select 'standard' or 'ensemble' """.format(mode)) print('Exiting...') - sys.exit(1) + raise ValidationException # instantiate demeter dm = Demeter(config=ini) diff --git a/example/config.ini b/example/config.ini index 0cfc49b..e3d2535 100644 --- a/example/config.ini +++ b/example/config.ini @@ -1,5 +1,5 @@ [STRUCTURE] -root_dir = /users/ladmin/repos/github/demeter/example +root_dir = /users/d3y010/repos/github/demeter/example in_dir = inputs out_dir = outputs @@ -72,7 +72,7 @@ observed_id_field = fid start_year = 2005 # last year to process -end_year = 2100 +end_year = 2015 # enter 1 to use non-kernel density constraints, 0 to ignore non-kernel density constraints use_constraints = 1 @@ -119,7 +119,7 @@ map_luc_steps = 0 map_transitions = 0 # years to save data for, default is all; otherwise a semicolon delimited string e.g, 2005;2050 -target_years_output = 2005; 2100 +target_years_output = 2005; 2015 # save tabular spatial landcover as CSV; define tabular_units below (default sqkm) save_tabular = 1 @@ -145,7 +145,7 @@ save_ascii_max = 0 permutations = 2 # the CSV file full path with file name of the limits to create parameter ensembles for -limits_file = /users/ladmin/repos/github/demeter/example/inputs/reference/limits.csv +limits_file = /users/d3y010/repos/github/demeter/example/inputs/reference/limits.csv # the number of CPUs to spread the processing over (-1 is all, -2 is all but one, 4 is four). n_jobs = -1 diff --git a/example/inputs/projected/gcam_ref_scenario_reg32aez.csv b/example/inputs/projected/gcam_ref_scenario_reg32aez.csv index e6e8047..cce1243 100644 --- a/example/inputs/projected/gcam_ref_scenario_reg32aez.csv +++ b/example/inputs/projected/gcam_ref_scenario_reg32aez.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f49efc1744cf1e80b5b10bba36d858f1b480bb4aab1d1ab0e4cb3f2e26ae3be2 -size 949386 +oid sha256:e0922c84eea047bcc7546de591ff14146fc6b30602730fd81d9f07ad33bb1ad3 +size 1461439