From 261d768d334e4cf9e4be3eba969083322262df86 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 19 Sep 2017 22:08:04 -0400 Subject: [PATCH 1/5] Use new param processing logic in dynamic elasticity --- webapp/apps/dynamic/views.py | 109 ++++++++++++-------------------- webapp/apps/taxbrain/compute.py | 15 +++-- webapp/apps/taxbrain/helpers.py | 2 +- webapp/apps/taxbrain/views.py | 6 +- 4 files changed, 53 insertions(+), 79 deletions(-) diff --git a/webapp/apps/dynamic/views.py b/webapp/apps/dynamic/views.py index bbad5e83..68547e75 100644 --- a/webapp/apps/dynamic/views.py +++ b/webapp/apps/dynamic/views.py @@ -35,7 +35,8 @@ DynamicElasticitySaveInputs, DynamicElasticityOutputUrl) from ..taxbrain.models import TaxSaveInputs, OutputUrl, ErrorMessageTaxCalculator from ..taxbrain.views import (growth_fixup, benefit_switch_fixup, make_bool, dropq_compute, - JOB_PROC_TIME_IN_SECONDS, get_reform_from_gui) + JOB_PROC_TIME_IN_SECONDS, get_reform_from_gui, + parse_fields) from ..taxbrain.helpers import (default_policy, taxcalc_results_to_tables, default_behavior, convert_val, to_json_reform) @@ -188,6 +189,7 @@ def dynamic_behavioral(request, pk): This view handles the dynamic behavioral input page and calls the function that handles the calculation on the inputs. """ + start_year = request.REQUEST.get('start_year') if request.method == 'POST': # Client is attempting to send inputs, validate as form data fields = dict(request.REQUEST) @@ -198,19 +200,8 @@ def dynamic_behavioral(request, pk): if dyn_mod_form.is_valid(): model = dyn_mod_form.save() - curr_dict = dict(model.__dict__) - for key, value in curr_dict.items(): - print "got this ", key, value - - for key, value in curr_dict.items(): - if type(value) == type(unicode()): - curr_dict[key] = [convert_val(x) for x in value.split(',') if x] - else: - print "missing this: ", key - - # get macrosim data from form - worker_data = {k:v for k, v in curr_dict.items() if v not in (u'', None, [])} + worker_data = parse_fields(curr_dict) #get microsim data outputsurl = OutputUrl.objects.get(pk=pk) @@ -238,9 +229,9 @@ def dynamic_behavioral(request, pk): submitted_ids, max_q_length = dropq_compute.submit_dropq_calculation( reform_dict, int(start_year), - is_file=is_file, + is_file=False, additional_data=assumptions_dict, - package_up_user_mods=False + pack_up_user_mods=False ) if not submitted_ids: @@ -280,11 +271,10 @@ def dynamic_behavioral(request, pk): form_personal_exemp = dyn_mod_form else: - # Probably a GET request, load a default form - start_year = request.REQUEST.get('start_year') form_personal_exemp = DynamicBehavioralInputsModelForm(first_year=start_year) + behavior_default_params = default_behavior(int(start_year)) init_context = { @@ -307,11 +297,7 @@ def dynamic_elasticities(request, pk): This view handles the dynamic macro elasticities input page and calls the function that handles the calculation on the inputs. """ - - # Probably a GET request, load a default form start_year = request.REQUEST.get('start_year') - elasticity_default_params = default_elasticity_parameters(int(start_year)) - if request.method=='POST': # Client is attempting to send inputs, validate as form data fields = dict(request.REQUEST) @@ -323,64 +309,49 @@ def dynamic_elasticities(request, pk): model = dyn_mod_form.save() curr_dict = dict(model.__dict__) - for key, value in curr_dict.items(): - print "got this ", key, value + # for key, value in curr_dict.items(): + # print "got this ", key, value - #Replace empty elasticity field with defaults - for k,v in elasticity_default_params.items(): - if k in curr_dict and not curr_dict[k]: - curr_dict[k] = elasticity_default_params[k].col_fields[0].values + # #Replace empty elasticity field with defaults + # for k,v in elasticity_default_params.items(): + # if k in curr_dict and not curr_dict[k]: + # curr_dict[k] = elasticity_default_params[k].col_fields[0].values - for key, value in curr_dict.items(): - if type(value) == type(unicode()): - try: - curr_dict[key] = [float(x) for x in value.split(',') if x] - except ValueError: - curr_dict[key] = [make_bool(x) for x in value.split(',') if x] - else: - print "missing this: ", key + # for key, value in curr_dict.items(): + # if type(value) == type(unicode()): + # try: + # curr_dict[key] = [float(x) for x in value.split(',') if x] + # except ValueError: + # curr_dict[key] = [make_bool(x) for x in value.split(',') if x] + # else: + # print "missing this: ", key - - # get macrosim data from form - worker_data = {k:v for k, v in curr_dict.items() if v not in (u'', None, [])} + worker_data = parse_fields(curr_dict) #get microsim data outputsurl = OutputUrl.objects.get(pk=pk) model.micro_sim = outputsurl taxbrain_model = outputsurl.unique_inputs + # necessary for simulations before PR 641 if not taxbrain_model.json_text: - taxbrain_dict = dict(taxbrain_model.__dict__) - growth_fixup(taxbrain_dict) - for key, value in taxbrain_dict.items(): - if type(value) == type(unicode()): - taxbrain_dict[key] = [convert_val(x) for x in value.split(',') if x] - else: - print "missing this: ", key - - microsim_data = {k:v for k, v in taxbrain_dict.items() if not (v == [] or v == None)} - - #Don't need to pass around the microsim results - if 'tax_result' in microsim_data: - del microsim_data['tax_result'] - - benefit_switch_fixup(request.REQUEST, microsim_data, taxbrain_model) - microsim_data.update(worker_data) - # start calc job - submitted_ids, max_q_length = dropq_compute.submit_elastic_calculation(microsim_data, - int(start_year)) - - else: - el_keys = ('first_year', 'elastic_gdp') - elasticity_params = { k:v for k, v in worker_data.items() if k in el_keys} - additional_data = {'elasticity_params': elasticity_params} - reform_text = json.loads(taxbrain_model.json_text.reform_text) - reform_text[reform_text.keys()[0]]["elastic_gdp"] = elasticity_params["elastic_gdp"] - submitted_ids, max_q_length = dropq_compute.submit_elastic_calculation( - reform_text, - int(start_year), - is_file=True + (reform_dict, _, _, _, + errors_warnings) = get_reform_from_gui( + request, + taxbrain_model=taxbrain_model, + behavior_model=None ) + else: + reform_dict = json.loads(taxbrain_model.json_text.reform_text) + reform_dict[reform_dict.keys()[0]]['elastic_gdp'] = {'value': worker_data['elastic_gdp']} + + submitted_ids, max_q_length = dropq_compute.submit_elastic_calculation( + reform_dict, + int(start_year), + is_file=False, + additional_data={}, + pack_up_user_mods=False + ) if not submitted_ids: no_inputs = True @@ -421,9 +392,11 @@ def dynamic_elasticities(request, pk): form_personal_exemp = dyn_mod_form else: + # Probably a GET request, load a default form form_personal_exemp = DynamicElasticityInputsModelForm(first_year=start_year) + elasticity_default_params = default_elasticity_parameters(int(start_year)) init_context = { 'form': form_personal_exemp, diff --git a/webapp/apps/taxbrain/compute.py b/webapp/apps/taxbrain/compute.py index c8d3b959..a92bcf7c 100644 --- a/webapp/apps/taxbrain/compute.py +++ b/webapp/apps/taxbrain/compute.py @@ -58,35 +58,36 @@ def submit_json_dropq_calculation(self, user_mods, first_budget_year, additional additional_data=additional_data) def submit_dropq_calculation(self, user_mods, first_budget_year, additional_data={}, is_file=False, - package_up_user_mods=True): + pack_up_user_mods=True): url_template = "http://{hn}" + DROPQ_URL return self.submit_calculation(user_mods, first_budget_year, url_template, num_years=NUM_BUDGET_YEARS, additional_data=additional_data, - pack_up_user_mods=package_up_user_mods) + pack_up_user_mods=pack_up_user_mods) def submit_json_dropq_small_calculation(self, user_mods, first_budget_year): url_template = "http://{hn}" + DROPQ_SMALL_URL return self.submit_calculation(user_mods, first_budget_year, url_template, num_years=NUM_BUDGET_YEARS_QUICK, increment_counter=False, - pack_up_user_mods=False) + pack_up_user_mods=pack_up_user_mods) def submit_dropq_small_calculation(self, user_mods, first_budget_year, additional_data={}, is_file=False, - package_up_user_mods=True): + pack_up_user_mods=True): url_template = "http://{hn}" + DROPQ_SMALL_URL return self.submit_calculation(user_mods, first_budget_year, url_template, num_years=NUM_BUDGET_YEARS_QUICK, additional_data=additional_data, increment_counter=False, - pack_up_user_mods=package_up_user_mods) + pack_up_user_mods=pack_up_user_mods) - def submit_elastic_calculation(self, user_mods, first_budget_year, is_file=False, additional_data={}): + def submit_elastic_calculation(self, user_mods, first_budget_year, is_file=False, additional_data={}, + pack_up_user_mods=True): url_template = "http://{hn}/elastic_gdp_start_job" return self.submit_calculation(user_mods, first_budget_year, url_template, start_budget_year=1, additional_data=additional_data, - pack_up_user_mods=not is_file) + pack_up_user_mods=pack_up_user_mods) def submit_calculation(self, user_mods, first_budget_year, url_template, diff --git a/webapp/apps/taxbrain/helpers.py b/webapp/apps/taxbrain/helpers.py index 8e72515b..7a54c55b 100644 --- a/webapp/apps/taxbrain/helpers.py +++ b/webapp/apps/taxbrain/helpers.py @@ -65,7 +65,7 @@ def convert_val(x): def parse_fields(param_dict): for k, v in param_dict.copy().items(): - if v == u'' or v is None: + if v == u'' or v is None or v == []: del param_dict[k] continue if type(v) == type(unicode()): #TODO: isinstance(value, unicode) diff --git a/webapp/apps/taxbrain/views.py b/webapp/apps/taxbrain/views.py index 1396e0b8..66230db4 100644 --- a/webapp/apps/taxbrain/views.py +++ b/webapp/apps/taxbrain/views.py @@ -405,14 +405,14 @@ def submit_reform(request, user=None): assert not taxcalc_errors # try: # TODO: is try-catch necessary here? log_ip(request) - # TODO: drop is_file and package_up_user_mods keywords + # TODO: drop is_file and pack_up_user_mods keywords if do_full_calc: submitted_ids, max_q_length = dropq_compute.submit_dropq_calculation( reform_dict, int(start_year), is_file=is_file, additional_data=assumptions_dict, - package_up_user_mods=False + pack_up_user_mods=False ) else: submitted_ids, max_q_length = dropq_compute.submit_dropq_small_calculation( @@ -420,7 +420,7 @@ def submit_reform(request, user=None): int(start_year), is_file=is_file, additional_data=assumptions_dict, - package_up_user_mods=False + pack_up_user_mods=False ) job_ids = denormalize(submitted_ids) json_reform = JSONReformTaxCalculator() From f8b27337e848fa321548e879ae2231a5bd6751f2 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 19 Sep 2017 22:28:45 -0400 Subject: [PATCH 2/5] Remove nested dictionary --- webapp/apps/dynamic/views.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/webapp/apps/dynamic/views.py b/webapp/apps/dynamic/views.py index 68547e75..b44387f1 100644 --- a/webapp/apps/dynamic/views.py +++ b/webapp/apps/dynamic/views.py @@ -309,23 +309,6 @@ def dynamic_elasticities(request, pk): model = dyn_mod_form.save() curr_dict = dict(model.__dict__) - # for key, value in curr_dict.items(): - # print "got this ", key, value - - # #Replace empty elasticity field with defaults - # for k,v in elasticity_default_params.items(): - # if k in curr_dict and not curr_dict[k]: - # curr_dict[k] = elasticity_default_params[k].col_fields[0].values - - # for key, value in curr_dict.items(): - # if type(value) == type(unicode()): - # try: - # curr_dict[key] = [float(x) for x in value.split(',') if x] - # except ValueError: - # curr_dict[key] = [make_bool(x) for x in value.split(',') if x] - # else: - # print "missing this: ", key - worker_data = parse_fields(curr_dict) #get microsim data @@ -343,7 +326,7 @@ def dynamic_elasticities(request, pk): else: reform_dict = json.loads(taxbrain_model.json_text.reform_text) - reform_dict[reform_dict.keys()[0]]['elastic_gdp'] = {'value': worker_data['elastic_gdp']} + reform_dict[reform_dict.keys()[0]]['elastic_gdp'] = worker_data['elastic_gdp'] submitted_ids, max_q_length = dropq_compute.submit_elastic_calculation( reform_dict, @@ -392,8 +375,7 @@ def dynamic_elasticities(request, pk): form_personal_exemp = dyn_mod_form else: - # Probably a GET request, load a default form - + # Probably a GET request, load a default form form_personal_exemp = DynamicElasticityInputsModelForm(first_year=start_year) elasticity_default_params = default_elasticity_parameters(int(start_year)) From 37770d9d9ba5ed8cd63d56e6af05e317697930c4 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Thu, 21 Sep 2017 12:00:27 -0400 Subject: [PATCH 3/5] Set pack_up_user_mods keyword argument to false for json files --- webapp/apps/taxbrain/compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/apps/taxbrain/compute.py b/webapp/apps/taxbrain/compute.py index a92bcf7c..923ce53a 100644 --- a/webapp/apps/taxbrain/compute.py +++ b/webapp/apps/taxbrain/compute.py @@ -70,7 +70,7 @@ def submit_json_dropq_small_calculation(self, user_mods, first_budget_year): return self.submit_calculation(user_mods, first_budget_year, url_template, num_years=NUM_BUDGET_YEARS_QUICK, increment_counter=False, - pack_up_user_mods=pack_up_user_mods) + pack_up_user_mods=False) def submit_dropq_small_calculation(self, user_mods, first_budget_year, additional_data={}, is_file=False, pack_up_user_mods=True): From e8ab8e9dd4c3cd497c1601e6d1131e5b125afeff Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Thu, 21 Sep 2017 15:08:30 -0400 Subject: [PATCH 4/5] Remove unnecessary boolean variable --- webapp/apps/dynamic/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/webapp/apps/dynamic/views.py b/webapp/apps/dynamic/views.py index b44387f1..2f71c5a2 100644 --- a/webapp/apps/dynamic/views.py +++ b/webapp/apps/dynamic/views.py @@ -207,7 +207,6 @@ def dynamic_behavioral(request, pk): outputsurl = OutputUrl.objects.get(pk=pk) model.micro_sim = outputsurl taxbrain_model = outputsurl.unique_inputs - is_file = False # necessary for simulations before PR 641 if not taxbrain_model.json_text: (reform_dict, assumptions_dict, _, _, @@ -217,7 +216,6 @@ def dynamic_behavioral(request, pk): behavior_model=model ) else: - is_file = True reform_dict = json.loads(taxbrain_model.json_text.reform_text) (_, assumptions_dict, _, _, errors_warnings) = get_reform_from_gui( From bf1736c32899777b093eac62fe4b0f8e3ef6824b Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Thu, 21 Sep 2017 16:11:11 -0400 Subject: [PATCH 5/5] Set gdp elasticity for first year in reform --- webapp/apps/dynamic/views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webapp/apps/dynamic/views.py b/webapp/apps/dynamic/views.py index 2f71c5a2..ea405a5e 100644 --- a/webapp/apps/dynamic/views.py +++ b/webapp/apps/dynamic/views.py @@ -324,13 +324,14 @@ def dynamic_elasticities(request, pk): else: reform_dict = json.loads(taxbrain_model.json_text.reform_text) - reform_dict[reform_dict.keys()[0]]['elastic_gdp'] = worker_data['elastic_gdp'] + min_year = min(reform_dict.keys(), key=float) + reform_dict[min_year]['elastic_gdp'] = worker_data['elastic_gdp'] submitted_ids, max_q_length = dropq_compute.submit_elastic_calculation( reform_dict, int(start_year), is_file=False, - additional_data={}, + additional_data=None, pack_up_user_mods=False )