From 260dc45720d2cf3a31fa562df57e37d84517385f Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Mon, 12 Feb 2018 19:24:01 -0500 Subject: [PATCH] Implement parameter processing logic to match taxcalc reqs, see #795, #786 --- webapp/apps/taxbrain/param_formatters.py | 110 +++++++++++++++--- .../apps/test_assets/test_param_formatters.py | 38 +++++- 2 files changed, 133 insertions(+), 15 deletions(-) diff --git a/webapp/apps/taxbrain/param_formatters.py b/webapp/apps/taxbrain/param_formatters.py index 3be4bd85..6efb2c37 100644 --- a/webapp/apps/taxbrain/param_formatters.py +++ b/webapp/apps/taxbrain/param_formatters.py @@ -1,11 +1,12 @@ from collections import defaultdict, namedtuple import six import json +import ast import taxcalc from helpers import (INPUTS_META, BOOL_PARAMS, is_reverse, is_wildcard, - make_bool, convert_val) + make_bool, convert_val, TRUE_REGEX, FALSE_REGEX) MetaParam = namedtuple("MetaParam", ["param_name", "param_meta"]) @@ -84,21 +85,100 @@ def switch_fixup(taxbrain_data, fields, taxbrain_model): taxbrain_model) -def parse_fields(param_dict): - for k, v in param_dict.copy().items(): - if v == u'' or v is None or v == []: - del param_dict[k] - continue - if isinstance(v, six.string_types): - if k in BOOL_PARAMS: - converter = make_bool +def parse_value(value, meta_param): + """ + Parse the value to the type that the upstream package specification + requires, but we let the upstream package deal with creating error messages + + returns: parsed value + """ + # expecting a string + assert isinstance(value, six.string_types) + + value = value.strip() + + # if value is wildcard or reverse operator, then return it + if is_wildcard(value) or is_reverse(value): + return value.strip() + + # get type requirements from model specification + boolean_value = meta_param.param_meta["boolean_value"] + if not boolean_value: + integer_value = meta_param.param_meta["integer_value"] + else: + integer_value = False + float_value = not(boolean_value or integer_value) + + # Try to parse case-insensitive True/False strings + if TRUE_REGEX.match(value, endpos=4): + prepped = "True" + elif FALSE_REGEX.match(value, endpos=5): + prepped = "False" + else: + # ast.literal_eval won't parse negative numbers + negate = 1 + if value.rfind("-") == 0: + negate = -1 + prepped = value[1:] + else: + prepped = value + + # value is not an integer, float, or boolean string + try: + parsed = ast.literal_eval(prepped) + except ValueError: + return parsed + + # Use information given to us by upstream specs to convert this value + # into desired type or fail silently + if isinstance(parsed, bool): + return parsed + elif isinstance(parsed, int): + if boolean_value: + return True if parsed else False + elif float_value: + return negate * float(parsed) + else: + return negate * parsed + elif isinstance(parsed, float): + if boolean_value: + return True if parsed else False + elif integer_value: + # Don't want to lose info when we cast the float down to int + # Note: 5.0 % 1.0 == 0.0 but 5.2 % 1.0 == 0.2 + if parsed % 1.0 > 0: + return negate * parsed else: - converter = convert_val - param_dict[k] = [converter(x.strip()) for x in v.split(',') if x] - return param_dict + return negate * int(parsed) + else: + return negate * parsed + else: + return parsed + +def parse_fields(param_dict, default_params): + """ + Parses the raw GUI input into the correct types and maps the names to the + appropriate default names + + returns: dictionary with parsed data + """ + parsed = {} + for k, v in param_dict.items(): + # user did not specify a value for this param + if not v: + continue + + # get upstream package parameter name and meta info + meta_param = get_default_policy_param(k, default_params) + values = [] + for item in v.split(","): + values.append(parse_value(item, meta_param)) + parsed[meta_param.param_name] = values + + return parsed -def get_default_policy_param(param, default_params):qq +def get_default_policy_param(param, default_params): """ Map TaxBrain field name to Tax-Calculator parameter name For example: STD_0 maps to _STD_single @@ -321,8 +401,10 @@ def get_reform_from_gui(fields, taxbrain_model=None, behavior_model=None, # prepare taxcalc params from TaxSaveInputs model if taxbrain_model is not None: + default_params = taxcalc.Policy.default_data(start_year=start_year, + metadata=True) taxbrain_data = taxbrain_model.raw_fields - taxbrain_data = parse_fields(taxbrain_data) + taxbrain_data = parse_fields(taxbrain_data, default_params) switch_fixup(taxbrain_data, fields, taxbrain_model) # convert GUI input to json style taxcalc reform policy_dict_json, map_back_to_tb = to_json_reform(taxbrain_data, diff --git a/webapp/apps/test_assets/test_param_formatters.py b/webapp/apps/test_assets/test_param_formatters.py index 1f470308..3fec7d8c 100644 --- a/webapp/apps/test_assets/test_param_formatters.py +++ b/webapp/apps/test_assets/test_param_formatters.py @@ -9,7 +9,8 @@ parse_errors_warnings, append_errors_warnings, get_default_policy_param, - to_json_reform, MetaParam) + to_json_reform, MetaParam, + parse_value, parse_fields) START_YEAR = 2017 CUR_PATH = os.path.abspath(os.path.dirname(__file__)) @@ -79,6 +80,41 @@ def test_meta_param(): assert meta_param.param_name == name_part assert meta_param.param_meta == dict_part +############################################################################## +# Test parse_value +@pytest.mark.parametrize( + "name,value,exp", + [("FICA_ss_trt", "0.10", 0.10), ("ID_BenefitCap_Switch_0", "True", True), + ("EITC_MinEligAge", "22", 22), ("EITC_indiv", "True", True), + ("AMEDT_ec_0", "300000", 300000.0), + ("ID_BenefitCap_Switch_0", "0", False), ("STD_1", "<", "<"), + ("STD_2", "*", "*"), ("EITC_MinEligAge", "22.2", 22.2), + ("EITC_MinEligAge", "22.0", 22), ("ID_BenefitCap_Switch_0", "fAlse", False) + ] +) +def test_parse_values(name, value, exp, default_params_Policy): + meta_param = get_default_policy_param(name, default_params_Policy) + assert parse_value(value, meta_param) == exp + +# Test meta_param construction and attribute access +def test_parse_fields(default_params_Policy): + params = {"FICA_ss_trt": "<,0.10", "ID_BenefitCap_Switch_0": "True,*,False", + "EITC_MinEligAge": "22", + "AMEDT_ec_0": "300000,*,250000.0", + "STD_0": "", "STD_1": "15000,<", + "ID_BenefitCap_Switch_1": "True,fALse,<,TRUE, true"} + act = parse_fields(params, default_params_Policy) + exp = { + '_AMEDT_ec_single': [300000.0, '*', 250000.0], + '_EITC_MinEligAge': [22], + '_FICA_ss_trt': ['<', 0.1], + '_ID_BenefitCap_Switch_medical': [True, '*', False], + "_STD_joint": [15000.0,"<"], + "_ID_BenefitCap_Switch_statelocal": [True, False, "<", True, True] + } + + assert act == exp + ############################################################################### # Test to_json_reform # 2 Cases: