From 9c644fc76a9fe1005020f24c10499a3f70e2751a Mon Sep 17 00:00:00 2001 From: bstabler Date: Wed, 3 Apr 2019 18:02:30 -0700 Subject: [PATCH 1/6] expression file fixes school location: - preschool kids go to school cdap: - num young child age <= 5 - preschool kids now have a school taz - add preschool kid age misuse for now - income > 100k not less than - add commented out telecommuting expression to match tm1 test uecs mtf: - preschool kids can only have 1 school tour (not 2) like TM1 nmtf: - num non-mandatory tours should be num non-mandatory tours not mandatory tours - log max residual time window fix --- example/configs/annotate_households.csv | 67 +++++++++---------- example/configs/annotate_persons.csv | 6 +- example/configs/annotate_persons_nmtf.csv | 18 ++--- .../cdap_fixed_relative_proportions.csv | 9 ++- example/configs/cdap_indiv_and_hhsize1.csv | 19 +++--- example/configs/mandatory_tour_frequency.csv | 2 +- .../configs/non_mandatory_tour_frequency.csv | 12 ++-- 7 files changed, 66 insertions(+), 67 deletions(-) diff --git a/example/configs/annotate_households.csv b/example/configs/annotate_households.csv index 56b9c2249..70109602c 100644 --- a/example/configs/annotate_households.csv +++ b/example/configs/annotate_households.csv @@ -1,35 +1,34 @@ -Description,Target,Expression -#,, annotate households table after import -,_PERSON_COUNT,"lambda query, persons, households: persons.query(query).groupby('household_id').size().reindex(households.index).fillna(0).astype(np.int8)" +Description,Target,Expression, +#,, annotate households table after import, +,_PERSON_COUNT,"lambda query, persons, households: persons.query(query).groupby('household_id').size().reindex(households.index).fillna(0).astype(np.int8)", #,,FIXME households.income can be negative, so we clip? -income_in_thousands,income_in_thousands,(households.income / 1000).clip(lower=0) -income_segment,income_segment,"pd.cut(income_in_thousands, bins=[-np.inf, 30, 60, 100, np.inf], labels=[1, 2, 3, 4]).astype(int)" -#,, -,_MIN_VOT,setting('min_value_of_time') -,_MAX_VOT,setting('max_value_of_time') -,_MU,setting('distributed_vot_mu') -,_SIGMA,setting('distributed_vot_sigma') -median_value_of_time,median_value_of_time,"income_segment.map({k: v for k, v in setting('household_median_value_of_time').items()})" -hh_value_of_time,hh_value_of_time,"rng.lognormal_for_df(df, mu=np.log(median_value_of_time * _MU), sigma=_SIGMA).clip(_MIN_VOT, _MAX_VOT)" -#,, -#num_workers was renamed in import,, -#,num_workers,households.workers -number of non_workers,num_non_workers,households.hhsize - households.num_workers -#,, -#,,we assume that everyone 16 and older is a potential driver -number of drivers,num_drivers,"_PERSON_COUNT('16 <= age', persons, households)" -num_adults,num_adults,"_PERSON_COUNT('adult', persons, households)" -num_children,num_children,"_PERSON_COUNT('~adult', persons, households)" -#,,FIXME num_young_children is under 5 here but used in mandatory_tour_frequency where CTRAMP uec wants num_preschool which java code has as under 6 -num_young_children,num_young_children,"_PERSON_COUNT('age <= 4', persons, households)" -num_children_5_to_15,num_children_5_to_15,"_PERSON_COUNT('5 <= age <= 15', persons, households)" -num_children_16_to_17,num_children_16_to_17,"_PERSON_COUNT('16 <= age <= 17', persons, households)" -num_college_age,num_college_age,"_PERSON_COUNT('18 <= age <= 24', persons, households)" -num_young_adults,num_young_adults,"_PERSON_COUNT('25 <= age <= 34', persons, households)" -non_family,non_family,households.HHT.isin(constants.HHT_NONFAMILY) -family,family,households.HHT.isin(constants.HHT_FAMILY) -home_is_urban,home_is_urban,"reindex(land_use.area_type, households.TAZ) < setting('urban_threshold')" -home_is_rural,home_is_rural,"reindex(land_use.area_type, households.TAZ) > setting('rural_threshold')" -#,, default for work and school location logsums before auto_ownership model is run -,auto_ownership,households.VEHICL -#home_taz,home_taz,households.TAZ +income_in_thousands,income_in_thousands,(households.income / 1000).clip(lower=0), +income_segment,income_segment,"pd.cut(income_in_thousands, bins=[-np.inf, 30, 60, 100, np.inf], labels=[1, 2, 3, 4]).astype(int)", +#,,, +,_MIN_VOT,setting('min_value_of_time'), +,_MAX_VOT,setting('max_value_of_time'), +,_MU,setting('distributed_vot_mu'), +,_SIGMA,setting('distributed_vot_sigma'), +median_value_of_time,median_value_of_time,"income_segment.map({k: v for k, v in setting('household_median_value_of_time').items()})", +hh_value_of_time,hh_value_of_time,"rng.lognormal_for_df(df, mu=np.log(median_value_of_time * _MU), sigma=_SIGMA).clip(_MIN_VOT, _MAX_VOT)", +#,,, +#num_workers was renamed in import,,, +#,num_workers,households.workers, +number of non_workers,num_non_workers,households.hhsize - households.num_workers, +#,,, +#,,we assume that everyone 16 and older is a potential driver, +number of drivers,num_drivers,"_PERSON_COUNT('16 <= age', persons, households)", +num_adults,num_adults,"_PERSON_COUNT('adult', persons, households)", +num_children,num_children,"_PERSON_COUNT('~adult', persons, households)", +num_young_children,num_young_children,"_PERSON_COUNT('age <= 5', persons, households)", +num_children_5_to_15,num_children_5_to_15,"_PERSON_COUNT('5 <= age <= 15', persons, households)", +num_children_16_to_17,num_children_16_to_17,"_PERSON_COUNT('16 <= age <= 17', persons, households)", +num_college_age,num_college_age,"_PERSON_COUNT('18 <= age <= 24', persons, households)", +num_young_adults,num_young_adults,"_PERSON_COUNT('25 <= age <= 34', persons, households)", +non_family,non_family,households.HHT.isin(constants.HHT_NONFAMILY), +family,family,households.HHT.isin(constants.HHT_FAMILY), +home_is_urban,home_is_urban,"reindex(land_use.area_type, households.TAZ) < setting('urban_threshold')", +home_is_rural,home_is_rural,"reindex(land_use.area_type, households.TAZ) > setting('rural_threshold')", +#,, default for work and school location logsums before auto_ownership model is run, +,auto_ownership,households.VEHICL, +#home_taz,home_taz,households.TAZ, diff --git a/example/configs/annotate_persons.csv b/example/configs/annotate_persons.csv index c54375f84..3690abc8a 100644 --- a/example/configs/annotate_persons.csv +++ b/example/configs/annotate_persons.csv @@ -24,10 +24,8 @@ nonstudent_to_school,nonstudent_to_school,"(persons.ptype.isin([constants.PTYPE_ ,pstudent,"pstudent.where((persons.ptype!=constants.PTYPE_DRIVING) & (persons.ptype!=constants.PTYPE_SCHOOL), constants.PSTUDENT_GRADE_OR_HIGH)" #,, is_student,is_student,"pstudent.isin([constants.PSTUDENT_GRADE_OR_HIGH, constants.PSTUDENT_UNIVERSITY])" -#babies go to school like tm1 bug,is_student,"is_student.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, True)" -#babies go to school like tm1 bug,pstudent,"pstudent.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, constants.PSTUDENT_GRADE_OR_HIGH)" -babies dont go to school,is_student,"is_student.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, False)" -babies dont go to school,pstudent,"pstudent.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, constants.PSTUDENT_NOT)" +preschool age can go to preschool,is_student,"is_student.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, True)" +preschool age can go to preschool,pstudent,"pstudent.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, constants.PSTUDENT_GRADE_OR_HIGH)" is_gradeschool,is_gradeschool,(pstudent == constants.PSTUDENT_GRADE_OR_HIGH) & (persons.age <= constants.GRADE_SCHOOL_MAX_AGE) is_highschool,is_highschool,(pstudent == constants.PSTUDENT_GRADE_OR_HIGH) & (persons.age > constants.GRADE_SCHOOL_MAX_AGE) is_university,is_university,pstudent == constants.PSTUDENT_UNIVERSITY diff --git a/example/configs/annotate_persons_nmtf.csv b/example/configs/annotate_persons_nmtf.csv index 08b7f0de3..058bf618c 100644 --- a/example/configs/annotate_persons_nmtf.csv +++ b/example/configs/annotate_persons_nmtf.csv @@ -1,13 +1,13 @@ Description,Target,Expression #,, annotate persons table after non_mandatory_tour_frequency model has run -num_non_mand,num_non_mand,"tours[tours.tour_category=='mandatory'].groupby('person_id').size().reindex(persons.index).fillna(0)" -#num_escort_tours,num_escort_tours,"tours[tours.tour_type == 'escort'].groupby('person_id').size().reindex(persons.index).fillna(0)" +num_non_mand,num_non_mand,tours[tours.tour_category=='non_mandatory'].groupby('person_id').size().reindex(persons.index).fillna(0) +#num_escort_tours,num_escort_tours,tours[tours.tour_type == 'escort'].groupby('person_id').size().reindex(persons.index).fillna(0) #num_non_escort_tours,num_non_escort_tours,"tours[~tours.tour_type.isin(['escort', 'work', 'school'])].groupby('person_id').size().reindex(persons.index).fillna(0)" -#num_eatout_tours,num_eatout_tours,"tours[tours.tour_type == 'eatout'].groupby('person_id').size().reindex(persons.index).fillna(0)" +#num_eatout_tours,num_eatout_tours,tours[tours.tour_type == 'eatout'].groupby('person_id').size().reindex(persons.index).fillna(0) #,, -num_escort_tours,num_escort_tours,"tours[tours.tour_type == 'escort'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8)" -num_eatout_tours,num_eatout_tours,"tours[tours.tour_type == 'eatout'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8)" -num_shop_tours,num_shop_tours,"tours[tours.tour_type == 'shopping'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8)" -num_maint_tours,num_maint_tours,"tours[tours.tour_type == 'othmaint'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8)" -num_social_tours,num_social_tours,"tours[tours.tour_type == 'social'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8)" -num_non_escort_tours,num_non_escort_tours,"num_non_mand-num_escort_tours" +num_escort_tours,num_escort_tours,tours[tours.tour_type == 'escort'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8) +num_eatout_tours,num_eatout_tours,tours[tours.tour_type == 'eatout'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8) +num_shop_tours,num_shop_tours,tours[tours.tour_type == 'shopping'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8) +num_maint_tours,num_maint_tours,tours[tours.tour_type == 'othmaint'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8) +num_social_tours,num_social_tours,tours[tours.tour_type == 'social'].groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8) +num_non_escort_tours,num_non_escort_tours,num_non_mand-num_escort_tours diff --git a/example/configs/cdap_fixed_relative_proportions.csv b/example/configs/cdap_fixed_relative_proportions.csv index 2bdaca8bb..788f398b6 100644 --- a/example/configs/cdap_fixed_relative_proportions.csv +++ b/example/configs/cdap_fixed_relative_proportions.csv @@ -1,10 +1,9 @@ Description,Expression,M,N,H Full-time worker,ptype == 1,0.79647,0.09368,0.10985 Part-time worker,ptype == 2,0.61678,0.25757,0.12565 -University student,ptype == 3,0.69229,0.15641,0.15130 -Non-working adult,ptype == 4,0.00000,0.67169,0.32831 -Retired,ptype == 5,0.00000,0.54295,0.45705 +University student,ptype == 3,0.69229,0.15641,0.1513 +Non-working adult,ptype == 4,0,0.67169,0.32831 +Retired,ptype == 5,0,0.54295,0.45705 Driving-age child who is in school,ptype == 6,0.77609,0.06004,0.16387 Pre-driving-age child who is in school,ptype == 7,0.68514,0.09144,0.22342 -Pre-driving-age child who is too young for school,(ptype == 8) & (school_taz>=0),0.14056,0.06512,0.79432 -Pre-driving-age child who is too young for school,(ptype == 8) & (school_taz<0),0.00000,0.06512,0.79432 +Child who is too young for school,ptype == 8,0.14056,0.06512,0.79432 diff --git a/example/configs/cdap_indiv_and_hhsize1.csv b/example/configs/cdap_indiv_and_hhsize1.csv index f21996c45..4f4f8c4c7 100644 --- a/example/configs/cdap_indiv_and_hhsize1.csv +++ b/example/configs/cdap_indiv_and_hhsize1.csv @@ -9,11 +9,12 @@ Pre-driving-age child who is in school alternative-specific constants,ptype == 7 Pre-driving-age child who is in school interaction with age 6 to 9,(ptype == 7) & (age >= 6) & (age <= 9),-0.2943,, Pre-driving-age child who is in school interaction with age 13 to 15,(ptype == 7) & (age >= 13) & (age <= 15),-0.7141,-0.672, Pre-driving-age child who is too young for school alternative-specific constants,ptype == 8,1.052531189,-0.837567776, -Pre-driving-age child who is too young for school interaction with age 0 to 1,(ptype == 8) & (age >= 0) & (age <= 1),-0.4515,, +# tm1 age bug,,,, +Pre-driving-age child who is too young for school interaction with age 0 to 1,(ptype == 8) * age,-0.4515,, +# asim corrected age bug,,,, +# Pre-driving-age child who is too young for school interaction with age 0 to 1,(ptype == 8) & (age >= 0) & (age <= 1),-0.4515,, Pre-driving-age child who is too young for school interaction with age 4 to 5,(ptype == 8) & (age >= 4) & (age <= 5),0.6107,, -# FIXME,,,, -Pre-driving-age child who is too young for school ,(ptype == 8) & (school_taz<0),-999,, -# +#,,,, Full-time worker interaction with age less than 40,(ptype == 1) & (age < 40),0.2091,, Retired interaction with age more than 80,(ptype == 5) & (age > 80),,,0.7666 Full-time worker interaction with female gender,(ptype == 1) & (sex == 2),-0.1259,, @@ -32,14 +33,14 @@ Full-time worker interaction with income less than $20k,(ptype == 1) & (income_i Retired interaction with income less than $20k,(ptype == 5) & (income_in_thousands < 20),,,0.533 Part-time worker interaction with income less than $20k,(ptype == 2) & (income_in_thousands < 20),,,0.3232 Part-time worker interaction with income between $50k and $100k,(ptype == 2) & (income_in_thousands >= 50) & (income_in_thousands <= 100),,,-0.4032 -Part-time worker interaction with income more than $100k,(ptype == 2) & (income_in_thousands < 100),,0.4207,-0.3534 +Part-time worker interaction with income more than $100k,(ptype == 2) & (income_in_thousands >= 100),,0.4207,-0.3534 Non-working adult interaction with income between $50k and $100k,(ptype == 4) & (income_in_thousands >= 50) & (income_in_thousands <= 100),,,-0.5602 -Non-working adult interaction with income more than $100k,(ptype == 4) & (income_in_thousands < 100),,,-0.7188 +Non-working adult interaction with income more than $100k,(ptype == 4) & (income_in_thousands >= 100),,,-0.7188 Driving-age child who is in school interaction with less than $20k,(ptype == 6) & (income_in_thousands < 20),,,1.307 Driving-age child who is in school interaction income between $50k and $100k,(ptype == 6) & (income_in_thousands >= 50) & (income_in_thousands <= 100),,,-0.5031 -Driving-age child who is in school interaction with income more than $100k,(ptype == 6) & (income_in_thousands < 100),,,-2.046 +Driving-age child who is in school interaction with income more than $100k,(ptype == 6) & (income_in_thousands >= 100),,,-2.046 Pre-driving-age child who is too young for school interaction with income between $50k and $100k,(ptype == 8) & (income_in_thousands >= 50) & (income_in_thousands <= 100),,,-0.5708 -Pre-driving-age child who is too young for school interaction with income more than $100k,(ptype == 8) & (income_in_thousands < 100),,,-0.6186 +Pre-driving-age child who is too young for school interaction with income more than $100k,(ptype == 8) & (income_in_thousands >= 100),,,-0.6186 Full-time worker intraction with peak accessibility to all employment,(ptype == 1) * auPkTotal,0.1212,, Part-time worker interaction with peak accessibility to all employment,(ptype == 2) * auPkTotal,0.2004,, Non-working adult interaction with peak accessibility to all employment,(ptype == 4) * auPkTotal,0.2314,, @@ -57,3 +58,5 @@ Pre-driving-age child who is too young for school interaction with off-peak acce # Part-time worker interaction with no usual work location,(ptype == 2) * noUsualWorkLocation,-0.5935,, # Driving-age child who is in school interaction with no usual school location,(ptype == 6) * noUsualWorkLocation,-0.866,, # Pre-driving age child who is in school interaction with no usual school location,(ptype == 7) * noUsualWorkLocation,-0.866,, +#tm1 scenario test,,,, +#Simulate telecommuting by reducing mandatory patterns,(ptype == 1) * (income_in_thousands >= 50),-0.142930042,, diff --git a/example/configs/mandatory_tour_frequency.csv b/example/configs/mandatory_tour_frequency.csv index ad1c42212..4c452b7b1 100644 --- a/example/configs/mandatory_tour_frequency.csv +++ b/example/configs/mandatory_tour_frequency.csv @@ -96,6 +96,6 @@ Unavailable: Non-working adult,ptype == 4,,,,-999,-999 Unavailable: Retired,ptype == 5,,,,-999,-999 Unavailable: Driving-age child,ptype == 6,-999,-999,,, Unavailable: Pre-driving age child who is in school,ptype == 7,,-999,,,-999 -Unavailable: Pre-driving age child who is not in school,ptype == 8,-999,-999,,,-999 +Unavailable: Pre-driving age child who is not in school,ptype == 8,-999,-999,,-999,-999 Unavailable: Work tours for those with no usual work location,~(workplace_taz > -1),-999,-999,,,-999 Unavailable: School tours for those with no usual school location,~(school_taz > -1),,,-999,-999,-999 diff --git a/example/configs/non_mandatory_tour_frequency.csv b/example/configs/non_mandatory_tour_frequency.csv index ec9d079a5..e42071ada 100644 --- a/example/configs/non_mandatory_tour_frequency.csv +++ b/example/configs/non_mandatory_tour_frequency.csv @@ -24,12 +24,12 @@ Number of Joint Maintenance tours,num_hh_joint_maint_tours,0,0,0,0,0,0,0,0 Number of Joint Eating Out tours,num_hh_joint_eatout_tours,-0.5866,0,0,-0.7727,0,0,0,0 Number of Joint Visit tours,num_hh_joint_social_tours,0,0,0,0,0,0,0,0 Number of Joint Discretionary tours,num_hh_joint_othdiscr_tours,0,0,0.6713,0,0,0,0,0 -"Logged Maximum Residual Window, tour frequency =0",max_window*(tot_tours == 0),0,0,1.1858,0,0,0,0,0 -"Logged Maximum Residual Window, tour frequency =1",max_window*(tot_tours == 1),1.2562,1.5748,1.4842,1.7637,1.8357,1.3298,1.3298,0 -"Logged Maximum Residual Window, tour frequency =2",max_window*(tot_tours == 2),1.2868,2.0026,1.4842,1.7928,2.2707,1.3759,1.3759,0 -"Logged Maximum Residual Window, tour frequency =3",max_window*(tot_tours == 3),1.3993,2.0026,1.4842,1.7928,4.4023,3.2808,3.2808,0 -"Logged Maximum Residual Window, tour frequency =4",max_window*(tot_tours == 4),1.3993,2.0026,1.4842,1.7928,4.4023,3.2808,3.2808,0 -"Logged Maximum Residual Window, tour frequency =5+",max_window*(tot_tours > 4),1.3993,2.0026,1.4842,1.7928,4.4023,3.2808,3.2808,0 +"Logged Maximum Residual Window, tour frequency =0",log_max_window*(tot_tours == 0),0,0,1.1858,0,0,0,0,0 +"Logged Maximum Residual Window, tour frequency =1",log_max_window*(tot_tours == 1),1.2562,1.5748,1.4842,1.7637,1.8357,1.3298,1.3298,0 +"Logged Maximum Residual Window, tour frequency =2",log_max_window*(tot_tours == 2),1.2868,2.0026,1.4842,1.7928,2.2707,1.3759,1.3759,0 +"Logged Maximum Residual Window, tour frequency =3",log_max_window*(tot_tours == 3),1.3993,2.0026,1.4842,1.7928,4.4023,3.2808,3.2808,0 +"Logged Maximum Residual Window, tour frequency =4",log_max_window*(tot_tours == 4),1.3993,2.0026,1.4842,1.7928,4.4023,3.2808,3.2808,0 +"Logged Maximum Residual Window, tour frequency =5+",log_max_window*(tot_tours > 4),1.3993,2.0026,1.4842,1.7928,4.4023,3.2808,3.2808,0 Dummy for Mediumlow Income group (20K-50K) & tour frequency=1,medium_low_income & (tot_tours == 1),0.4981,0.5981,0,0.5709,0,0,0,0 Dummy for Mediumlow Income group (20K-50K) & tour frequency=2,medium_low_income & (tot_tours == 2),0.8345,0.9178,0,0.8315,0,0,0,0 Dummy for Mediumlow Income group (20K-50K) & tour frequency=3,medium_low_income & (tot_tours == 3),1.0213,1.7539,0,0.8315,0,0,0,0 From c7a2e332a5f8bcc425ac42bd79cf24cb44faf63a Mon Sep 17 00:00:00 2001 From: Jeff Doyle Date: Thu, 4 Apr 2019 11:39:16 -0400 Subject: [PATCH 2/6] fix confusion about in and out semantics --- activitysim/abm/models/util/logsums.py | 2 +- activitysim/abm/test/test_pipeline.py | 14 +++++++------- activitysim/core/config.py | 10 +++++++--- activitysim/core/tracing.py | 7 +++---- example/configs/school_location.yaml | 1 + example/simulation.py | 7 ++++--- verification/configs/settings.yaml | 2 +- verification/simulation.py | 2 +- 8 files changed, 25 insertions(+), 20 deletions(-) diff --git a/activitysim/abm/models/util/logsums.py b/activitysim/abm/models/util/logsums.py index f1949e908..4eb0a550c 100644 --- a/activitysim/abm/models/util/logsums.py +++ b/activitysim/abm/models/util/logsums.py @@ -91,7 +91,7 @@ def compute_logsums(choosers, choosers['out_period'] = expressions.skim_time_period_label(model_settings['OUT_PERIOD']) assert ('duration' not in choosers) - choosers['duration'] = model_settings['OUT_PERIOD'] - model_settings['IN_PERIOD'] + choosers['duration'] = model_settings['IN_PERIOD'] - model_settings['OUT_PERIOD'] nest_spec = config.get_logit_model_settings(logsum_settings) constants = config.get_model_constants(logsum_settings) diff --git a/activitysim/abm/test/test_pipeline.py b/activitysim/abm/test/test_pipeline.py index 09b77dc70..9ae22c38c 100644 --- a/activitysim/abm/test/test_pipeline.py +++ b/activitysim/abm/test/test_pipeline.py @@ -129,8 +129,8 @@ def regress_mini_mtf(): mtf_choice = pipeline.get_table("persons").sort_index().mandatory_tour_frequency # these choices are for pure regression - their appropriateness has not been checked - per_ids = [2566698, 2566700, 2566701] - choices = ['work1', 'school1', 'school2'] + per_ids = [2877283, 2877284, 2877285] + choices = ['work1', 'work_and_school', 'school1'] expected_choice = pd.Series(choices, index=pd.Index(per_ids, name='person_id'), name='mandatory_tour_frequency') @@ -142,11 +142,11 @@ def regress_mini_mtf(): """ mtf_choice person_id - 2566698 work1 - 2566700 school1 - 2566701 school2 - 2877282 work1 - 2877283 work1 + 2566698 work1 + 2877282 work1 + 2877283 work1 + 2877284 work_and_school + 2877285 school1 Name: mandatory_tour_frequency, dtype: object """ pdt.assert_series_equal(mtf_choice.reindex(per_ids), expected_choice) diff --git a/activitysim/core/config.py b/activitysim/core/config.py index 86c91d7c7..73d7ae001 100644 --- a/activitysim/core/config.py +++ b/activitysim/core/config.py @@ -77,7 +77,9 @@ def str2bool(v): @inject.injectable(cache=True) def settings(): - return read_settings_file('settings.yaml', mandatory=True) + settings_dict = read_settings_file('settings.yaml', mandatory=True) + + return settings_dict def handle_standard_args(parser=None): @@ -338,14 +340,16 @@ def backfill_settings(settings, backfill): file_path = os.path.join(dir, file_name) if os.path.exists(file_path): if settings: - logger.debug("inherit settings for %s from %s" % (file_name, file_path)) + logger.warn("read settings for %s from %s" % (file_name, file_path)) with open(file_path) as f: s = yaml.load(f) settings = backfill_settings(settings, s) + print("settings after %s\n%s\n\n" % (file_path, settings)) + if s.get('inherit_settings', False): - logger.debug("inherit_settings flag set for %s in %s" % (file_name, file_path)) + logger.warn("inherit_settings flag set for %s in %s" % (file_name, file_path)) continue else: break diff --git a/activitysim/core/tracing.py b/activitysim/core/tracing.py index 086fb93fd..a915c16c5 100644 --- a/activitysim/core/tracing.py +++ b/activitysim/core/tracing.py @@ -275,10 +275,9 @@ def write_df_csv(df, file_path, index_label=None, columns=None, column_labels=No df.to_csv(file_path, mode='a', index=df.index.name is not None, header=need_header) return - df_t = df.transpose() - if df.index.name is not None: - df_t.index.name = df.index.name - elif index_label: + df_t = df.transpose() if df.index.name in df else df.reset_index().transpose() + + if index_label: df_t.index.name = index_label if need_header: diff --git a/example/configs/school_location.yaml b/example/configs/school_location.yaml index 5be78100d..5132c6ea6 100644 --- a/example/configs/school_location.yaml +++ b/example/configs/school_location.yaml @@ -3,6 +3,7 @@ SAMPLE_SIZE: 30 SIMULATE_CHOOSER_COLUMNS: - TAZ - school_segment + - household_id # model-specific logsum-related settings CHOOSER_ORIG_COL_NAME: TAZ diff --git a/example/simulation.py b/example/simulation.py index 04e69a4d3..275827b11 100644 --- a/example/simulation.py +++ b/example/simulation.py @@ -16,7 +16,7 @@ from activitysim import abm from activitysim.core import tracing -from activitysim.core.config import handle_standard_args +from activitysim.core import config from activitysim.core.config import setting from activitysim.core import pipeline @@ -24,13 +24,14 @@ def run(): - - handle_standard_args() + config.handle_standard_args() # specify None for a pseudo random base seed # inject.add_injectable('rng_base_seed', 0) tracing.config_logger() + config.filter_warnings() + tracing.delete_csv_files() # If you provide a resume_after argument to pipeline.run diff --git a/verification/configs/settings.yaml b/verification/configs/settings.yaml index 4f7ba3a3f..091e6e499 100644 --- a/verification/configs/settings.yaml +++ b/verification/configs/settings.yaml @@ -44,7 +44,7 @@ num_processes: 3 # - tracing #trace_hh_id: 324105 trace_od: -trace_hh_id: 107599 +trace_hh_id: 324105 #trace_od: [5, 11] # to resume after last successful checkpoint, specify resume_after: _ diff --git a/verification/simulation.py b/verification/simulation.py index defc27867..e7c882a2d 100644 --- a/verification/simulation.py +++ b/verification/simulation.py @@ -83,8 +83,8 @@ def log_settings(injectables): injectables = config.handle_standard_args() - config.filter_warnings() tracing.config_logger() + config.filter_warnings() log_settings(injectables) From 8df86ba21adb472b753644c046bfb90e5fb572cf Mon Sep 17 00:00:00 2001 From: Jeff Doyle Date: Thu, 4 Apr 2019 13:35:33 -0400 Subject: [PATCH 3/6] passing tests --- .../abm/test/configs/school_location.yaml | 3 +- activitysim/abm/test/test_pipeline.py | 70 +++++-------------- 2 files changed, 21 insertions(+), 52 deletions(-) diff --git a/activitysim/abm/test/configs/school_location.yaml b/activitysim/abm/test/configs/school_location.yaml index 288e798b4..5132c6ea6 100644 --- a/activitysim/abm/test/configs/school_location.yaml +++ b/activitysim/abm/test/configs/school_location.yaml @@ -3,11 +3,12 @@ SAMPLE_SIZE: 30 SIMULATE_CHOOSER_COLUMNS: - TAZ - school_segment + - household_id # model-specific logsum-related settings CHOOSER_ORIG_COL_NAME: TAZ ALT_DEST_COL_NAME: alt_dest -IN_PERIOD: 17 +IN_PERIOD: 14 OUT_PERIOD: 8 DEST_CHOICE_COLUMN_NAME: school_taz diff --git a/activitysim/abm/test/test_pipeline.py b/activitysim/abm/test/test_pipeline.py index 9ae22c38c..81c6f64f7 100644 --- a/activitysim/abm/test/test_pipeline.py +++ b/activitysim/abm/test/test_pipeline.py @@ -318,7 +318,7 @@ def get_trace_csv(file_name): return df -EXPECT_TOUR_COUNT = 386 +EXPECT_TOUR_COUNT = 232 def regress_tour_modes(tours_df): @@ -332,37 +332,23 @@ def regress_tour_modes(tours_df): print("mode_df\n", tours_df[mode_cols]) """ - tour_id tour_mode person_id tour_type tour_num tour_category - tour_id - 91533577 SHARED3FREE 3156330 escort 1 non_mandatory - 91533578 WALK 3156330 escort 2 non_mandatory - 91533576 WALK 3156330 eatout 3 non_mandatory - 91533606 DRIVEALONEFREE 3156331 escort 1 non_mandatory - 91533607 DRIVEALONEFREE 3156331 escort 2 non_mandatory - 91533620 BIKE 3156331 othdiscr 3 non_mandatory - 91533647 DRIVEALONEFREE 3156332 maint 1 atwork - 91533655 DRIVEALONEFREE 3156332 work 1 mandatory - 91533635 SHARED3FREE 3156332 escort 1 non_mandatory - 91533650 WALK_LOC 3156332 othmaint 2 non_mandatory - 91533649 WALK_LOC 3156332 othdiscr 3 non_mandatory - 91533585 SHARED2FREE 3156333 shopping 1 joint - 91533680 WALK 3156333 school 1 mandatory - 91533682 WALK 3156333 shopping 1 non_mandatory - 91533709 WALK 3156334 school 1 mandatory - 91533708 SHARED3FREE 3156334 othmaint 1 non_mandatory - 91533738 WALK 3156335 school 1 mandatory - 91533737 SHARED2FREE 3156335 othmaint 1 non_mandatory + tour_mode person_id tour_type tour_num tour_category + tour_id + 91533592 WALK 3156330 othmaint 1 non_mandatory + 91533620 WALK_LOC 3156331 othdiscr 1 non_mandatory + 91533647 SHARED2FREE 3156332 maint 1 atwork + 91533655 WALK 3156332 work 1 mandatory + 91533650 WALK_LOC 3156332 othmaint 1 non_mandatory + 91533583 SHARED2FREE 3156333 othmaint 1 joint + 91533680 WALK 3156333 school 1 mandatory + 91533683 SHARED2FREE 3156333 social 1 non_mandatory + 91533709 WALK 3156334 school 1 mandatory + 91533738 WALK 3156335 school 1 mandatory """ EXPECT_PERSON_IDS = [ 3156330, - 3156330, - 3156330, - 3156331, 3156331, - 3156331, - 3156332, - 3156332, 3156332, 3156332, 3156332, @@ -370,51 +356,33 @@ def regress_tour_modes(tours_df): 3156333, 3156333, 3156334, - 3156334, - 3156335, 3156335, ] EXPECT_TOUR_TYPES = [ - 'escort', - 'escort', - 'eatout', - 'escort', - 'escort', + 'othmaint', 'othdiscr', 'maint', 'work', - 'escort', 'othmaint', - 'othdiscr', - 'shopping', + 'othmaint', 'school', - 'shopping', + 'social', 'school', - 'othmaint', 'school', - 'othmaint', ] EXPECT_MODES = [ - 'SHARED3FREE', 'WALK', - 'WALK', - 'DRIVEALONEFREE', - 'DRIVEALONEFREE', - 'BIKE', - 'DRIVEALONEFREE', - 'DRIVEALONEFREE', - 'SHARED3FREE', - 'WALK_LOC', 'WALK_LOC', 'SHARED2FREE', 'WALK', + 'WALK_LOC', + 'SHARED2FREE', 'WALK', + 'SHARED2FREE', 'WALK', - 'SHARED3FREE', 'WALK', - 'SHARED2FREE', ] assert len(tours_df) == len(EXPECT_PERSON_IDS) From 9f8162842cb9d989d1f30f34a8169590acad126e Mon Sep 17 00:00:00 2001 From: Jeff Doyle Date: Thu, 4 Apr 2019 14:05:05 -0400 Subject: [PATCH 4/6] pycodestyle --- activitysim/abm/test/test_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/abm/test/test_pipeline.py b/activitysim/abm/test/test_pipeline.py index 81c6f64f7..1ad0a002d 100644 --- a/activitysim/abm/test/test_pipeline.py +++ b/activitysim/abm/test/test_pipeline.py @@ -333,7 +333,7 @@ def regress_tour_modes(tours_df): """ tour_mode person_id tour_type tour_num tour_category - tour_id + tour_id 91533592 WALK 3156330 othmaint 1 non_mandatory 91533620 WALK_LOC 3156331 othdiscr 1 non_mandatory 91533647 SHARED2FREE 3156332 maint 1 atwork From 99a41fdb3d3670f8853e6f3d55459c55309f7313 Mon Sep 17 00:00:00 2001 From: bstabler Date: Thu, 4 Apr 2019 11:23:18 -0700 Subject: [PATCH 5/6] ensure pyyaml version works #253 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5d0d7b757..527d9fbe3 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'numpy == 1.15.4', # https://github.com/PyTables/PyTables/issues/719 'openmatrix >= 0.3.4.1', 'pandas >= 0.20.3', - 'pyyaml >= 3.0', + 'pyyaml >= 3.0, <5.1', 'tables >= 3.3.0', 'toolz >= 0.8.1', 'zbox >= 1.2', From 57e697d6d2419e2333497fbd461f1153e48e0ca0 Mon Sep 17 00:00:00 2001 From: Jeff Doyle Date: Thu, 4 Apr 2019 16:12:36 -0400 Subject: [PATCH 6/6] yaml.load argument Loader now required --- activitysim/abm/test/test_pipeline.py | 2 +- activitysim/core/config.py | 4 +--- activitysim/core/mp_tasks.py | 2 +- activitysim/core/tracing.py | 3 ++- setup.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/activitysim/abm/test/test_pipeline.py b/activitysim/abm/test/test_pipeline.py index 1ad0a002d..fb2fe15a0 100644 --- a/activitysim/abm/test/test_pipeline.py +++ b/activitysim/abm/test/test_pipeline.py @@ -68,7 +68,7 @@ def close_handlers(): def inject_settings(configs_dir, **kwargs): with open(os.path.join(configs_dir, 'settings.yaml')) as f: - settings = yaml.load(f) + settings = yaml.load(f, Loader=yaml.SafeLoader) for k in kwargs: settings[k] = kwargs[k] diff --git a/activitysim/core/config.py b/activitysim/core/config.py index 73d7ae001..07b27f398 100644 --- a/activitysim/core/config.py +++ b/activitysim/core/config.py @@ -343,11 +343,9 @@ def backfill_settings(settings, backfill): logger.warn("read settings for %s from %s" % (file_name, file_path)) with open(file_path) as f: - s = yaml.load(f) + s = yaml.load(f, Loader=yaml.SafeLoader) settings = backfill_settings(settings, s) - print("settings after %s\n%s\n\n" % (file_path, settings)) - if s.get('inherit_settings', False): logger.warn("inherit_settings flag set for %s in %s" % (file_name, file_path)) continue diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 887854a06..54fc6f9be 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -1524,7 +1524,7 @@ def read_breadcrumbs(): if not os.path.exists(file_path): raise IOError("Could not find saved breadcrumbs file '%s'" % file_path) with open(file_path, 'r') as f: - breadcrumbs = yaml.load(f) + breadcrumbs = yaml.load(f, Loader=yaml.SafeLoader) # convert array to ordered dict keyed by step name breadcrumbs = OrderedDict([(step['name'], step) for step in breadcrumbs]) return breadcrumbs diff --git a/activitysim/core/tracing.py b/activitysim/core/tracing.py index a915c16c5..027881f81 100644 --- a/activitysim/core/tracing.py +++ b/activitysim/core/tracing.py @@ -129,7 +129,8 @@ def config_logger(basic=False): if log_config_file: with open(log_config_file) as f: - config_dict = yaml.load(f) + # FIXME need alternative to yaml.UnsafeLoader? + config_dict = yaml.load(f, Loader=yaml.UnsafeLoader) config_dict = config_dict['logging'] config_dict.setdefault('version', 1) logging.config.dictConfig(config_dict) diff --git a/setup.py b/setup.py index 527d9fbe3..5d0d7b757 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'numpy == 1.15.4', # https://github.com/PyTables/PyTables/issues/719 'openmatrix >= 0.3.4.1', 'pandas >= 0.20.3', - 'pyyaml >= 3.0, <5.1', + 'pyyaml >= 3.0', 'tables >= 3.3.0', 'toolz >= 0.8.1', 'zbox >= 1.2',