expression file fixes

activitysim/abm/models/util/logsums.py

@@ -91,7 +91,7 @@ def compute_logsums(choosers,
  choosers['out_period'] = expressions.skim_time_period_label(model_settings['OUT_PERIOD'])
 
  assert ('duration' not in choosers)
- choosers['duration'] = model_settings['OUT_PERIOD'] - model_settings['IN_PERIOD']
+ choosers['duration'] = model_settings['IN_PERIOD'] - model_settings['OUT_PERIOD']
 
  nest_spec = config.get_logit_model_settings(logsum_settings)
  constants = config.get_model_constants(logsum_settings)

activitysim/abm/test/configs/school_location.yaml

@@ -3,11 +3,12 @@ SAMPLE_SIZE: 30
 SIMULATE_CHOOSER_COLUMNS:
  - TAZ
  - school_segment
+ - household_id
 
 # model-specific logsum-related settings
 CHOOSER_ORIG_COL_NAME: TAZ
 ALT_DEST_COL_NAME: alt_dest
-IN_PERIOD: 17
+IN_PERIOD: 14
 OUT_PERIOD: 8
 
 DEST_CHOICE_COLUMN_NAME: school_taz

activitysim/abm/test/test_pipeline.py

@@ -68,7 +68,7 @@ def close_handlers():
 def inject_settings(configs_dir, **kwargs):
 
  with open(os.path.join(configs_dir, 'settings.yaml')) as f:
- settings = yaml.load(f)
+ settings = yaml.load(f, Loader=yaml.SafeLoader)
 
  for k in kwargs:
  settings[k] = kwargs[k]
@@ -129,8 +129,8 @@ def regress_mini_mtf():
  mtf_choice = pipeline.get_table("persons").sort_index().mandatory_tour_frequency
 
  # these choices are for pure regression - their appropriateness has not been checked
- per_ids = [2566698, 2566700, 2566701]
- choices = ['work1', 'school1', 'school2']
+ per_ids = [2877283, 2877284, 2877285]
+ choices = ['work1', 'work_and_school', 'school1']
  expected_choice = pd.Series(choices, index=pd.Index(per_ids, name='person_id'),
  name='mandatory_tour_frequency')
 
@@ -142,11 +142,11 @@ def regress_mini_mtf():
  """
  mtf_choice
  person_id
- 2566698 work1
- 2566700 school1
- 2566701 school2
- 2877282  work1
- 2877283 work1
+ 2566698  work1
+ 2877282  work1
+ 2877283  work1
+ 2877284 work_and_school
+ 2877285  school1
  Name: mandatory_tour_frequency, dtype: object
  """
  pdt.assert_series_equal(mtf_choice.reindex(per_ids), expected_choice)
@@ -318,7 +318,7 @@ def get_trace_csv(file_name):
  return df
 
 
-EXPECT_TOUR_COUNT = 386
+EXPECT_TOUR_COUNT = 232
 
 
 def regress_tour_modes(tours_df):
@@ -332,89 +332,57 @@ def regress_tour_modes(tours_df):
  print("mode_df\n", tours_df[mode_cols])
 
  """
- tour_id tour_mode person_id tour_type tour_num tour_category
+   tour_mode person_id tour_type tour_num tour_category
  tour_id
- 91533577 SHARED3FREE 3156330 escort 1 non_mandatory
- 91533578 WALK 3156330 escort 2 non_mandatory
- 91533576 WALK 3156330 eatout 3 non_mandatory
- 91533606 DRIVEALONEFREE 3156331 escort 1 non_mandatory
- 91533607 DRIVEALONEFREE 3156331 escort 2 non_mandatory
- 91533620 BIKE 3156331 othdiscr 3 non_mandatory
- 91533647 DRIVEALONEFREE 3156332 maint 1 atwork
- 91533655 DRIVEALONEFREE 3156332 work 1 mandatory
- 91533635 SHARED3FREE 3156332 escort 1 non_mandatory
- 91533650 WALK_LOC 3156332 othmaint 2 non_mandatory
- 91533649 WALK_LOC 3156332 othdiscr 3 non_mandatory
- 91533585 SHARED2FREE 3156333 shopping 1 joint
- 91533680 WALK 3156333 school 1 mandatory
- 91533682 WALK 3156333 shopping 1 non_mandatory
- 91533709 WALK 3156334 school 1 mandatory
- 91533708 SHARED3FREE 3156334 othmaint 1 non_mandatory
- 91533738 WALK 3156335 school 1 mandatory
- 91533737 SHARED2FREE 3156335 othmaint 1 non_mandatory
+ 91533592 WALK 3156330 othmaint 1 non_mandatory
+ 91533620 WALK_LOC 3156331 othdiscr 1 non_mandatory
+ 91533647 SHARED2FREE 3156332 maint 1 atwork
+ 91533655 WALK 3156332 work 1 mandatory
+ 91533650 WALK_LOC 3156332 othmaint 1 non_mandatory
+ 91533583 SHARED2FREE 3156333 othmaint 1 joint
+ 91533680 WALK 3156333 school 1 mandatory
+ 91533683 SHARED2FREE 3156333 social 1 non_mandatory
+ 91533709 WALK 3156334 school 1 mandatory
+ 91533738 WALK 3156335 school 1 mandatory
  """
 
  EXPECT_PERSON_IDS = [
  3156330,
- 3156330,
- 3156330,
- 3156331,
  3156331,
- 3156331,
- 3156332,
- 3156332,
  3156332,
  3156332,
  3156332,
  3156333,
  3156333,
  3156333,
  3156334,
- 3156334,
- 3156335,
  3156335,
  ]
 
  EXPECT_TOUR_TYPES = [
- 'escort',
- 'escort',
- 'eatout',
- 'escort',
- 'escort',
+ 'othmaint',
  'othdiscr',
  'maint',
  'work',
- 'escort',
  'othmaint',
- 'othdiscr',
- 'shopping',
+ 'othmaint',
  'school',
- 'shopping',
+ 'social',
  'school',
- 'othmaint',
  'school',
- 'othmaint',
  ]
 
  EXPECT_MODES = [
- 'SHARED3FREE',
  'WALK',
- 'WALK',
- 'DRIVEALONEFREE',
- 'DRIVEALONEFREE',
- 'BIKE',
- 'DRIVEALONEFREE',
- 'DRIVEALONEFREE',
- 'SHARED3FREE',
- 'WALK_LOC',
  'WALK_LOC',
  'SHARED2FREE',
  'WALK',
+ 'WALK_LOC',
+ 'SHARED2FREE',
  'WALK',
+ 'SHARED2FREE',
  'WALK',
- 'SHARED3FREE',
  'WALK',
- 'SHARED2FREE',
  ]
 
  assert len(tours_df) == len(EXPECT_PERSON_IDS)

activitysim/core/config.py

@@ -77,7 +77,9 @@ def str2bool(v):
 
 @inject.injectable(cache=True)
 def settings():
- return read_settings_file('settings.yaml', mandatory=True)
+ settings_dict = read_settings_file('settings.yaml', mandatory=True)
+
+ return settings_dict
 
 
 def handle_standard_args(parser=None):
@@ -338,14 +340,14 @@ def backfill_settings(settings, backfill):
  file_path = os.path.join(dir, file_name)
  if os.path.exists(file_path):
  if settings:
- logger.debug("inherit settings for %s from %s" % (file_name, file_path))
+ logger.warn("read settings for %s from %s" % (file_name, file_path))
 
  with open(file_path) as f:
- s = yaml.load(f)
+ s = yaml.load(f, Loader=yaml.SafeLoader)
  settings = backfill_settings(settings, s)
 
  if s.get('inherit_settings', False):
- logger.debug("inherit_settings flag set for %s in %s" % (file_name, file_path))
+ logger.warn("inherit_settings flag set for %s in %s" % (file_name, file_path))
  continue
  else:
  break

activitysim/core/mp_tasks.py

@@ -1524,7 +1524,7 @@ def read_breadcrumbs():
  if not os.path.exists(file_path):
  raise IOError("Could not find saved breadcrumbs file '%s'" % file_path)
  with open(file_path, 'r') as f:
- breadcrumbs = yaml.load(f)
+ breadcrumbs = yaml.load(f, Loader=yaml.SafeLoader)
  # convert array to ordered dict keyed by step name
  breadcrumbs = OrderedDict([(step['name'], step) for step in breadcrumbs])
  return breadcrumbs

activitysim/core/tracing.py

@@ -129,7 +129,8 @@ def config_logger(basic=False):
 
  if log_config_file:
  with open(log_config_file) as f:
- config_dict = yaml.load(f)
+ # FIXME need alternative to yaml.UnsafeLoader?
+ config_dict = yaml.load(f, Loader=yaml.UnsafeLoader)
  config_dict = config_dict['logging']
  config_dict.setdefault('version', 1)
  logging.config.dictConfig(config_dict)
@@ -275,10 +276,9 @@ def write_df_csv(df, file_path, index_label=None, columns=None, column_labels=No
  df.to_csv(file_path, mode='a', index=df.index.name is not None, header=need_header)
  return
 
- df_t = df.transpose()
- if df.index.name is not None:
- df_t.index.name = df.index.name
- elif index_label:
+ df_t = df.transpose() if df.index.name in df else df.reset_index().transpose()
+
+ if index_label:
  df_t.index.name = index_label
 
  if need_header:

example/configs/annotate_households.csv

@@ -1,35 +1,34 @@
-Description,Target,Expression
-#,, annotate households table after import
-,_PERSON_COUNT,"lambda query, persons, households: persons.query(query).groupby('household_id').size().reindex(households.index).fillna(0).astype(np.int8)"
+Description,Target,Expression,
+#,, annotate households table after import,
+,_PERSON_COUNT,"lambda query, persons, households: persons.query(query).groupby('household_id').size().reindex(households.index).fillna(0).astype(np.int8)",
 #,,FIXME households.income can be negative, so we clip?
-income_in_thousands,income_in_thousands,(households.income / 1000).clip(lower=0)
-income_segment,income_segment,"pd.cut(income_in_thousands, bins=[-np.inf, 30, 60, 100, np.inf], labels=[1, 2, 3, 4]).astype(int)"
-#,,
-,_MIN_VOT,setting('min_value_of_time')
-,_MAX_VOT,setting('max_value_of_time')
-,_MU,setting('distributed_vot_mu')
-,_SIGMA,setting('distributed_vot_sigma')
-median_value_of_time,median_value_of_time,"income_segment.map({k: v for k, v in setting('household_median_value_of_time').items()})"
-hh_value_of_time,hh_value_of_time,"rng.lognormal_for_df(df, mu=np.log(median_value_of_time * _MU), sigma=_SIGMA).clip(_MIN_VOT, _MAX_VOT)"
-#,,
-#num_workers was renamed in import,,
-#,num_workers,households.workers
-number of non_workers,num_non_workers,households.hhsize - households.num_workers
-#,,
-#,,we assume that everyone 16 and older is a potential driver
-number of drivers,num_drivers,"_PERSON_COUNT('16 <= age', persons, households)"
-num_adults,num_adults,"_PERSON_COUNT('adult', persons, households)"
-num_children,num_children,"_PERSON_COUNT('~adult', persons, households)"
-#,,FIXME num_young_children is under 5 here but used in mandatory_tour_frequency where CTRAMP uec wants num_preschool which java code has as under 6
-num_young_children,num_young_children,"_PERSON_COUNT('age <= 4', persons, households)"
-num_children_5_to_15,num_children_5_to_15,"_PERSON_COUNT('5 <= age <= 15', persons, households)"
-num_children_16_to_17,num_children_16_to_17,"_PERSON_COUNT('16 <= age <= 17', persons, households)"
-num_college_age,num_college_age,"_PERSON_COUNT('18 <= age <= 24', persons, households)"
-num_young_adults,num_young_adults,"_PERSON_COUNT('25 <= age <= 34', persons, households)"
-non_family,non_family,households.HHT.isin(constants.HHT_NONFAMILY)
-family,family,households.HHT.isin(constants.HHT_FAMILY)
-home_is_urban,home_is_urban,"reindex(land_use.area_type, households.TAZ) < setting('urban_threshold')"
-home_is_rural,home_is_rural,"reindex(land_use.area_type, households.TAZ) > setting('rural_threshold')"
-#,, default for work and school location logsums before auto_ownership model is run
-,auto_ownership,households.VEHICL
-#home_taz,home_taz,households.TAZ
+income_in_thousands,income_in_thousands,(households.income / 1000).clip(lower=0),
+income_segment,income_segment,"pd.cut(income_in_thousands, bins=[-np.inf, 30, 60, 100, np.inf], labels=[1, 2, 3, 4]).astype(int)",
+#,,,
+,_MIN_VOT,setting('min_value_of_time'),
+,_MAX_VOT,setting('max_value_of_time'),
+,_MU,setting('distributed_vot_mu'),
+,_SIGMA,setting('distributed_vot_sigma'),
+median_value_of_time,median_value_of_time,"income_segment.map({k: v for k, v in setting('household_median_value_of_time').items()})",
+hh_value_of_time,hh_value_of_time,"rng.lognormal_for_df(df, mu=np.log(median_value_of_time * _MU), sigma=_SIGMA).clip(_MIN_VOT, _MAX_VOT)",
+#,,,
+#num_workers was renamed in import,,,
+#,num_workers,households.workers,
+number of non_workers,num_non_workers,households.hhsize - households.num_workers,
+#,,,
+#,,we assume that everyone 16 and older is a potential driver,
+number of drivers,num_drivers,"_PERSON_COUNT('16 <= age', persons, households)",
+num_adults,num_adults,"_PERSON_COUNT('adult', persons, households)",
+num_children,num_children,"_PERSON_COUNT('~adult', persons, households)",
+num_young_children,num_young_children,"_PERSON_COUNT('age <= 5', persons, households)",
+num_children_5_to_15,num_children_5_to_15,"_PERSON_COUNT('5 <= age <= 15', persons, households)",
+num_children_16_to_17,num_children_16_to_17,"_PERSON_COUNT('16 <= age <= 17', persons, households)",
+num_college_age,num_college_age,"_PERSON_COUNT('18 <= age <= 24', persons, households)",
+num_young_adults,num_young_adults,"_PERSON_COUNT('25 <= age <= 34', persons, households)",
+non_family,non_family,households.HHT.isin(constants.HHT_NONFAMILY),
+family,family,households.HHT.isin(constants.HHT_FAMILY),
+home_is_urban,home_is_urban,"reindex(land_use.area_type, households.TAZ) < setting('urban_threshold')",
+home_is_rural,home_is_rural,"reindex(land_use.area_type, households.TAZ) > setting('rural_threshold')",
+#,, default for work and school location logsums before auto_ownership model is run,
+,auto_ownership,households.VEHICL,
+#home_taz,home_taz,households.TAZ,

example/configs/annotate_persons.csv

@@ -24,10 +24,8 @@ nonstudent_to_school,nonstudent_to_school,"(persons.ptype.isin([constants.PTYPE_
 ,pstudent,"pstudent.where((persons.ptype!=constants.PTYPE_DRIVING) & (persons.ptype!=constants.PTYPE_SCHOOL), constants.PSTUDENT_GRADE_OR_HIGH)"
 #,,
 is_student,is_student,"pstudent.isin([constants.PSTUDENT_GRADE_OR_HIGH, constants.PSTUDENT_UNIVERSITY])"
-#babies go to school like tm1 bug,is_student,"is_student.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, True)"
-#babies go to school like tm1 bug,pstudent,"pstudent.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, constants.PSTUDENT_GRADE_OR_HIGH)"
-babies dont go to school,is_student,"is_student.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, False)"
-babies dont go to school,pstudent,"pstudent.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, constants.PSTUDENT_NOT)"
+preschool age can go to preschool,is_student,"is_student.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, True)"
+preschool age can go to preschool,pstudent,"pstudent.where(persons.age > constants.GRADE_SCHOOL_MIN_AGE, constants.PSTUDENT_GRADE_OR_HIGH)"
 is_gradeschool,is_gradeschool,(pstudent == constants.PSTUDENT_GRADE_OR_HIGH) & (persons.age <= constants.GRADE_SCHOOL_MAX_AGE)
 is_highschool,is_highschool,(pstudent == constants.PSTUDENT_GRADE_OR_HIGH) & (persons.age > constants.GRADE_SCHOOL_MAX_AGE)
 is_university,is_university,pstudent == constants.PSTUDENT_UNIVERSITY

example/configs/cdap_fixed_relative_proportions.csv

@@ -1,10 +1,9 @@
 Description,Expression,M,N,H
 Full-time worker,ptype == 1,0.79647,0.09368,0.10985
 Part-time worker,ptype == 2,0.61678,0.25757,0.12565
-University student,ptype == 3,0.69229,0.15641,0.15130
-Non-working adult,ptype == 4,0.00000,0.67169,0.32831
-Retired,ptype == 5,0.00000,0.54295,0.45705
+University student,ptype == 3,0.69229,0.15641,0.1513
+Non-working adult,ptype == 4,0,0.67169,0.32831
+Retired,ptype == 5,0,0.54295,0.45705
 Driving-age child who is in school,ptype == 6,0.77609,0.06004,0.16387
 Pre-driving-age child who is in school,ptype == 7,0.68514,0.09144,0.22342
-Pre-driving-age child who is too young for school,(ptype == 8) & (school_taz>=0),0.14056,0.06512,0.79432
-Pre-driving-age child who is too young for school,(ptype == 8) & (school_taz<0),0.00000,0.06512,0.79432
+Child who is too young for school,ptype == 8,0.14056,0.06512,0.79432

example/configs/cdap_indiv_and_hhsize1.csv

@@ -9,11 +9,12 @@ Pre-driving-age child who is in school alternative-specific constants,ptype == 7
 Pre-driving-age child who is in school interaction with age 6 to 9,(ptype == 7) & (age >= 6) & (age <= 9),-0.2943,,
 Pre-driving-age child who is in school interaction with age 13 to 15,(ptype == 7) & (age >= 13) & (age <= 15),-0.7141,-0.672,
 Pre-driving-age child who is too young for school alternative-specific constants,ptype == 8,1.052531189,-0.837567776,
-Pre-driving-age child who is too young for school interaction with age 0 to 1,(ptype == 8) & (age >= 0) & (age <= 1),-0.4515,,
+# tm1 age bug,,,,
+Pre-driving-age child who is too young for school interaction with age 0 to 1,(ptype == 8) * age,-0.4515,,
+# asim corrected age bug,,,,
+# Pre-driving-age child who is too young for school interaction with age 0 to 1,(ptype == 8) & (age >= 0) & (age <= 1),-0.4515,,
 Pre-driving-age child who is too young for school interaction with age 4 to 5,(ptype == 8) & (age >= 4) & (age <= 5),0.6107,,
-# FIXME,,,,
-Pre-driving-age child who is too young for school ,(ptype == 8) & (school_taz<0),-999,,
-#
+#,,,,
 Full-time worker interaction with age less than 40,(ptype == 1) & (age < 40),0.2091,,
 Retired interaction with age more than 80,(ptype == 5) & (age > 80),,,0.7666
 Full-time worker interaction with female gender,(ptype == 1) & (sex == 2),-0.1259,,
@@ -32,14 +33,14 @@ Full-time worker interaction with income less than $20k,(ptype == 1) & (income_i
 Retired interaction with income less than $20k,(ptype == 5) & (income_in_thousands < 20),,,0.533
 Part-time worker interaction with income less than $20k,(ptype == 2) & (income_in_thousands < 20),,,0.3232
 Part-time worker interaction with income between $50k and $100k,(ptype == 2) & (income_in_thousands >= 50) & (income_in_thousands <= 100),,,-0.4032
-Part-time worker interaction with income more than $100k,(ptype == 2) & (income_in_thousands < 100),,0.4207,-0.3534
+Part-time worker interaction with income more than $100k,(ptype == 2) & (income_in_thousands >= 100),,0.4207,-0.3534
 Non-working adult interaction with income between $50k and $100k,(ptype == 4) & (income_in_thousands >= 50) & (income_in_thousands <= 100),,,-0.5602
-Non-working adult interaction with income more than $100k,(ptype == 4) & (income_in_thousands < 100),,,-0.7188
+Non-working adult interaction with income more than $100k,(ptype == 4) & (income_in_thousands >= 100),,,-0.7188
 Driving-age child who is in school interaction with less than $20k,(ptype == 6) & (income_in_thousands < 20),,,1.307
 Driving-age child who is in school interaction income between $50k and $100k,(ptype == 6) & (income_in_thousands >= 50) & (income_in_thousands <= 100),,,-0.5031
-Driving-age child who is in school interaction with income more than $100k,(ptype == 6) & (income_in_thousands < 100),,,-2.046
+Driving-age child who is in school interaction with income more than $100k,(ptype == 6) & (income_in_thousands >= 100),,,-2.046
 Pre-driving-age child who is too young for school interaction with income between $50k and $100k,(ptype == 8) & (income_in_thousands >= 50) & (income_in_thousands <= 100),,,-0.5708
-Pre-driving-age child who is too young for school interaction with income more than $100k,(ptype == 8) & (income_in_thousands < 100),,,-0.6186
+Pre-driving-age child who is too young for school interaction with income more than $100k,(ptype == 8) & (income_in_thousands >= 100),,,-0.6186
 Full-time worker intraction with peak accessibility to all employment,(ptype == 1) * auPkTotal,0.1212,,
 Part-time worker interaction with peak accessibility to all employment,(ptype == 2) * auPkTotal,0.2004,,
 Non-working adult interaction with peak accessibility to all employment,(ptype == 4) * auPkTotal,0.2314,,
@@ -57,3 +58,5 @@ Pre-driving-age child who is too young for school interaction with off-peak acce
 # Part-time worker interaction with no usual work location,(ptype == 2) * noUsualWorkLocation,-0.5935,,
 # Driving-age child who is in school interaction with no usual school location,(ptype == 6) * noUsualWorkLocation,-0.866,,
 # Pre-driving age child who is in school interaction with no usual school location,(ptype == 7) * noUsualWorkLocation,-0.866,,
+#tm1 scenario test,,,,
+#Simulate telecommuting by reducing mandatory patterns,(ptype == 1) * (income_in_thousands >= 50),-0.142930042,,