Poisson EET with Outer loop documentation & testing updates

.github/workflows/core_tests.yml

@@ -140,6 +140,7 @@ jobs:
       matrix:
         region:
           - prototype_mtc
+          - prototype_arc
           - placeholder_psrc
           - prototype_marin
           - prototype_mtc_extended

.pre-commit-config.yaml

@@ -17,6 +17,9 @@ repos:
   rev: 22.6.0
   hooks:
     - id: black
+      language_version: python3.13
+      # explicitly set the Python version for the hook environment for black,
+      # as older versions of black don't work with the latest Python.
 
 #- repo: https://github.com/PyCQA/flake8
 #  rev: 5.0.4

activitysim/abm/models/disaggregate_accessibility.py

@@ -15,7 +15,11 @@
 from activitysim.abm.models.util import tour_destination
 from activitysim.abm.tables import shadow_pricing
 from activitysim.core import estimation, los, tracing, util, workflow
-from activitysim.core.configuration.base import PreprocessorSettings, PydanticReadable
+from activitysim.core.configuration.base import (
+    PreprocessorSettings,
+    PydanticReadable,
+    ComputeSettings,
+)
 from activitysim.core.configuration.logit import TourLocationComponentSettings
 from activitysim.core.expressions import assign_columns
 
@@ -184,6 +188,8 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
     If not supplied or None, will default to the chunk size in the location choice model settings.
     """
 
+    compute_settings: ComputeSettings | None = None
+
 
 def read_disaggregate_accessibility_yaml(
     state: workflow.State, file_name
@@ -783,6 +789,11 @@ def get_disaggregate_logsums(
         if disagg_model_settings.explicit_chunk is not None:
             model_settings.explicit_chunk = disagg_model_settings.explicit_chunk
 
+        # Can set compute settings for disaggregate accessibility
+        # Otherwise this will be set to whatever is in the location model settings
+        if disagg_model_settings.compute_settings is not None:
+            model_settings.compute_settings = disagg_model_settings.compute_settings
+
         # Include the suffix tags to pass onto downstream logsum models (e.g., tour mode choice)
         if model_settings.LOGSUM_SETTINGS:
             suffixes = util.concat_suffix_dict(disagg_model_settings.suffixes)

activitysim/abm/models/joint_tour_participation.py

@@ -20,8 +20,8 @@
 )
 from activitysim.core.configuration.base import ComputeSettings, PreprocessorSettings
 from activitysim.core.configuration.logit import LogitComponentSettings
-from activitysim.core.util import assign_in_place, reindex
 from activitysim.core.exceptions import InvalidTravelError
+from activitysim.core.util import assign_in_place, reindex
 
 logger = logging.getLogger(__name__)
 
@@ -127,7 +127,7 @@ def get_tour_satisfaction(candidates, participate):
 
 def participants_chooser(
     state: workflow.State,
-    probs: pd.DataFrame,
+    probs_or_utils: pd.DataFrame,
     choosers: pd.DataFrame,
     spec: pd.DataFrame,
     trace_label: str,
@@ -147,9 +147,10 @@ def participants_chooser(
 
     Parameters
     ----------
-    probs : pandas.DataFrame
+    probs_or_utils : pandas.DataFrame
         Rows for choosers and columns for the alternatives from which they
-        are choosing. Values are expected to be valid probabilities across
+        are choosing. If running with explicit_error_terms, these are utilities.
+        Otherwise, values are expected to be valid probabilities across
         each row, e.g. they should sum to 1.
     choosers : pandas.dataframe
         simple_simulate choosers df
@@ -166,7 +167,7 @@ def participants_chooser(
 
     """
 
-    assert probs.index.equals(choosers.index)
+    assert probs_or_utils.index.equals(choosers.index)
 
     # choice is boolean (participate or not)
     model_settings = JointTourParticipationSettings.read_settings_file(
@@ -202,7 +203,7 @@ def participants_chooser(
                 "%s max iterations exceeded (%s).", trace_label, MAX_ITERATIONS
             )
             diagnostic_cols = ["tour_id", "household_id", "composition", "adult"]
-            unsatisfied_candidates = candidates[diagnostic_cols].join(probs)
+            unsatisfied_candidates = candidates[diagnostic_cols].join(probs_or_utils)
             state.tracing.write_csv(
                 unsatisfied_candidates,
                 file_name="%s.UNSATISFIED" % trace_label,
@@ -215,9 +216,31 @@ def participants_chooser(
                     f"Forcing joint tour participation for {num_tours_remaining} tours."
                 )
                 # anybody with probability > 0 is forced to join the joint tour
-                probs[choice_col] = np.where(probs[choice_col] > 0, 1, 0)
-                non_choice_col = [col for col in probs.columns if col != choice_col][0]
-                probs[non_choice_col] = 1 - probs[choice_col]
+                if state.settings.use_explicit_error_terms:
+                    # need "is valid choice" such that we certainly choose those with non-zero values,
+                    # and do not choose others. Let's use 3.0 as large value here.
+                    probs_or_utils[choice_col] = np.where(
+                        probs_or_utils[choice_col] > logit.UTIL_MIN,
+                        3.0,
+                        logit.UTIL_UNAVAILABLE,
+                    )
+                    non_choice_col = [
+                        col for col in probs_or_utils.columns if col != choice_col
+                    ][0]
+                    probs_or_utils[non_choice_col] = np.where(
+                        probs_or_utils[choice_col] <= logit.UTIL_MIN,
+                        3.0,
+                        logit.UTIL_UNAVAILABLE,
+                    )
+                else:
+                    probs_or_utils[choice_col] = np.where(
+                        probs_or_utils[choice_col] > 0, 1, 0
+                    )
+                    non_choice_col = [
+                        col for col in probs_or_utils.columns if col != choice_col
+                    ][0]
+                    probs_or_utils[non_choice_col] = 1 - probs_or_utils[choice_col]
+
                 if iter > MAX_ITERATIONS + 1:
                     raise InvalidTravelError(
                         f"{num_tours_remaining} tours could not be satisfied even with forcing participation"
@@ -227,8 +250,13 @@ def participants_chooser(
                     f"{num_tours_remaining} tours could not be satisfied after {iter} iterations"
                 )
 
-        choices, rands = logit.make_choices(
-            state, probs, trace_label=trace_label, trace_choosers=choosers
+        choice_function = (
+            logit.make_choices_utility_based
+            if state.settings.use_explicit_error_terms
+            else logit.make_choices
+        )
+        choices, rands = choice_function(
+            state, probs_or_utils, trace_label=trace_label, trace_choosers=choosers
         )
         participate = choices == PARTICIPATE_CHOICE
 
@@ -252,7 +280,7 @@ def participants_chooser(
             rands_list.append(rands[satisfied])
 
             # remove candidates of satisfied tours
-            probs = probs[~satisfied]
+            probs_or_utils = probs_or_utils[~satisfied]
             candidates = candidates[~satisfied]
 
         logger.debug(
@@ -401,6 +429,15 @@ def joint_tour_participation(
         if i not in model_settings.compute_settings.protect_columns:
             model_settings.compute_settings.protect_columns.append(i)
 
+    # This is related to the difference in nested logit and logit choice. As soon as alt_order_array
+    # is removed from arguments to make_choices_explicit_error_term_nl this guard can be removed.
+    if state.settings.use_explicit_error_terms:
+        assert (
+            nest_spec is None
+        ), "Nested logit model custom chooser for EET requires name_mapping, currently not implemented in jtp"
+
+    custom_chooser = participants_chooser
+
     choices = simulate.simple_simulate_by_chunk_id(
         state,
         choosers=candidates,
@@ -409,7 +446,7 @@ def joint_tour_participation(
         locals_d=constants,
         trace_label=trace_label,
         trace_choice_name="participation",
-        custom_chooser=participants_chooser,
+        custom_chooser=custom_chooser,
         estimator=estimator,
         compute_settings=model_settings.compute_settings,
     )

activitysim/abm/models/location_choice.py

@@ -15,10 +15,11 @@
     TourLocationComponentSettings,
     TourModeComponentSettings,
 )
+from activitysim.core.exceptions import DuplicateWorkflowTableError
 from activitysim.core.interaction_sample import interaction_sample
 from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
+from activitysim.core.logit import AltsContext
 from activitysim.core.util import reindex
-from activitysim.core.exceptions import DuplicateWorkflowTableError
 
 """
 The school/workplace location model predicts the zones in which various people will
@@ -603,6 +604,7 @@ def run_location_simulate(
     chunk_tag,
     trace_label,
     skip_choice=False,
+    alts_context: AltsContext | None = None,
 ):
     """
     run location model on location_sample annotated with mode_choice logsum
@@ -713,6 +715,7 @@ def run_location_simulate(
         compute_settings=model_settings.compute_settings.subcomponent_settings(
             "simulate"
         ),
+        alts_context=alts_context,
     )
 
     if not want_logsums:
@@ -738,6 +741,7 @@ def run_location_choice(
     chunk_tag,
     trace_label,
     skip_choice=False,
+    alts_context: AltsContext | None = None,
 ):
     """
     Run the three-part location choice algorithm to generate a location choice for each chooser
@@ -757,6 +761,8 @@ def run_location_choice(
     model_settings : dict
     chunk_size : int
     trace_label : str
+    skip_choice : bool
+    alts_context : AltsContext or None
 
     Returns
     -------
@@ -789,6 +795,9 @@ def run_location_choice(
         if choosers.shape[0] == 0:
             logger.info(f"{trace_label} skipping segment {segment_name}: no choosers")
             continue
+        # dest_size_terms contains 0-attraction zones so using this directly here, important for stable error terms
+        # when a zone goes from 0 base -> nonzero project
+        alts_context = AltsContext.from_series(dest_size_terms.index)
 
         # - location_sample
         location_sample_df = run_location_sample(
@@ -842,6 +851,7 @@ def run_location_choice(
                 trace_label, "simulate.%s" % segment_name
             ),
             skip_choice=skip_choice,
+            alts_context=alts_context,
         )
 
         if estimator:
@@ -1020,6 +1030,10 @@ def iterate_location_choice(
     ) = None  # initialize to None, will be populated in first iteration
 
     for iteration in range(1, max_iterations + 1):
+        # Force reset the setting at the start of each shadow price iteration for consistency
+        logger.info("Resetting random number seeds for iteration {}".format(iteration))
+        state.get_rn_generator().end_step(trace_label)
+        state.get_rn_generator().begin_step(trace_label)
         persons_merged_df_ = persons_merged_df.copy()
 
         if spc.use_shadow_pricing and iteration > 1:
@@ -1032,6 +1046,17 @@ def iterate_location_choice(
                 ]
                 persons_merged_df_ = persons_merged_df_.sort_index()
 
+        # reset rng offsets to identical state on each iteration. This ensures that the same set of random numbers is
+        # used on each iteration. Note this has to happen AFTER updating shadow prices because the simulation method
+        # draws random numbers.
+        # Only applying when using EET for now because this will need changes to integration
+        # tests, but it's probably a good idea for MC simulation as well.
+        if state.settings.use_explicit_error_terms and iteration > 1:
+            logger.debug(
+                f"{trace_label} resetting random number generator offsets for iteration {iteration}"
+            )
+            state.get_rn_generator().reset_offsets_for_step(state.current_model_name)
+
         choices_df_, save_sample_df = run_location_choice(
             state,
             persons_merged_df_,

activitysim/abm/models/parking_location_choice.py

@@ -21,6 +21,7 @@
 from activitysim.core.configuration.base import PreprocessorSettings
 from activitysim.core.configuration.logit import LogitComponentSettings
 from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
+from activitysim.core.logit import AltsContext
 from activitysim.core.tracing import print_elapsed_time
 from activitysim.core.util import assign_in_place, drop_unused_columns
 from activitysim.core.exceptions import DuplicateWorkflowTableError
@@ -112,6 +113,7 @@ def parking_destination_simulate(
     chunk_size,
     trace_hh_id,
     trace_label,
+    alts_context: AltsContext | None = None,
 ):
     """
     Chose destination from destination_sample (with od_logsum and dp_logsum columns added)
@@ -150,6 +152,7 @@ def parking_destination_simulate(
         trace_label=trace_label,
         trace_choice_name="parking_loc",
         explicit_chunk_size=model_settings.explicit_chunk,
+        alts_context=alts_context,
     )
 
     # drop any failed zero_prob destinations
@@ -211,6 +214,9 @@ def choose_parking_location(
     )
     destination_sample.index = np.repeat(trips.index.values, len(alternatives))
     destination_sample.index.name = trips.index.name
+    # use full land_use index to ensure AltsContext spans full range of potential zones
+    land_use = state.get_dataframe("land_use")
+    alts_context = AltsContext.from_series(land_use.index)
 
     destinations = parking_destination_simulate(
         state,
@@ -223,6 +229,7 @@ def choose_parking_location(
         chunk_size=chunk_size,
         trace_hh_id=trace_hh_id,
         trace_label=trace_label,
+        alts_context=alts_context,
     )
 
     if want_sample_table: