Seldonian Engine documentation

Welcome to the documentation for the Seldonian Engine, the core library of the the Seldonian Machine Learning Toolkit. The source code is hosted on GitHub.

If you are completely new to this library, see the Overview.

Table of contents

• Overview
  • Interface
    • Spec object
    • Dataset object
    • Model object
    • Behavioral constraints
      • Constraint strings
  • What does SA.run() do?
    • Candidate Selection
  • Safety Test
• Getting Started
  • Installation
  • A first tutorial
• API
  • seldonian
    • seldonian.RL
      • seldonian.RL.Agents
        • seldonian.RL.Agents.Agent
          • seldonian.RL.Agents.Agent.Agent
            • Agent
              • Agent.__init__()
              • Agent.__repr__()
              • Agent.choose_action()
              • Agent.get_params()
              • Agent.get_policy()
              • Agent.get_prob_this_action()
              • Agent.set_new_params()
              • Agent.update()
        • seldonian.RL.Agents.Bases
          • seldonian.RL.Agents.Bases.Fourier
            • seldonian.RL.Agents.Bases.Fourier.Fourier
              • Fourier
                • Fourier.__init__()
                • Fourier.__repr__()
                • Fourier.calculate_num_features()
                • Fourier.construct_basis_matrix()
                • Fourier.get_features()
                • Fourier.get_normalized_observation()
        • seldonian.RL.Agents.Discrete_Random_Agent
          • seldonian.RL.Agents.Discrete_Random_Agent.Discrete_Random_Agent
            • Discrete_Random_Agent
              • Discrete_Random_Agent.__init__()
              • Discrete_Random_Agent.__repr__()
              • Discrete_Random_Agent.choose_action()
              • Discrete_Random_Agent.get_params()
              • Discrete_Random_Agent.get_policy()
              • Discrete_Random_Agent.get_prob_this_action()
              • Discrete_Random_Agent.set_new_params()
              • Discrete_Random_Agent.update()
        • seldonian.RL.Agents.Function_Approximators
          • seldonian.RL.Agents.Function_Approximators.Function_Approximator
            • seldonian.RL.Agents.Function_Approximators.Function_Approximator.Function_Approximator
              • Function_Approximator
                • Function_Approximator.__init__()
                • Function_Approximator.__repr__()
                • Function_Approximator.set_new_params()
          • seldonian.RL.Agents.Function_Approximators.Linear_FA
            • seldonian.RL.Agents.Function_Approximators.Linear_FA.Linear_FA
              • Linear_FA
                • Linear_FA.__init__()
                • Linear_FA.__repr__()
                • Linear_FA.set_new_params()
            • seldonian.RL.Agents.Function_Approximators.Linear_FA.Linear_state_action_value_FA
              • Linear_state_action_value_FA
                • Linear_state_action_value_FA.__init__()
                • Linear_state_action_value_FA.__repr__()
                • Linear_state_action_value_FA.set_new_params()
          • seldonian.RL.Agents.Function_Approximators.Table
            • seldonian.RL.Agents.Function_Approximators.Table.construct_Q_Table_From_Env_Description
              • construct_Q_Table_From_Env_Description()
            • seldonian.RL.Agents.Function_Approximators.Table.Q_Table
              • Q_Table
                • Q_Table.__init__()
                • Q_Table.__repr__()
                • Q_Table.from_environment_state_to_0_indexed_state()
                • Q_Table.get_action_values_given_state()
                • Q_Table.get_action_values_given_zero_indexed_state()
                • Q_Table.set_new_params()
            • seldonian.RL.Agents.Function_Approximators.Table.Table
              • Table
                • Table.__init__()
                • Table.__repr__()
                • Table.from_environment_state_to_0_indexed_state()
                • Table.set_new_params()
        • seldonian.RL.Agents.Parameterized_non_learning_softmax_agent
          • seldonian.RL.Agents.Parameterized_non_learning_softmax_agent.Parameterized_non_learning_softmax_agent
            • Parameterized_non_learning_softmax_agent
              • Parameterized_non_learning_softmax_agent.__init__()
              • Parameterized_non_learning_softmax_agent.__repr__()
              • Parameterized_non_learning_softmax_agent.choose_action()
              • Parameterized_non_learning_softmax_agent.get_action_values()
              • Parameterized_non_learning_softmax_agent.get_params()
              • Parameterized_non_learning_softmax_agent.get_policy()
              • Parameterized_non_learning_softmax_agent.get_prob_this_action()
              • Parameterized_non_learning_softmax_agent.set_new_params()
              • Parameterized_non_learning_softmax_agent.update()
        • seldonian.RL.Agents.Policies
          • seldonian.RL.Agents.Policies.Policy
            • seldonian.RL.Agents.Policies.Policy.Discrete_Action_Policy
              • Discrete_Action_Policy
                • Discrete_Action_Policy.__init__()
                • Discrete_Action_Policy.__repr__()
                • Discrete_Action_Policy.choose_action()
                • Discrete_Action_Policy.construct_basis_and_linear_FA()
                • Discrete_Action_Policy.from_0_indexed_action_to_environment_action()
                • Discrete_Action_Policy.from_environment_action_to_0_indexed_action()
                • Discrete_Action_Policy.get_action_values_given_state()
                • Discrete_Action_Policy.get_params()
                • Discrete_Action_Policy.get_prob_this_action()
                • Discrete_Action_Policy.get_probs_from_observations_and_actions()
                • Discrete_Action_Policy.make_state_action_FA()
                • Discrete_Action_Policy.set_new_params()
            • seldonian.RL.Agents.Policies.Policy.Policy
              • Policy
                • Policy.__init__()
                • Policy.__repr__()
                • Policy.choose_action()
                • Policy.get_params()
                • Policy.get_prob_this_action()
                • Policy.get_probs_from_observations_and_actions()
                • Policy.set_new_params()
          • seldonian.RL.Agents.Policies.SimglucosePolicyFixedArea
            • seldonian.RL.Agents.Policies.SimglucosePolicyFixedArea.SigmoidPolicyFixedArea
              • SigmoidPolicyFixedArea
                • SigmoidPolicyFixedArea.__init__()
                • SigmoidPolicyFixedArea.__repr__()
                • SigmoidPolicyFixedArea.choose_action()
                • SigmoidPolicyFixedArea.get_params()
                • SigmoidPolicyFixedArea.get_prob_this_action()
                • SigmoidPolicyFixedArea.get_probs_from_observations_and_actions()
                • SigmoidPolicyFixedArea.set_new_params()
                • SigmoidPolicyFixedArea.theta2crcf()
          • seldonian.RL.Agents.Policies.Softmax
            • seldonian.RL.Agents.Policies.Softmax.DiscreteSoftmax
              • DiscreteSoftmax
                • DiscreteSoftmax.__init__()
                • DiscreteSoftmax.__repr__()
                • DiscreteSoftmax._arg()
                • DiscreteSoftmax._denom()
                • DiscreteSoftmax.choose_action()
                • DiscreteSoftmax.choose_action_from_action_values()
                • DiscreteSoftmax.construct_basis_and_linear_FA()
                • DiscreteSoftmax.from_0_indexed_action_to_environment_action()
                • DiscreteSoftmax.from_environment_action_to_0_indexed_action()
                • DiscreteSoftmax.get_action_probs_from_action_values()
                • DiscreteSoftmax.get_action_values_given_state()
                • DiscreteSoftmax.get_e_to_the_something_terms()
                • DiscreteSoftmax.get_params()
                • DiscreteSoftmax.get_prob_this_action()
                • DiscreteSoftmax.get_probs_from_observations_and_actions()
                • DiscreteSoftmax.make_state_action_FA()
                • DiscreteSoftmax.set_new_params()
            • seldonian.RL.Agents.Policies.Softmax.MixedSoftmax
              • MixedSoftmax
                • MixedSoftmax.__init__()
                • MixedSoftmax.__repr__()
                • MixedSoftmax.choose_action()
                • MixedSoftmax.choose_action_from_action_values()
                • MixedSoftmax.construct_basis_and_linear_FA()
                • MixedSoftmax.from_0_indexed_action_to_environment_action()
                • MixedSoftmax.from_environment_action_to_0_indexed_action()
                • MixedSoftmax.get_action_probs_from_action_values()
                • MixedSoftmax.get_action_values_given_state()
                • MixedSoftmax.get_e_to_the_something_terms()
                • MixedSoftmax.get_params()
                • MixedSoftmax.get_prob_this_action()
                • MixedSoftmax.get_probs_from_observations_and_actions()
                • MixedSoftmax.make_state_action_FA()
                • MixedSoftmax.set_new_params()
            • seldonian.RL.Agents.Policies.Softmax.Softmax
              • Softmax
                • Softmax.__init__()
                • Softmax.__repr__()
                • Softmax.choose_action()
                • Softmax.choose_action_from_action_values()
                • Softmax.construct_basis_and_linear_FA()
                • Softmax.from_0_indexed_action_to_environment_action()
                • Softmax.from_environment_action_to_0_indexed_action()
                • Softmax.get_action_probs_from_action_values()
                • Softmax.get_action_values_given_state()
                • Softmax.get_e_to_the_something_terms()
                • Softmax.get_params()
                • Softmax.get_prob_this_action()
                • Softmax.get_probs_from_observations_and_actions()
                • Softmax.make_state_action_FA()
                • Softmax.set_new_params()
        • seldonian.RL.Agents.keyboard_gridworld
          • seldonian.RL.Agents.keyboard_gridworld.Keyboard_gridworld
            • Keyboard_gridworld
              • Keyboard_gridworld.__init__()
              • Keyboard_gridworld.__repr__()
              • Keyboard_gridworld.choose_action()
              • Keyboard_gridworld.get_params()
              • Keyboard_gridworld.get_policy()
              • Keyboard_gridworld.get_prob_this_action()
              • Keyboard_gridworld.set_new_params()
              • Keyboard_gridworld.update()
        • seldonian.RL.Agents.mountain_car_rough_solution
          • seldonian.RL.Agents.mountain_car_rough_solution.Mountain_car_rough_solution
            • Mountain_car_rough_solution
              • Mountain_car_rough_solution.__init__()
              • Mountain_car_rough_solution.__repr__()
              • Mountain_car_rough_solution.choose_action()
              • Mountain_car_rough_solution.get_params()
              • Mountain_car_rough_solution.get_policy()
              • Mountain_car_rough_solution.get_prob_this_action()
              • Mountain_car_rough_solution.set_new_params()
              • Mountain_car_rough_solution.update()
        • seldonian.RL.Agents.simglucose_custom_fixedarea_random_agent
          • seldonian.RL.Agents.simglucose_custom_fixedarea_random_agent.SimglucoseFixedAreaAgent
            • SimglucoseFixedAreaAgent
              • SimglucoseFixedAreaAgent.__init__()
              • SimglucoseFixedAreaAgent.__repr__()
              • SimglucoseFixedAreaAgent.choose_action()
              • SimglucoseFixedAreaAgent.get_params()
              • SimglucoseFixedAreaAgent.get_policy()
              • SimglucoseFixedAreaAgent.get_prob_this_action()
              • SimglucoseFixedAreaAgent.set_new_params()
              • SimglucoseFixedAreaAgent.update()
      • seldonian.RL.Env_Description
        • seldonian.RL.Env_Description.Env_Description
          • seldonian.RL.Env_Description.Env_Description.check_space_type
            • check_space_type()
          • seldonian.RL.Env_Description.Env_Description.Env_Description
            • Env_Description
              • Env_Description.__init__()
              • Env_Description.__repr__()
              • Env_Description.get_min_action()
              • Env_Description.get_min_state()
              • Env_Description.get_num_actions()
              • Env_Description.get_num_observation_dims()
              • Env_Description.get_num_states()
        • seldonian.RL.Env_Description.Spaces
          • seldonian.RL.Env_Description.Spaces.Continuous_Space
            • Continuous_Space
              • Continuous_Space.__init__()
              • Continuous_Space.__repr__()
              • Continuous_Space.check_bounds_valid()
              • Continuous_Space.get_num_dims()
          • seldonian.RL.Env_Description.Spaces.Discrete_Space
            • Discrete_Space
              • Discrete_Space.__init__()
              • Discrete_Space.__repr__()
              • Discrete_Space.get_num_values()
      • seldonian.RL.RL_model
        • seldonian.RL.RL_model.RL_model
          • RL_model
            • RL_model.__init__()
            • RL_model.__repr__()
            • RL_model.get_probs_from_observations_and_actions()
      • seldonian.RL.RL_runner
        • seldonian.RL.RL_runner.create_agent_fromdict
          • create_agent_fromdict()
        • seldonian.RL.RL_runner.run_episode
          • run_episode()
        • seldonian.RL.RL_runner.run_episode_from_dict
          • run_episode_from_dict()
        • seldonian.RL.RL_runner.run_episodes_par
          • run_episodes_par()
        • seldonian.RL.RL_runner.run_trial
          • run_trial()
        • seldonian.RL.RL_runner.run_trial_given_agent_and_env
          • run_trial_given_agent_and_env()
      • seldonian.RL.environments
        • seldonian.RL.environments.Environment
          • seldonian.RL.environments.Environment.Environment
            • Environment
              • Environment.__init__()
              • Environment.__repr__()
              • Environment.get_env_description()
              • Environment.get_observation()
              • Environment.reset()
              • Environment.start_visualizing()
              • Environment.stop_visualizing()
              • Environment.terminated()
              • Environment.transition()
              • Environment.visualize()
        • seldonian.RL.environments.gridworld
          • seldonian.RL.environments.gridworld.Gridworld
            • Gridworld
              • Gridworld.__init__()
              • Gridworld.__repr__()
              • Gridworld.create_env_description()
              • Gridworld.get_env_description()
              • Gridworld.get_observation()
              • Gridworld.is_in_goal_state()
              • Gridworld.reset()
              • Gridworld.start_visualizing()
              • Gridworld.stop_visualizing()
              • Gridworld.terminated()
              • Gridworld.transition()
              • Gridworld.update_position()
              • Gridworld.visualize()
        • seldonian.RL.environments.mountaincar
          • seldonian.RL.environments.mountaincar.Mountaincar
            • Mountaincar
              • Mountaincar.__init__()
              • Mountaincar.__repr__()
              • Mountaincar.check_valid_mc_action()
              • Mountaincar.create_env_description()
              • Mountaincar.get_env_description()
              • Mountaincar.get_observation()
              • Mountaincar.position_and_termination_update()
              • Mountaincar.reset()
              • Mountaincar.start_visualizing()
              • Mountaincar.stop_visualizing()
              • Mountaincar.terminated()
              • Mountaincar.transition()
              • Mountaincar.update_velocity()
              • Mountaincar.visualize()
        • seldonian.RL.environments.n_step_mountaincar
          • seldonian.RL.environments.n_step_mountaincar.N_step_mountaincar
            • N_step_mountaincar
              • N_step_mountaincar.__init__()
              • N_step_mountaincar.__repr__()
              • N_step_mountaincar.get_env_description()
              • N_step_mountaincar.get_observation()
              • N_step_mountaincar.reset()
              • N_step_mountaincar.start_visualizing()
              • N_step_mountaincar.stop_visualizing()
              • N_step_mountaincar.terminated()
              • N_step_mountaincar.transition()
              • N_step_mountaincar.visualize()
    • seldonian.candidate_selection
      • seldonian.candidate_selection.candidate_selection
        • seldonian.candidate_selection.candidate_selection.CandidateSelection
          • CandidateSelection
            • CandidateSelection.__init__()
            • CandidateSelection.__repr__()
            • CandidateSelection.calculate_batches()
            • CandidateSelection.calculate_batches_addl_datasets()
            • CandidateSelection.evaluate_primary_objective()
            • CandidateSelection.get_constraint_upper_bounds()
            • CandidateSelection.get_importance_weights()
            • CandidateSelection.objective_with_barrier()
            • CandidateSelection.precalculate_addl_dataset_batch_indices()
            • CandidateSelection.run()
    • seldonian.dataset
      • seldonian.dataset.load_RL_metadata
        • load_RL_metadata()
      • seldonian.dataset.load_custom_metadata
        • load_custom_metadata()
      • seldonian.dataset.load_supervised_metadata
        • load_supervised_metadata()
      • seldonian.dataset.CustomDataSet
        • CustomDataSet
          • CustomDataSet.__init__()
          • CustomDataSet.__repr__()
      • seldonian.dataset.CustomMetaData
        • CustomMetaData
          • CustomMetaData.__init__()
          • CustomMetaData.__repr__()
      • seldonian.dataset.DataSet
        • DataSet
          • DataSet.__init__()
          • DataSet.__repr__()
      • seldonian.dataset.DataSetLoader
        • DataSetLoader
          • DataSetLoader.__init__()
          • DataSetLoader.__repr__()
          • DataSetLoader.load_RL_dataset_from_csv()
          • DataSetLoader.load_RL_dataset_from_episode_file()
          • DataSetLoader.load_supervised_dataset()
      • seldonian.dataset.Episode
        • Episode
          • Episode.__init__()
          • Episode.__repr__()
      • seldonian.dataset.MetaData
        • MetaData
          • MetaData.__init__()
          • MetaData.__repr__()
      • seldonian.dataset.RLDataSet
        • RLDataSet
          • RLDataSet.__init__()
          • RLDataSet.__repr__()
      • seldonian.dataset.RLMetaData
        • RLMetaData
          • RLMetaData.__init__()
          • RLMetaData.__repr__()
      • seldonian.dataset.SupervisedDataSet
        • SupervisedDataSet
          • SupervisedDataSet.__init__()
          • SupervisedDataSet.__repr__()
      • seldonian.dataset.SupervisedMetaData
        • SupervisedMetaData
          • SupervisedMetaData.__init__()
          • SupervisedMetaData.__repr__()
    • seldonian.hyperparam_search
      • seldonian.hyperparam_search.HyperSchema
        • HyperSchema
          • HyperSchema.__init__()
          • HyperSchema.__repr__()
          • HyperSchema._validate()
      • seldonian.hyperparam_search.HyperparamSearch
        • HyperparamSearch
          • HyperparamSearch.__init__()
          • HyperparamSearch.__repr__()
          • HyperparamSearch._get_theta_init_from_hyper_dict()
          • HyperparamSearch._unpack_theta_to_hyperparam_values()
          • HyperparamSearch.aggregate_est_prob_pass()
          • HyperparamSearch.candidate_safety_combine()
          • HyperparamSearch.candidate_safety_split()
          • HyperparamSearch.cmaes_objective()
          • HyperparamSearch.create_bootstrap_trial_spec()
          • HyperparamSearch.create_dataset()
          • HyperparamSearch.find_best_frac_data_in_safety()
          • HyperparamSearch.find_best_hyperparameters()
          • HyperparamSearch.generate_all_bootstrap_datasets()
          • HyperparamSearch.get_all_greater_est_prob_pass()
          • HyperparamSearch.get_bootstrap_dataset_size()
          • HyperparamSearch.get_est_prob_pass()
          • HyperparamSearch.get_gridsearchable_hyperparameter_iterator()
          • HyperparamSearch.get_safety_size()
          • HyperparamSearch.powell_objective()
          • HyperparamSearch.run_bootstrap_trial()
          • HyperparamSearch.run_cmaes()
          • HyperparamSearch.run_powell()
    • seldonian.models
      • seldonian.models.custom_text_model
        • seldonian.models.custom_text_model.CustomTextModel
          • CustomTextModel
            • CustomTextModel.__init__()
            • CustomTextModel.__repr__()
            • CustomTextModel.predict()
      • seldonian.models.models
        • seldonian.models.models.BaseLogisticRegressionModel
          • BaseLogisticRegressionModel
            • BaseLogisticRegressionModel.__init__()
            • BaseLogisticRegressionModel.__repr__()
            • BaseLogisticRegressionModel.fit()
        • seldonian.models.models.BinaryLogisticRegressionModel
          • BinaryLogisticRegressionModel
            • BinaryLogisticRegressionModel.__init__()
            • BinaryLogisticRegressionModel.__repr__()
            • BinaryLogisticRegressionModel.fit()
            • BinaryLogisticRegressionModel.predict()
        • seldonian.models.models.BoundedLinearRegressionModel
          • BoundedLinearRegressionModel
            • BoundedLinearRegressionModel.__init__()
            • BoundedLinearRegressionModel.__repr__()
            • BoundedLinearRegressionModel.fit()
            • BoundedLinearRegressionModel.predict()
        • seldonian.models.models.ClassificationModel
          • ClassificationModel
            • ClassificationModel.__init__()
            • ClassificationModel.__repr__()
        • seldonian.models.models.DummyClassifierModel
          • DummyClassifierModel
            • DummyClassifierModel.__init__()
            • DummyClassifierModel.__repr__()
            • DummyClassifierModel.predict()
        • seldonian.models.models.LinearRegressionModel
          • LinearRegressionModel
            • LinearRegressionModel.__init__()
            • LinearRegressionModel.__repr__()
            • LinearRegressionModel.fit()
            • LinearRegressionModel.predict()
        • seldonian.models.models.LinearRegressionModelListFeatures
          • LinearRegressionModelListFeatures
            • LinearRegressionModelListFeatures.__init__()
            • LinearRegressionModelListFeatures.__repr__()
            • LinearRegressionModelListFeatures.fit()
            • LinearRegressionModelListFeatures.predict()
        • seldonian.models.models.MultiClassLogisticRegressionModel
          • MultiClassLogisticRegressionModel
            • MultiClassLogisticRegressionModel.__init__()
            • MultiClassLogisticRegressionModel.__repr__()
            • MultiClassLogisticRegressionModel.fit()
            • MultiClassLogisticRegressionModel.predict()
        • seldonian.models.models.RandomClassifierModel
          • RandomClassifierModel
            • RandomClassifierModel.__init__()
            • RandomClassifierModel.__repr__()
            • RandomClassifierModel.predict()
        • seldonian.models.models.RegressionModel
          • RegressionModel
            • RegressionModel.__init__()
            • RegressionModel.__repr__()
        • seldonian.models.models.SeldonianModel
          • SeldonianModel
            • SeldonianModel.__init__()
            • SeldonianModel.__repr__()
        • seldonian.models.models.SupervisedModel
          • SupervisedModel
            • SupervisedModel.__init__()
            • SupervisedModel.__repr__()
      • seldonian.models.objectives
        • seldonian.models.objectives.Error_Rate
          • Error_Rate()
        • seldonian.models.objectives.False_Negative_Rate
          • False_Negative_Rate()
        • seldonian.models.objectives.False_Positive_Rate
          • False_Positive_Rate()
        • seldonian.models.objectives.IS_estimate
          • IS_estimate()
        • seldonian.models.objectives.Mean_Error
          • Mean_Error()
        • seldonian.models.objectives.Mean_Squared_Error
          • Mean_Squared_Error()
        • seldonian.models.objectives.Negative_Rate
          • Negative_Rate()
        • seldonian.models.objectives.PDIS_estimate
          • PDIS_estimate()
        • seldonian.models.objectives.Positive_Rate
          • Positive_Rate()
        • seldonian.models.objectives.True_Negative_Rate
          • True_Negative_Rate()
        • seldonian.models.objectives.True_Positive_Rate
          • True_Positive_Rate()
        • seldonian.models.objectives.US_estimate
          • US_estimate()
        • seldonian.models.objectives.WIS_estimate
          • WIS_estimate()
        • seldonian.models.objectives.binary_logistic_loss
          • binary_logistic_loss()
        • seldonian.models.objectives.confusion_matrix
          • confusion_matrix()
        • seldonian.models.objectives.gradient_Bounded_Squared_Error
          • gradient_Bounded_Squared_Error()
        • seldonian.models.objectives.gradient_Mean_Squared_Error
          • gradient_Mean_Squared_Error()
        • seldonian.models.objectives.gradient_binary_logistic_loss
          • gradient_binary_logistic_loss()
        • seldonian.models.objectives.multiclass_logistic_loss
          • multiclass_logistic_loss()
      • seldonian.models.sklearn_lr
        • seldonian.models.sklearn_lr.SkLearnLinearRegressor
          • SkLearnLinearRegressor
            • SkLearnLinearRegressor.__init__()
            • SkLearnLinearRegressor.__repr__()
            • SkLearnLinearRegressor.backward_pass()
            • SkLearnLinearRegressor.create_model()
            • SkLearnLinearRegressor.forward_pass()
            • SkLearnLinearRegressor.get_model_params()
            • SkLearnLinearRegressor.predict()
            • SkLearnLinearRegressor.update_model_params()
      • seldonian.models.sklearn_model
        • seldonian.models.sklearn_model.sklearn_predict_vjp
          • sklearn_predict_vjp()
        • seldonian.models.sklearn_model.SupervisedSkLearnBaseModel
          • SupervisedSkLearnBaseModel
            • SupervisedSkLearnBaseModel.__init__()
            • SupervisedSkLearnBaseModel.__repr__()
            • SupervisedSkLearnBaseModel.backward_pass()
            • SupervisedSkLearnBaseModel.create_model()
            • SupervisedSkLearnBaseModel.forward_pass()
            • SupervisedSkLearnBaseModel.get_model_params()
            • SupervisedSkLearnBaseModel.predict()
            • SupervisedSkLearnBaseModel.update_model_params()
      • seldonian.models.trees
        • seldonian.models.trees.skrandomforest_model
          • seldonian.models.trees.skrandomforest_model.sklearn_predict_vjp
            • sklearn_predict_vjp()
          • seldonian.models.trees.skrandomforest_model.SeldonianRandomForest
            • SeldonianRandomForest
              • SeldonianRandomForest.__init__()
              • SeldonianRandomForest.__repr__()
              • SeldonianRandomForest.fit()
              • SeldonianRandomForest.forward_pass()
              • SeldonianRandomForest.get_jacobian()
              • SeldonianRandomForest.get_leaf_node_probs()
              • SeldonianRandomForest.predict()
              • SeldonianRandomForest.set_leaf_node_values()
        • seldonian.models.trees.sktree_model
          • seldonian.models.trees.sktree_model.probs2theta
            • probs2theta()
          • seldonian.models.trees.sktree_model.sigmoid
            • sigmoid()
          • seldonian.models.trees.sktree_model.sklearn_predict_vjp
            • sklearn_predict_vjp()
          • seldonian.models.trees.sktree_model.SeldonianDecisionTree
            • SeldonianDecisionTree
              • SeldonianDecisionTree.__init__()
              • SeldonianDecisionTree.__repr__()
              • SeldonianDecisionTree.fit()
              • SeldonianDecisionTree.forward_pass()
              • SeldonianDecisionTree.get_jacobian()
              • SeldonianDecisionTree.get_leaf_node_probs()
              • SeldonianDecisionTree.predict()
              • SeldonianDecisionTree.set_leaf_node_values()
    • seldonian.optimizers
      • seldonian.optimizers.gradient_descent
        • seldonian.optimizers.gradient_descent.gradient_descent_adam
          • gradient_descent_adam()
        • seldonian.optimizers.gradient_descent.setup_gradients
          • setup_gradients()
    • seldonian.parse_tree
      • seldonian.parse_tree.nodes
        • seldonian.parse_tree.nodes.BaseNode
          • BaseNode
            • BaseNode.__init__()
            • BaseNode.__repr__()
            • BaseNode.calculate_bounds()
            • BaseNode.calculate_data_forbound()
            • BaseNode.calculate_value()
            • BaseNode.compute_HC_lowerbound()
            • BaseNode.compute_HC_upper_and_lowerbound()
            • BaseNode.compute_HC_upperbound()
            • BaseNode.mask_data()
            • BaseNode.predict_HC_lowerbound()
            • BaseNode.predict_HC_upper_and_lowerbound()
            • BaseNode.predict_HC_upperbound()
            • BaseNode.zhat()
        • seldonian.parse_tree.nodes.CVaRSQeBaseNode
          • CVaRSQeBaseNode
            • CVaRSQeBaseNode.__init__()
            • CVaRSQeBaseNode.__repr__()
            • CVaRSQeBaseNode.calculate_bounds()
            • CVaRSQeBaseNode.calculate_data_forbound()
            • CVaRSQeBaseNode.calculate_value()
            • CVaRSQeBaseNode.compute_HC_lowerbound()
            • CVaRSQeBaseNode.compute_HC_upper_and_lowerbound()
            • CVaRSQeBaseNode.compute_HC_upperbound()
            • CVaRSQeBaseNode.mask_data()
            • CVaRSQeBaseNode.predict_HC_lowerbound()
            • CVaRSQeBaseNode.predict_HC_upper_and_lowerbound()
            • CVaRSQeBaseNode.predict_HC_upperbound()
            • CVaRSQeBaseNode.zhat()
        • seldonian.parse_tree.nodes.ConfusionMatrixBaseNode
          • ConfusionMatrixBaseNode
            • ConfusionMatrixBaseNode.__init__()
            • ConfusionMatrixBaseNode.__repr__()
            • ConfusionMatrixBaseNode.calculate_bounds()
            • ConfusionMatrixBaseNode.calculate_data_forbound()
            • ConfusionMatrixBaseNode.calculate_value()
            • ConfusionMatrixBaseNode.compute_HC_lowerbound()
            • ConfusionMatrixBaseNode.compute_HC_upper_and_lowerbound()
            • ConfusionMatrixBaseNode.compute_HC_upperbound()
            • ConfusionMatrixBaseNode.mask_data()
            • ConfusionMatrixBaseNode.predict_HC_lowerbound()
            • ConfusionMatrixBaseNode.predict_HC_upper_and_lowerbound()
            • ConfusionMatrixBaseNode.predict_HC_upperbound()
            • ConfusionMatrixBaseNode.zhat()
        • seldonian.parse_tree.nodes.ConstantNode
          • ConstantNode
            • ConstantNode.__init__()
            • ConstantNode.__repr__()
        • seldonian.parse_tree.nodes.InternalNode
          • InternalNode
            • InternalNode.__init__()
            • InternalNode.__repr__()
        • seldonian.parse_tree.nodes.MEDCustomBaseNode
          • MEDCustomBaseNode
            • MEDCustomBaseNode.__init__()
            • MEDCustomBaseNode.__repr__()
            • MEDCustomBaseNode.calculate_bounds()
            • MEDCustomBaseNode.calculate_data_forbound()
            • MEDCustomBaseNode.calculate_value()
            • MEDCustomBaseNode.compute_HC_lowerbound()
            • MEDCustomBaseNode.compute_HC_upper_and_lowerbound()
            • MEDCustomBaseNode.compute_HC_upperbound()
            • MEDCustomBaseNode.mask_data()
            • MEDCustomBaseNode.precalculate_data()
            • MEDCustomBaseNode.predict_HC_lowerbound()
            • MEDCustomBaseNode.predict_HC_upper_and_lowerbound()
            • MEDCustomBaseNode.predict_HC_upperbound()
            • MEDCustomBaseNode.zhat()
        • seldonian.parse_tree.nodes.MultiClassBaseNode
          • MultiClassBaseNode
            • MultiClassBaseNode.__init__()
            • MultiClassBaseNode.__repr__()
            • MultiClassBaseNode.calculate_bounds()
            • MultiClassBaseNode.calculate_data_forbound()
            • MultiClassBaseNode.calculate_value()
            • MultiClassBaseNode.compute_HC_lowerbound()
            • MultiClassBaseNode.compute_HC_upper_and_lowerbound()
            • MultiClassBaseNode.compute_HC_upperbound()
            • MultiClassBaseNode.mask_data()
            • MultiClassBaseNode.predict_HC_lowerbound()
            • MultiClassBaseNode.predict_HC_upper_and_lowerbound()
            • MultiClassBaseNode.predict_HC_upperbound()
            • MultiClassBaseNode.zhat()
        • seldonian.parse_tree.nodes.NewPolicyPerformanceBaseNode
          • NewPolicyPerformanceBaseNode
            • NewPolicyPerformanceBaseNode.__init__()
            • NewPolicyPerformanceBaseNode.__repr__()
            • NewPolicyPerformanceBaseNode.calculate_bounds()
            • NewPolicyPerformanceBaseNode.calculate_data_forbound()
            • NewPolicyPerformanceBaseNode.calculate_value()
            • NewPolicyPerformanceBaseNode.compute_HC_lowerbound()
            • NewPolicyPerformanceBaseNode.compute_HC_upper_and_lowerbound()
            • NewPolicyPerformanceBaseNode.compute_HC_upperbound()
            • NewPolicyPerformanceBaseNode.mask_data()
            • NewPolicyPerformanceBaseNode.predict_HC_lowerbound()
            • NewPolicyPerformanceBaseNode.predict_HC_upper_and_lowerbound()
            • NewPolicyPerformanceBaseNode.predict_HC_upperbound()
            • NewPolicyPerformanceBaseNode.zhat()
        • seldonian.parse_tree.nodes.Node
          • Node
            • Node.__init__()
            • Node.__repr__()
        • seldonian.parse_tree.nodes.RLAltRewardBaseNode
          • RLAltRewardBaseNode
            • RLAltRewardBaseNode.__init__()
            • RLAltRewardBaseNode.__repr__()
            • RLAltRewardBaseNode.calculate_bounds()
            • RLAltRewardBaseNode.calculate_data_forbound()
            • RLAltRewardBaseNode.calculate_value()
            • RLAltRewardBaseNode.compute_HC_lowerbound()
            • RLAltRewardBaseNode.compute_HC_upper_and_lowerbound()
            • RLAltRewardBaseNode.compute_HC_upperbound()
            • RLAltRewardBaseNode.mask_data()
            • RLAltRewardBaseNode.predict_HC_lowerbound()
            • RLAltRewardBaseNode.predict_HC_upper_and_lowerbound()
            • RLAltRewardBaseNode.predict_HC_upperbound()
            • RLAltRewardBaseNode.zhat()
      • seldonian.parse_tree.operators
        • op_mapper
        • not_supported_op_mapper
        • bounds_required_dict
      • seldonian.parse_tree.parse_tree
        • seldonian.parse_tree.parse_tree.make_parse_trees_from_constraints
          • make_parse_trees_from_constraints()
        • seldonian.parse_tree.parse_tree.ParseTree
          • ParseTree
            • ParseTree.__init__()
            • ParseTree.__repr__()
            • ParseTree._abs()
            • ParseTree._add()
            • ParseTree._assign_bounds_helper()
            • ParseTree._assign_deltas_helper()
            • ParseTree._assign_infl_factors_helper()
            • ParseTree._ast2pt_node()
            • ParseTree._ast_tree_helper()
            • ParseTree._div()
            • ParseTree._evaluator_helper()
            • ParseTree._exp()
            • ParseTree._log()
            • ParseTree._max()
            • ParseTree._min()
            • ParseTree._mult()
            • ParseTree._parse_subscript()
            • ParseTree._pow()
            • ParseTree._preprocess_constraint_str()
            • ParseTree._propagate_value()
            • ParseTree._propagator_helper()
            • ParseTree._protect_nan()
            • ParseTree._sub()
            • ParseTree._validate_delta_vector()
            • ParseTree._validate_infl_factors()
            • ParseTree.assign_bounds_needed()
            • ParseTree.assign_deltas()
            • ParseTree.assign_infl_factors()
            • ParseTree.build_tree()
            • ParseTree.create_from_ast()
            • ParseTree.evaluate_constraint()
            • ParseTree.make_viz()
            • ParseTree.make_viz_helper()
            • ParseTree.propagate()
            • ParseTree.propagate_bounds()
            • ParseTree.reset_base_node_dict()
      • seldonian.parse_tree.zhat_funcs
        • measure_function_vector_mapper
        • seldonian.parse_tree.zhat_funcs.batcher
          • batcher()
        • seldonian.parse_tree.zhat_funcs.evaluate_statistic
          • evaluate_statistic()
        • seldonian.parse_tree.zhat_funcs.sample_from_statistic
          • sample_from_statistic()
        • seldonian.parse_tree.zhat_funcs.vector_Accuracy
          • vector_Accuracy()
        • seldonian.parse_tree.zhat_funcs.vector_Error
          • vector_Error()
        • seldonian.parse_tree.zhat_funcs.vector_Error_Rate
          • vector_Error_Rate()
        • seldonian.parse_tree.zhat_funcs.vector_False_Negative_Rate
          • vector_False_Negative_Rate()
        • seldonian.parse_tree.zhat_funcs.vector_False_Positive_Rate
          • vector_False_Positive_Rate()
        • seldonian.parse_tree.zhat_funcs.vector_IS_estimate
          • vector_IS_estimate()
        • seldonian.parse_tree.zhat_funcs.vector_Negative_Rate
          • vector_Negative_Rate()
        • seldonian.parse_tree.zhat_funcs.vector_PDIS_estimate
          • vector_PDIS_estimate()
        • seldonian.parse_tree.zhat_funcs.vector_Positive_Rate
          • vector_Positive_Rate()
        • seldonian.parse_tree.zhat_funcs.vector_Squared_Error
          • vector_Squared_Error()
        • seldonian.parse_tree.zhat_funcs.vector_True_Negative_Rate
          • vector_True_Negative_Rate()
        • seldonian.parse_tree.zhat_funcs.vector_True_Positive_Rate
          • vector_True_Positive_Rate()
        • seldonian.parse_tree.zhat_funcs.vector_WIS_estimate
          • vector_WIS_estimate()
        • seldonian.parse_tree.zhat_funcs.vector_auxiliary_return_US_estimate
          • vector_auxiliary_return_US_estimate()
        • seldonian.parse_tree.zhat_funcs.vector_confusion_matrix
          • vector_confusion_matrix()
    • seldonian.safety_test
      • seldonian.safety_test.safety_test
        • seldonian.safety_test.safety_test.SafetyTest
          • SafetyTest
            • SafetyTest.__init__()
            • SafetyTest.__repr__()
            • SafetyTest.evaluate_primary_objective()
            • SafetyTest.get_importance_weights()
            • SafetyTest.run()
    • seldonian.seldonian_algorithm
      • seldonian.seldonian_algorithm.SeldonianAlgorithm
        • SeldonianAlgorithm
          • SeldonianAlgorithm.__init__()
          • SeldonianAlgorithm.__repr__()
          • SeldonianAlgorithm.candidate_safety_split()
          • SeldonianAlgorithm.candidate_safety_split_addl_datasets()
          • SeldonianAlgorithm.candidate_selection()
          • SeldonianAlgorithm.evaluate_primary_objective()
          • SeldonianAlgorithm.get_cs_result()
          • SeldonianAlgorithm.get_importance_weights()
          • SeldonianAlgorithm.get_st_upper_bounds()
          • SeldonianAlgorithm.parse_trees
          • SeldonianAlgorithm.run()
          • SeldonianAlgorithm.run_candidate_selection()
          • SeldonianAlgorithm.run_safety_test()
          • SeldonianAlgorithm.safety_test()
          • SeldonianAlgorithm.set_initial_solution()
    • seldonian.spec
      • seldonian.spec.createRLSpec
        • createRLSpec()
      • seldonian.spec.createSimpleSupervisedSpec
        • createSimpleSupervisedSpec()
      • seldonian.spec.createSupervisedSpec
        • createSupervisedSpec()
      • seldonian.spec.HyperparameterSelectionSpec
        • HyperparameterSelectionSpec
          • HyperparameterSelectionSpec.__init__()
          • HyperparameterSelectionSpec.__repr__()
      • seldonian.spec.RLSpec
        • RLSpec
          • RLSpec.__init__()
          • RLSpec.__repr__()
          • RLSpec.validate_additional_datasets()
          • RLSpec.validate_custom_datasets()
          • RLSpec.validate_parse_trees()
      • seldonian.spec.Spec
        • Spec
          • Spec.__init__()
          • Spec.__repr__()
          • Spec.validate_additional_datasets()
          • Spec.validate_custom_datasets()
          • Spec.validate_parse_trees()
      • seldonian.spec.SupervisedSpec
        • SupervisedSpec
          • SupervisedSpec.__init__()
          • SupervisedSpec.__repr__()
          • SupervisedSpec.validate_additional_datasets()
          • SupervisedSpec.validate_custom_datasets()
          • SupervisedSpec.validate_parse_trees()
    • seldonian.utils
      • seldonian.utils.RL_utils
        • seldonian.utils.RL_utils.argmax_multi
          • argmax_multi()
        • seldonian.utils.RL_utils.clamp
          • clamp()
        • seldonian.utils.RL_utils.error
          • error()
      • seldonian.utils.hyperparam_utils
        • seldonian.utils.hyperparam_utils.bootstrap_sample_dataset
          • bootstrap_sample_dataset()
        • seldonian.utils.hyperparam_utils.clopper_pearson_bound
          • clopper_pearson_bound()
        • seldonian.utils.hyperparam_utils.create_shuffled_dataset
          • create_shuffled_dataset()
        • seldonian.utils.hyperparam_utils.rebuild_parse_trees
          • rebuild_parse_trees()
        • seldonian.utils.hyperparam_utils.set_spec_with_hyperparam_setting
          • set_spec_with_hyperparam_setting()
        • seldonian.utils.hyperparam_utils.ttest_bound
          • ttest_bound()
      • seldonian.utils.io_utils
        • seldonian.utils.io_utils.cmaes_logger
          • cmaes_logger()
        • seldonian.utils.io_utils.dir_path
          • dir_path()
        • seldonian.utils.io_utils.load_json
          • load_json()
        • seldonian.utils.io_utils.load_pickle
          • load_pickle()
        • seldonian.utils.io_utils.save_json
          • save_json()
        • seldonian.utils.io_utils.save_pickle
          • save_pickle()
      • seldonian.utils.plot_utils
        • seldonian.utils.plot_utils.plot_gradient_descent
          • plot_gradient_descent()
      • seldonian.utils.stats_utils
        • seldonian.utils.stats_utils.custom_cumprod
          • custom_cumprod()
        • seldonian.utils.stats_utils.default_supervised_initial_solution_fn
          • default_supervised_initial_solution_fn()
        • seldonian.utils.stats_utils.softmax
          • softmax()
        • seldonian.utils.stats_utils.stddev
          • stddev()
        • seldonian.utils.stats_utils.tinv
          • tinv()
        • seldonian.utils.stats_utils.weighted_sum_gamma
          • weighted_sum_gamma()
      • seldonian.utils.tutorial_utils
        • seldonian.utils.tutorial_utils.generate_clipped_data
          • generate_clipped_data()
        • seldonian.utils.tutorial_utils.generate_data
          • generate_data()
        • seldonian.utils.tutorial_utils.make_synthetic_regression_dataset
          • make_synthetic_regression_dataset()
    • seldonian.warnings
      • seldonian.warnings.custom_warnings
        • seldonian.warnings.custom_warnings.custom_formatwarning
          • custom_formatwarning()