Index A | B | C | D | E | F | G | H | I | K | L | M | N | O | P | R | S | T | U | V A Action (class in crab.core.models) (class in crab.core.models.action) action() (in module crab.core.decorators) action_list (crab.core.models.BackendOutput attribute) action_returns (crab.core.models.benchmark_interface.StepResult attribute) (crab.core.models.StepResult attribute) action_space (crab.core.environment.Environment property) (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) ActionNotFound ActionOutput (class in crab.core.models) adjlist (crab.core.models.GeneratedTask attribute) (crab.core.models.task.GeneratedTask attribute) arguments (crab.core.models.ActionOutput attribute) attribute (crab.core.models.SubTaskInstance attribute) (crab.core.models.task.SubTaskInstance attribute) attribute_dict (crab.core.models.SubTask attribute) (crab.core.models.task.SubTask attribute) B BackendOutput (class in crab.core.models) Benchmark (class in crab.core.benchmark) BenchmarkConfig (class in crab.core.models) (class in crab.core.models.config) C calculate_longest_unfinished_path_length() (crab.core.graph_evaluator.GraphEvaluator method) calculate_step_to_complete() (crab.core.graph_evaluator.GraphEvaluator method) change_evaluator_to_graph() (crab.core.models.Task class method) (crab.core.models.task.Task class method) close() (crab.core.environment.Environment method) close_task() (crab.core.benchmark.Benchmark method) ClosedAction (in module crab.core.models.action) combine() (crab.core.task_generator.TaskGenerator method) combine_subtask_list() (crab.core.task_generator.TaskGenerator method) combine_two_subtasks() (crab.core.task_generator.TaskGenerator method) common_setup (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) compute_radar_stats() (crab.core.graph_evaluator.GraphEvaluator method) crab module crab.benchmarks module crab.benchmarks.template module crab.core module crab.core.benchmark module crab.core.decorators module crab.core.environment module crab.core.exceptions module crab.core.graph_evaluator module crab.core.models module crab.core.models.action module crab.core.models.benchmark_interface module crab.core.models.config module crab.core.models.evaluator module crab.core.models.task module crab.core.task_generator module crab.environments module crab.environments.template module crab.server module crab.server.api module crab.server.config module crab.server.exception_handlers module crab.server.logger module crab.server.main module crab.server.middleware module crab.server.utils module create_benchmark() (in module crab.core.benchmark) create_environment() (in module crab.core.environment) D default_env (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) description (crab.core.environment.Environment attribute) (crab.core.models.Action attribute), [1] (crab.core.models.action.Action attribute), [1] (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) (crab.core.models.Evaluator attribute) (crab.core.models.GeneratedTask attribute) (crab.core.models.SubTask attribute) (crab.core.models.Task attribute) (crab.core.models.task.GeneratedTask attribute) (crab.core.models.task.SubTask attribute) (crab.core.models.task.Task attribute) DISPLAY (crab.server.config.EnvSettings attribute) dump_generated_task() (crab.core.task_generator.TaskGenerator static method) dump_model() (crab.core.models.SubTaskInstance method) (crab.core.models.task.SubTaskInstance method) E EMPTY_MODEL (in module crab.core.models.action) entry (crab.core.models.Action attribute), [1] (crab.core.models.action.Action attribute), [1] (crab.core.models.Evaluator attribute) entry() (crab.core.graph_evaluator.GraphEvaluator method) env (crab.core.models.ActionOutput attribute) env_name (crab.core.models.Action attribute), [1] (crab.core.models.action.Action attribute), [1] (crab.core.models.Evaluator attribute) Environment (class in crab.core.environment) ENVIRONMENT (crab.server.config.Settings attribute) EnvironmentConfig (class in crab.core.models) (class in crab.core.models.config) environments (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) EnvSettings (class in crab.server.config) evaluate() (crab.core.benchmark.Benchmark method) evaluation_results (crab.core.models.benchmark_interface.StepResult attribute) (crab.core.models.StepResult attribute) Evaluator (class in crab.core.models) (class in crab.core.models.evaluator) evaluator (crab.core.models.Task attribute) (crab.core.models.task.Task attribute) evaluator() (in module crab.core.decorators) evaluator_generator (crab.core.models.SubTask attribute) (crab.core.models.task.SubTask attribute) expand_attribute_type() (crab.core.models.SubTask class method) (crab.core.models.task.SubTask class method) export_action_space() (crab.core.benchmark.Benchmark method) extra_action (crab.core.models.SubTask attribute) (crab.core.models.Task attribute) (crab.core.models.task.SubTask attribute) (crab.core.models.task.Task attribute) extra_attributes (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) F from_config() (crab.core.task_generator.TaskGenerator class method) from_function() (crab.core.models.Action class method) (crab.core.models.action.Action class method) G generate_evaluator() (crab.core.task_generator.TaskGenerator static method) generate_length1_all() (in module crab.core.task_generator) generate_length1_by_id() (in module crab.core.task_generator) generate_length2_manual() (in module crab.core.task_generator) generate_single_node_task() (crab.core.task_generator.TaskGenerator static method) GeneratedTask (class in crab.core.models) (class in crab.core.models.task) get_benchmarks_environments() (in module crab.server.utils) get_completeness() (crab.core.graph_evaluator.GraphEvaluator method) get_completeness_per_action() (crab.core.graph_evaluator.GraphEvaluator method) get_env_descriptions() (crab.core.benchmark.Benchmark method) get_instances() (in module crab.server.utils) get_longest_unfinished_path_length() (crab.core.graph_evaluator.GraphEvaluator method) get_next_source_nodes() (crab.core.graph_evaluator.GraphEvaluator method) get_required_params() (crab.core.models.Action method) (crab.core.models.action.Action method) get_step_to_complete() (crab.core.graph_evaluator.GraphEvaluator method) get_task_from_file() (crab.core.task_generator.TaskGenerator method) gpt_choice() (crab.core.task_generator.TaskGenerator method) graph_generation() (crab.core.task_generator.TaskGenerator method) GraphEvaluator (class in crab.core.graph_evaluator) H HOST (crab.server.config.Settings attribute) human_evaluation() (crab.core.benchmark.Benchmark method) I id (crab.core.models.GeneratedTask attribute) (crab.core.models.SubTask attribute) (crab.core.models.SubTaskInstance attribute) (crab.core.models.Task attribute) (crab.core.models.task.GeneratedTask attribute) (crab.core.models.task.SubTask attribute) (crab.core.models.task.SubTaskInstance attribute) (crab.core.models.task.Task attribute) IMAGE_JPG_BASE64 (crab.core.models.MessageType attribute) info (crab.core.models.benchmark_interface.StepResult attribute) (crab.core.models.StepResult attribute) init() (in module crab.server.main) inside_environment (crab.core.models.config.VMEnvironmentConfig attribute) (crab.core.models.VMEnvironmentConfig attribute) is_complete() (crab.core.graph_evaluator.GraphEvaluator method) K kept_params (crab.core.models.Action attribute), [1] (crab.core.models.action.Action attribute), [1] (crab.core.models.Evaluator attribute) L load_subtasks() (in module crab.core.task_generator) local (crab.core.models.Action attribute) (crab.core.models.action.Action attribute) (crab.core.models.Evaluator attribute) log_request_middleware() (in module crab.server.middleware) M main() (in module crab.core.task_generator) message (crab.core.models.BackendOutput attribute) Message (in module crab.core.models) MessageType (class in crab.core.models) model_computed_fields (crab.core.models.Action attribute) (crab.core.models.action.Action attribute) (crab.core.models.ActionOutput attribute) (crab.core.models.BackendOutput attribute) (crab.core.models.benchmark_interface.StepResult attribute) (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.config.VMEnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) (crab.core.models.Evaluator attribute) (crab.core.models.evaluator.Evaluator attribute) (crab.core.models.GeneratedTask attribute) (crab.core.models.StepResult attribute) (crab.core.models.SubTask attribute) (crab.core.models.SubTaskInstance attribute) (crab.core.models.Task attribute) (crab.core.models.task.GeneratedTask attribute) (crab.core.models.task.SubTask attribute) (crab.core.models.task.SubTaskInstance attribute) (crab.core.models.task.Task attribute) (crab.core.models.VMEnvironmentConfig attribute) (crab.server.config.EnvSettings attribute) (crab.server.config.Settings attribute) model_config (crab.core.models.Action attribute) (crab.core.models.action.Action attribute) (crab.core.models.ActionOutput attribute) (crab.core.models.BackendOutput attribute) (crab.core.models.benchmark_interface.StepResult attribute) (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.config.VMEnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) (crab.core.models.Evaluator attribute) (crab.core.models.evaluator.Evaluator attribute) (crab.core.models.GeneratedTask attribute) (crab.core.models.StepResult attribute) (crab.core.models.SubTask attribute) (crab.core.models.SubTaskInstance attribute) (crab.core.models.Task attribute) (crab.core.models.task.GeneratedTask attribute) (crab.core.models.task.SubTask attribute) (crab.core.models.task.SubTaskInstance attribute) (crab.core.models.task.Task attribute) (crab.core.models.VMEnvironmentConfig attribute) (crab.server.config.EnvSettings attribute) (crab.server.config.Settings attribute) model_fields (crab.core.models.Action attribute) (crab.core.models.action.Action attribute) (crab.core.models.ActionOutput attribute) (crab.core.models.BackendOutput attribute) (crab.core.models.benchmark_interface.StepResult attribute) (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.config.VMEnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) (crab.core.models.Evaluator attribute) (crab.core.models.evaluator.Evaluator attribute) (crab.core.models.GeneratedTask attribute) (crab.core.models.StepResult attribute) (crab.core.models.SubTask attribute) (crab.core.models.SubTaskInstance attribute) (crab.core.models.Task attribute) (crab.core.models.task.GeneratedTask attribute) (crab.core.models.task.SubTask attribute) (crab.core.models.task.SubTaskInstance attribute) (crab.core.models.task.Task attribute) (crab.core.models.VMEnvironmentConfig attribute) (crab.server.config.EnvSettings attribute) (crab.server.config.Settings attribute) module crab crab.benchmarks crab.benchmarks.template crab.core crab.core.benchmark crab.core.decorators crab.core.environment crab.core.exceptions crab.core.graph_evaluator crab.core.models crab.core.models.action crab.core.models.benchmark_interface crab.core.models.config crab.core.models.evaluator crab.core.models.task crab.core.task_generator crab.environments crab.environments.template crab.server crab.server.api crab.server.config crab.server.exception_handlers crab.server.logger crab.server.main crab.server.middleware crab.server.utils multienv (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) must_return_bool() (crab.core.models.Evaluator class method) (crab.core.models.evaluator.Evaluator class method) N name (crab.core.environment.Environment attribute) (crab.core.models.Action attribute), [1] (crab.core.models.action.Action attribute), [1] (crab.core.models.ActionOutput attribute) (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) (crab.core.models.Evaluator attribute) O observation_space (crab.core.environment.Environment property) (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) observe() (crab.core.benchmark.Benchmark method) (crab.core.environment.Environment method) observe_with_prompt() (crab.core.benchmark.Benchmark method) (crab.core.environment.Environment method) output (crab.core.models.SubTaskInstance attribute) (crab.core.models.task.SubTaskInstance attribute) output_generator (crab.core.models.SubTask attribute) (crab.core.models.task.SubTask attribute) output_type (crab.core.models.SubTask attribute) (crab.core.models.task.SubTask attribute) P parameters (crab.core.models.Action attribute), [1] (crab.core.models.action.Action attribute), [1] (crab.core.models.Evaluator attribute) parse_args() (in module crab.server.config) PORT (crab.server.config.Settings attribute) prompting_tools (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) R random_walk() (crab.core.task_generator.TaskGenerator method) raw_action() (in module crab.server.api) remote_url (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.config.VMEnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) (crab.core.models.VMEnvironmentConfig attribute) request_validation_exception_handler() (in module crab.server.exception_handlers) require_submit (crab.core.models.Evaluator attribute) (crab.core.models.evaluator.Evaluator attribute) reset (crab.core.models.config.EnvironmentConfig attribute) (crab.core.models.EnvironmentConfig attribute) reset() (crab.core.benchmark.Benchmark method) (crab.core.environment.Environment method) (crab.core.graph_evaluator.GraphEvaluator method) returns (crab.core.models.Action attribute), [1] (crab.core.models.action.Action attribute), [1] (crab.core.models.Evaluator attribute) root_action_space (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) run() (crab.core.models.Action method) (crab.core.models.action.Action method) S set_action() (crab.core.environment.Environment method) set_kept_param() (crab.core.models.Action method) (crab.core.models.action.Action method) Settings (class in crab.server.config) setup (crab.core.models.SubTask attribute) (crab.core.models.Task attribute) (crab.core.models.task.SubTask attribute) (crab.core.models.task.Task attribute) start() (crab.core.environment.Environment method) start_task() (crab.core.benchmark.Benchmark method) stat() (crab.core.graph_evaluator.GraphEvaluator method) step() (crab.core.benchmark.Benchmark method) (crab.core.environment.Environment method) (crab.core.graph_evaluator.GraphEvaluator method) step_limit (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) StepResult (class in crab.core.models) (class in crab.core.models.benchmark_interface) SubTask (class in crab.core.models) (class in crab.core.models.task) SubTaskInstance (class in crab.core.models) (class in crab.core.models.task) T take_action() (crab.core.environment.Environment method) Task (class in crab.core.models) (class in crab.core.models.task) task (crab.core.models.SubTaskInstance attribute) (crab.core.models.task.SubTaskInstance attribute) task_generation() (crab.core.task_generator.TaskGenerator method) TaskGenerator (class in crab.core.task_generator) TaskNotFound tasks (crab.core.models.BenchmarkConfig attribute) (crab.core.models.config.BenchmarkConfig attribute) (crab.core.models.GeneratedTask attribute) (crab.core.models.task.GeneratedTask attribute) teardown (crab.core.models.SubTask attribute) (crab.core.models.Task attribute) (crab.core.models.task.SubTask attribute) (crab.core.models.task.Task attribute) terminated (crab.core.models.benchmark_interface.StepResult attribute) (crab.core.models.StepResult attribute) TEXT (crab.core.models.MessageType attribute) to_list() (crab.core.models.Task class method) (crab.core.models.task.Task class method) to_openai_json_schema() (crab.core.models.Action method) (crab.core.models.action.Action method) to_raw_action() (crab.core.models.Action method) (crab.core.models.action.Action method) trajectory (crab.core.environment.Environment attribute) truncated (crab.core.models.benchmark_interface.StepResult attribute) (crab.core.models.StepResult attribute) U unhandled_exception_handler() (in module crab.server.exception_handlers) update() (crab.core.graph_evaluator.GraphEvaluator method) V visualize() (crab.core.graph_evaluator.GraphEvaluator static method) VMEnvironmentConfig (class in crab.core.models) (class in crab.core.models.config)
