操作系统及版本
ubuntu22.04
安装工具的python环境
docker容器中的python环境
python版本
3.11
AISBench工具版本
3.1.20260415
AISBench执行命令
ais_bench --models vllm_api_general_chat --datasets omnidocbench_gen --debug >out/omnidocbench_gen.log 2>&1
模型配置文件或自定义配置文件内容
from ais_bench.benchmark.models import VLLMCustomAPIChat
from ais_bench.benchmark.utils.postprocess.model_postprocessors import extract_non_reasoning_content
models = [
dict(
attr="service",
type=VLLMCustomAPIChat,
abbr="vllm-api-general-chat",
path="/workspace/data/model/gemma-4-31B-it/gemma-4-31B-it/",
model="gemma-4-31B-it",
stream=False,
request_rate=0,
use_timestamp=False,
retry=2,
api_key="",
host_ip="127.0.0.1",
host_port=8008,
url="",
max_out_len=131072,
batch_size=10,
trust_remote_code=False,
generation_kwargs=dict(
temperature=1.0,
top_p=0.95,
top_k=64,
ignore_eos=False,
),
pred_postprocessor=dict(type=extract_non_reasoning_content),
)
预期行为
正常的精度测试
实际行为
[2026-04-21 05:45:07,639] [ais_bench] [INFO] Loading vllm_api_general_chat: /benchmark/ais_bench/benchmark/configs/./models/vllm_api/vllm_api_general_chat.py
[2026-04-21 05:45:07,644] [ais_bench] [INFO] Loading omnidocbench_gen: /benchmark/ais_bench/benchmark/configs/./datasets/omnidocbench/omnidocbench_gen.py
[2026-04-21 05:45:07,647] [ais_bench] [INFO] Loading example: /benchmark/ais_bench/benchmark/configs/./summarizers/example.py
[2026-04-21 05:45:07,680] [ais_bench] [INFO] Current exp folder: outputs/default/20260421_054455
[2026-04-21 05:45:07,751] [ais_bench] [INFO] Starting inference tasks...
[2026-04-21 05:45:07,755] [ais_bench] [INFO] Partitioned into 1 tasks.
[2026-04-21 05:45:07,783] [ais_bench] [INFO] Launch TasksMonitor, PID: 13845, Refresh interval: 0.5, Run in background: True
[2026-04-21 05:45:21,251] [ais_bench] [INFO] Debug mode, print progress directly
[2026-04-21 05:45:21,252] [ais_bench] [INFO] Task [vllm-api-general-chat/omnidocbench]
Traceback (most recent call last):
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_writer.py", line 243, in arrow_array
out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
File "pyarrow/array.pxi", line 375, in pyarrow.lib.array
File "pyarrow/array.pxi", line 46, in pyarrow.lib._sequence_to_array
File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Could not convert 'box_id_0' with type str: tried to convert to int64
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/benchmark/ais_bench/benchmark/tasks/openicl_api_infer.py", line 704, in
raise e
File "/benchmark/ais_bench/benchmark/tasks/openicl_api_infer.py", line 701, in
inferencer.run(task_state_manager)
File "/benchmark/ais_bench/benchmark/tasks/openicl_api_infer.py", line 480, in run
data_list, finish_data_count, global_indexes = self._get_data_list()
File "/benchmark/ais_bench/benchmark/tasks/openicl_api_infer.py", line 167, in _get_data_list
dataset = build_dataset_from_cfg(dataset_cfg, task_state_manager=self.task_state_manager)
File "/benchmark/ais_bench/benchmark/utils/config/build.py", line 124, in build_dataset_from_cfg
return LOAD_DATASET.build(dataset_cfg)
File "/usr/local/lib/python3.10/dist-packages/mmengine/registry/registry.py", line 570, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
File "/usr/local/lib/python3.10/dist-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
obj = obj_cls(**args) # type: ignore
File "/benchmark/ais_bench/benchmark/datasets/base.py", line 43, in init
self.dataset = self.load(**kwargs)
File "/benchmark/ais_bench/benchmark/datasets/omnidocbench/omnidocbench.py", line 36, in load
return Dataset.from_list(dataset)
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 986, in from_list
return cls.from_dict(mapping, features, info, split)
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 940, in from_dict
pa_table = InMemoryTable.from_pydict(mapping=mapping)
File "/usr/local/lib/python3.10/dist-packages/datasets/table.py", line 758, in from_pydict
return cls(pa.Table.from_pydict(*args, **kwargs))
File "pyarrow/table.pxi", line 1985, in pyarrow.lib._Tabular.from_pydict
File "pyarrow/table.pxi", line 6402, in pyarrow.lib._from_pydict
File "pyarrow/array.pxi", line 405, in pyarrow.lib.asarray
File "pyarrow/array.pxi", line 256, in pyarrow.lib.array
File "pyarrow/array.pxi", line 118, in pyarrow.lib._handle_arrow_array_protocol
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_writer.py", line 311, in arrow_array
out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False))
File "pyarrow/array.pxi", line 375, in pyarrow.lib.array
File "pyarrow/array.pxi", line 46, in pyarrow.lib._sequence_to_array
File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Could not convert 'box_id_0' with type str: tried to convert to int64
[2026-04-21 05:45:28,044] [ais_bench] [INFO] Inference tasks completed.
[2026-04-21 05:45:28,053] [ais_bench] [INFO] Starting evaluation tasks...
[2026-04-21 05:45:28,057] [ais_bench] [INFO] Partitioned into 1 tasks.
[2026-04-21 05:45:28,081] [ais_bench] [INFO] Launch TasksMonitor, PID: 13975, Refresh interval: 0.5, Run in background: True
[2026-04-21 05:45:41,314] [ais_bench] [INFO] Debug mode, print progress directly
Traceback (most recent call last):
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_writer.py", line 243, in arrow_array
out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
File "pyarrow/array.pxi", line 375, in pyarrow.lib.array
File "pyarrow/array.pxi", line 46, in pyarrow.lib._sequence_to_array
File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Could not convert 'box_id_0' with type str: tried to convert to int64
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/benchmark/ais_bench/benchmark/tasks/openicl_eval.py", line 521, in
raise e
File "/benchmark/ais_bench/benchmark/tasks/openicl_eval.py", line 518, in
evaluator.run(task_state_manager)
File "/benchmark/ais_bench/benchmark/tasks/openicl_eval.py", line 99, in run
self._score()
File "/benchmark/ais_bench/benchmark/tasks/openicl_eval.py", line 115, in _score
test_set = build_dataset_from_cfg(self.dataset_cfg, task_state_manager=self.task_state_manager).test
File "/benchmark/ais_bench/benchmark/utils/config/build.py", line 124, in build_dataset_from_cfg
return LOAD_DATASET.build(dataset_cfg)
File "/usr/local/lib/python3.10/dist-packages/mmengine/registry/registry.py", line 570, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
File "/usr/local/lib/python3.10/dist-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
obj = obj_cls(**args) # type: ignore
File "/benchmark/ais_bench/benchmark/datasets/base.py", line 43, in init
self.dataset = self.load(**kwargs)
File "/benchmark/ais_bench/benchmark/datasets/omnidocbench/omnidocbench.py", line 36, in load
return Dataset.from_list(dataset)
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 986, in from_list
return cls.from_dict(mapping, features, info, split)
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 940, in from_dict
pa_table = InMemoryTable.from_pydict(mapping=mapping)
File "/usr/local/lib/python3.10/dist-packages/datasets/table.py", line 758, in from_pydict
return cls(pa.Table.from_pydict(*args, **kwargs))
File "pyarrow/table.pxi", line 1985, in pyarrow.lib._Tabular.from_pydict
File "pyarrow/table.pxi", line 6402, in pyarrow.lib._from_pydict
File "pyarrow/array.pxi", line 405, in pyarrow.lib.asarray
File "pyarrow/array.pxi", line 256, in pyarrow.lib.array
File "pyarrow/array.pxi", line 118, in pyarrow.lib._handle_arrow_array_protocol
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_writer.py", line 311, in arrow_array
out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False))
File "pyarrow/array.pxi", line 375, in pyarrow.lib.array
File "pyarrow/array.pxi", line 46, in pyarrow.lib._sequence_to_array
File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Could not convert 'box_id_0' with type str: tried to convert to int64
前置检查
操作系统及版本
ubuntu22.04
安装工具的python环境
docker容器中的python环境
python版本
3.11
AISBench工具版本
3.1.20260415
AISBench执行命令
ais_bench --models vllm_api_general_chat --datasets omnidocbench_gen --debug >out/omnidocbench_gen.log 2>&1
模型配置文件或自定义配置文件内容
from ais_bench.benchmark.models import VLLMCustomAPIChat
from ais_bench.benchmark.utils.postprocess.model_postprocessors import extract_non_reasoning_content
models = [
dict(
attr="service",
type=VLLMCustomAPIChat,
abbr="vllm-api-general-chat",
path="/workspace/data/model/gemma-4-31B-it/gemma-4-31B-it/",
model="gemma-4-31B-it",
stream=False,
request_rate=0,
use_timestamp=False,
retry=2,
api_key="",
host_ip="127.0.0.1",
host_port=8008,
url="",
max_out_len=131072,
batch_size=10,
trust_remote_code=False,
generation_kwargs=dict(
temperature=1.0,
top_p=0.95,
top_k=64,
ignore_eos=False,
),
pred_postprocessor=dict(type=extract_non_reasoning_content),
)
预期行为
正常的精度测试
实际行为
[2026-04-21 05:45:07,639] [ais_bench] [INFO] Loading vllm_api_general_chat: /benchmark/ais_bench/benchmark/configs/./models/vllm_api/vllm_api_general_chat.py
[2026-04-21 05:45:07,644] [ais_bench] [INFO] Loading omnidocbench_gen: /benchmark/ais_bench/benchmark/configs/./datasets/omnidocbench/omnidocbench_gen.py
[2026-04-21 05:45:07,647] [ais_bench] [INFO] Loading example: /benchmark/ais_bench/benchmark/configs/./summarizers/example.py
[2026-04-21 05:45:07,680] [ais_bench] [INFO] Current exp folder: outputs/default/20260421_054455
[2026-04-21 05:45:07,751] [ais_bench] [INFO] Starting inference tasks...
[2026-04-21 05:45:07,755] [ais_bench] [INFO] Partitioned into 1 tasks.
[2026-04-21 05:45:07,783] [ais_bench] [INFO] Launch TasksMonitor, PID: 13845, Refresh interval: 0.5, Run in background: True
[2026-04-21 05:45:21,251] [ais_bench] [INFO] Debug mode, print progress directly
[2026-04-21 05:45:21,252] [ais_bench] [INFO] Task [vllm-api-general-chat/omnidocbench]
Traceback (most recent call last):
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_writer.py", line 243, in arrow_array
out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
File "pyarrow/array.pxi", line 375, in pyarrow.lib.array
File "pyarrow/array.pxi", line 46, in pyarrow.lib._sequence_to_array
File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Could not convert 'box_id_0' with type str: tried to convert to int64
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/benchmark/ais_bench/benchmark/tasks/openicl_api_infer.py", line 704, in
raise e
File "/benchmark/ais_bench/benchmark/tasks/openicl_api_infer.py", line 701, in
inferencer.run(task_state_manager)
File "/benchmark/ais_bench/benchmark/tasks/openicl_api_infer.py", line 480, in run
data_list, finish_data_count, global_indexes = self._get_data_list()
File "/benchmark/ais_bench/benchmark/tasks/openicl_api_infer.py", line 167, in _get_data_list
dataset = build_dataset_from_cfg(dataset_cfg, task_state_manager=self.task_state_manager)
File "/benchmark/ais_bench/benchmark/utils/config/build.py", line 124, in build_dataset_from_cfg
return LOAD_DATASET.build(dataset_cfg)
File "/usr/local/lib/python3.10/dist-packages/mmengine/registry/registry.py", line 570, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
File "/usr/local/lib/python3.10/dist-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
obj = obj_cls(**args) # type: ignore
File "/benchmark/ais_bench/benchmark/datasets/base.py", line 43, in init
self.dataset = self.load(**kwargs)
File "/benchmark/ais_bench/benchmark/datasets/omnidocbench/omnidocbench.py", line 36, in load
return Dataset.from_list(dataset)
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 986, in from_list
return cls.from_dict(mapping, features, info, split)
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 940, in from_dict
pa_table = InMemoryTable.from_pydict(mapping=mapping)
File "/usr/local/lib/python3.10/dist-packages/datasets/table.py", line 758, in from_pydict
return cls(pa.Table.from_pydict(*args, **kwargs))
File "pyarrow/table.pxi", line 1985, in pyarrow.lib._Tabular.from_pydict
File "pyarrow/table.pxi", line 6402, in pyarrow.lib._from_pydict
File "pyarrow/array.pxi", line 405, in pyarrow.lib.asarray
File "pyarrow/array.pxi", line 256, in pyarrow.lib.array
File "pyarrow/array.pxi", line 118, in pyarrow.lib._handle_arrow_array_protocol
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_writer.py", line 311, in arrow_array
out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False))
File "pyarrow/array.pxi", line 375, in pyarrow.lib.array
File "pyarrow/array.pxi", line 46, in pyarrow.lib._sequence_to_array
File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Could not convert 'box_id_0' with type str: tried to convert to int64
[2026-04-21 05:45:28,044] [ais_bench] [INFO] Inference tasks completed.
[2026-04-21 05:45:28,053] [ais_bench] [INFO] Starting evaluation tasks...
[2026-04-21 05:45:28,057] [ais_bench] [INFO] Partitioned into 1 tasks.
[2026-04-21 05:45:28,081] [ais_bench] [INFO] Launch TasksMonitor, PID: 13975, Refresh interval: 0.5, Run in background: True
[2026-04-21 05:45:41,314] [ais_bench] [INFO] Debug mode, print progress directly
Traceback (most recent call last):
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_writer.py", line 243, in arrow_array
out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
File "pyarrow/array.pxi", line 375, in pyarrow.lib.array
File "pyarrow/array.pxi", line 46, in pyarrow.lib._sequence_to_array
File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Could not convert 'box_id_0' with type str: tried to convert to int64
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/benchmark/ais_bench/benchmark/tasks/openicl_eval.py", line 521, in
raise e
File "/benchmark/ais_bench/benchmark/tasks/openicl_eval.py", line 518, in
evaluator.run(task_state_manager)
File "/benchmark/ais_bench/benchmark/tasks/openicl_eval.py", line 99, in run
self._score()
File "/benchmark/ais_bench/benchmark/tasks/openicl_eval.py", line 115, in _score
test_set = build_dataset_from_cfg(self.dataset_cfg, task_state_manager=self.task_state_manager).test
File "/benchmark/ais_bench/benchmark/utils/config/build.py", line 124, in build_dataset_from_cfg
return LOAD_DATASET.build(dataset_cfg)
File "/usr/local/lib/python3.10/dist-packages/mmengine/registry/registry.py", line 570, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
File "/usr/local/lib/python3.10/dist-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
obj = obj_cls(**args) # type: ignore
File "/benchmark/ais_bench/benchmark/datasets/base.py", line 43, in init
self.dataset = self.load(**kwargs)
File "/benchmark/ais_bench/benchmark/datasets/omnidocbench/omnidocbench.py", line 36, in load
return Dataset.from_list(dataset)
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 986, in from_list
return cls.from_dict(mapping, features, info, split)
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 940, in from_dict
pa_table = InMemoryTable.from_pydict(mapping=mapping)
File "/usr/local/lib/python3.10/dist-packages/datasets/table.py", line 758, in from_pydict
return cls(pa.Table.from_pydict(*args, **kwargs))
File "pyarrow/table.pxi", line 1985, in pyarrow.lib._Tabular.from_pydict
File "pyarrow/table.pxi", line 6402, in pyarrow.lib._from_pydict
File "pyarrow/array.pxi", line 405, in pyarrow.lib.asarray
File "pyarrow/array.pxi", line 256, in pyarrow.lib.array
File "pyarrow/array.pxi", line 118, in pyarrow.lib._handle_arrow_array_protocol
File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_writer.py", line 311, in arrow_array
out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False))
File "pyarrow/array.pxi", line 375, in pyarrow.lib.array
File "pyarrow/array.pxi", line 46, in pyarrow.lib._sequence_to_array
File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Could not convert 'box_id_0' with type str: tried to convert to int64
前置检查