[Optimization] Accelerate the speed of tokenizer. (#7544)

* Change default workers and max-concurrency when launch api-server

* Change convert_tokens_to_ids to encode to get token ids

---------

Co-authored-by: zhangxiao35 <zhangxiao35@baidu.com>
Co-authored-by: “liuruian” <liuruian@baidu.com>

Author: 3 people

fastdeploy/input/base_processor.py

@@ -163,8 +163,19 @@ def messages2ids(self, request, **kwargs):
         )
         request["prompt_tokens"] = spliced_message
         req_id = request.get("request_id", None) if isinstance(request, dict) else None
-        tokens = self.tokenizer.tokenize(spliced_message)
-        token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
+        if self.tokenizer_type == "ernie4_5":
+            # NOTE: ernie4_5 tokenizer will hang when meet long input when use .encode()
+            token_ids = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(spliced_message))
+        else:
+            token_ids = self.tokenizer.encode(spliced_message, add_special_tokens=False)
+            if hasattr(token_ids, "input_ids") or (isinstance(token_ids, dict) and "input_ids" in token_ids):
+                token_ids = token_ids["input_ids"]
+                if hasattr(token_ids, "ndim") and token_ids.ndim > 1:
+                    token_ids = token_ids[0]
+            if hasattr(token_ids, "tolist"):
+                token_ids = token_ids.tolist()
+            if not isinstance(token_ids, list):
+                token_ids = list(token_ids)
         log_request(
             level=1,
             message="req_id:{req_id}, token_ids: {token_ids}",

tests/input/test_text_processor.py

@@ -77,6 +77,9 @@ def tokenize(self, text):
     def convert_tokens_to_ids(self, tokens):
         return [self._value(token) for token in tokens]
 
+    def encode(self, text, add_special_tokens=True, **kwargs):
+        return self.convert_tokens_to_ids(self.tokenize(text))
+
     def decode(self, token_ids, **kwargs):
         return " ".join(str(t) for t in token_ids)
 
@@ -387,6 +390,89 @@ def test_process_request_dict_messages_template(self):
         self.assertTrue(processed["enable_thinking"])
         self.assertEqual(processed["prompt_tokens"], "system prompt hello")
 
+    def test_process_request_dict_messages_template_batch_encoding(self):
+        """encode() 返回 BatchEncoding-like 对象时，messages2ids 应正确提取 input_ids"""
+
+        class BatchEncodingLike:
+            """模拟 HuggingFace BatchEncoding (UserDict 子类，hasattr input_ids = True)"""
+
+            def __init__(self, ids):
+                self.input_ids = ids
+
+            def __getitem__(self, key):
+                return getattr(self, key)
+
+        class BatchEncodingTokenizer(DummyTokenizer):
+            def encode(self, text, add_special_tokens=True, **kwargs):
+                return BatchEncodingLike([len(text)])
+
+        module = self.text_processor_module
+        processor = module.DataProcessor("stub-model")
+        processor.tokenizer = BatchEncodingTokenizer()
+
+        request = {
+            "request_id": "chat",
+            "messages": [{"role": "user", "content": "hello"}],
+            "chat_template_kwargs": {"system": "system prompt"},
+        }
+        processed = processor.process_request_dict(request, max_model_len=100)
+        token_ids = processed["prompt_token_ids"]
+        self.assertIsInstance(token_ids, list)
+        self.assertTrue(all(isinstance(x, int) for x in token_ids))
+
+    def test_process_request_dict_messages_template_tensor(self):
+        """encode() 返回带 tolist() 的 tensor-like 对象时，messages2ids 应正确转换为 list"""
+
+        class TensorLike:
+            """模拟 numpy/paddle/torch tensor，有 tolist() 方法"""
+
+            def __init__(self, ids):
+                self._ids = ids
+
+            def tolist(self):
+                return self._ids
+
+        class TensorTokenizer(DummyTokenizer):
+            def encode(self, text, add_special_tokens=True, **kwargs):
+                return TensorLike([len(text)])
+
+        module = self.text_processor_module
+        processor = module.DataProcessor("stub-model")
+        processor.tokenizer = TensorTokenizer()
+
+        request = {
+            "request_id": "chat",
+            "messages": [{"role": "user", "content": "hello"}],
+            "chat_template_kwargs": {"system": "system prompt"},
+        }
+        processed = processor.process_request_dict(request, max_model_len=100)
+        token_ids = processed["prompt_token_ids"]
+        self.assertIsInstance(token_ids, list)
+        self.assertTrue(all(isinstance(x, int) for x in token_ids))
+
+    def test_process_request_dict_messages_template_plain_dict(self):
+        """encode() 返回 plain dict 时，messages2ids 应正确提取 input_ids 而非返回 key 列表"""
+
+        class PlainDictTokenizer(DummyTokenizer):
+            def encode(self, text, add_special_tokens=True, **kwargs):
+                return {"input_ids": [len(text)], "attention_mask": [1]}
+
+        module = self.text_processor_module
+        processor = module.DataProcessor("stub-model")
+        processor.tokenizer = PlainDictTokenizer()
+
+        request = {
+            "request_id": "chat",
+            "messages": [{"role": "user", "content": "hello"}],
+            "chat_template_kwargs": {"system": "system prompt"},
+        }
+        processed = processor.process_request_dict(request, max_model_len=100)
+        token_ids = processed["prompt_token_ids"]
+        self.assertIsInstance(token_ids, list)
+        self.assertTrue(all(isinstance(x, int) for x in token_ids))
+        # 确保不是 key 列表 ['input_ids', 'attention_mask']
+        self.assertNotIn("input_ids", token_ids)
+
     def test_process_request_dict_handles_sequences(self):
         request = {
             "prompt": [1, 2, 3, 4, 5, 6],