{"id":9399,"date":"2025-06-06T10:09:28","date_gmt":"2025-06-06T02:09:28","guid":{"rendered":"\/?p=9399"},"modified":"2025-06-06T10:09:28","modified_gmt":"2025-06-06T02:09:28","slug":"vllm%e6%8e%a8%e7%90%86%e5%88%9d%e4%bd%93%e9%aa%8c","status":"publish","type":"post","link":"\/?p=9399","title":{"rendered":"vllm\u63a8\u7406\u521d\u4f53\u9a8c"},"content":{"rendered":"

\u4e00\u3001\u73af\u5883<\/h2>\n

\u7cfb\u7edf\u5df2\u88c5 cuda11.8<\/p>\n

\u6559\u7a0b\u6587\u6863\uff1ahttps:\/\/docs.vllm.com.cn\/en\/latest\/getting_started\/quickstart.html<\/a><\/p>\n

$ conda create -n vllm python=3.12 -y\n$ conda activate vllm\n$ pip install vllm<\/code><\/pre>\n
llamafactory:<\/p>\n
$ git clone --depth 1 https:\/\/github.com\/hiyouga\/LLaMA-Factory.git\n$ cd LLaMA-Factory\n$ pip install -e ".[torch,metrics]"\n$ llamafactory-cli version<\/code><\/pre>\n
\u6a21\u578b\uff1a<\/p>\n
git clone https:\/\/www.modelscope.cn\/baichuan-inc\/baichuan-7B.git<\/code><\/pre>\n
\u4e8c\u3001vllm\u63a8\u7406<\/h2>\n
1.\u63a8\u7406\u793a\u4f8b<\/h3>\n
# \u6587\u4ef6\u60c5\u51b5\uff1a\ntotal 16\ndrwxr-xr-x  4 root root 4096 May 16 01:38 .\/\ndrwxr-xr-x 20 root root 4096 May 16 01:35 ..\/\ndrwxr-xr-x  3 root root 4096 May 16 01:46 baichuan-7B\/\ndrwxr-xr-x 12 root root 4096 May 16 01:38 LLaMA-Factory\/<\/code><\/pre>\n
\"image-20250516015133660\"<\/p>\n
\u51fd\u6570\u63a8\u7406\u4ee3\u7801\uff1a<\/p>\n
from vllm import LLM, SamplingParams\nfrom transformers import AutoTokenizer\n\n# \u521d\u59cb\u5316\u5206\u8bcd\u5668\ntokenizer = AutoTokenizer.from_pretrained(\n    ".\/baichuan-7B\/", \n    trust_remote_code=True,\n    use_fast=False  # \u5fc5\u987b\u7981\u7528fast\u5206\u8bcd\u5668\n)\n\n# \u52a0\u8f7dvLLM\u6a21\u578b\nllm = LLM(\n    model=".\/baichuan-7B\/",  # \u6307\u5b9a\u672c\u5730\u6a21\u578b\u8def\u5f84\n    trust_remote_code=True,\n    dtype="float16",  # \u63a8\u8350\u4f7f\u7528\u534a\u7cbe\u5ea6\n    gpu_memory_utilization=0.9  # \u663e\u5b58\u5229\u7528\u7387\n)\n\n# \u8bbe\u7f6e\u751f\u6210\u53c2\u6570\nsampling_params = SamplingParams(\n    temperature=0.7,\n    top_p=0.9,\n    max_tokens=512,\n    stop_token_ids=[7]\n)<\/code><\/pre>\n
shell\u6267\u884c\u7ed3\u679c\uff1a<\/p>\n
In [1]: from vllm import LLM, SamplingParams\n   ...: from transformers import AutoTokenizer\nINFO 05-16 01:49:08 [__init__.py:239] Automatically detected platform cuda.\n\nIn [2]: # \u521d\u59cb\u5316\u5206\u8bcd\u5668\n   ...: tokenizer = AutoTokenizer.from_pretrained(\n   ...:     ".\/baichuan-7B\/",\n   ...:     trust_remote_code=True,\n   ...:     use_fast=False  # \u5fc5\u987b\u7981\u7528fast\u5206\u8bcd\u5668\n   ...: )\n\nIn [3]: # \u52a0\u8f7dvLLM\u6a21\u578b\n   ...: llm = LLM(\n   ...:     model=".\/baichuan-7B\/",  # \u6307\u5b9a\u672c\u5730\u6a21\u578b\u8def\u5f84\n   ...:     trust_remote_code=True,\n   ...:     dtype="float16",  # \u63a8\u8350\u4f7f\u7528\u534a\u7cbe\u5ea6\n   ...:     gpu_memory_utilization=0.9  # \u663e\u5b58\u5229\u7528\u7387\n   ...: )\nINFO 05-16 01:49:23 [config.py:2968] Downcasting torch.float32 to torch.float16.\nINFO 05-16 01:49:28 [config.py:717] This model supports multiple tasks: {'generate', 'score', 'embed', 'reward', 'classify'}. Defaulting to 'generate'.\nINFO 05-16 01:49:28 [config.py:2003] Chunked prefill is enabled with max_num_batched_tokens=8192.\nWARNING 05-16 01:49:29 [tokenizer.py:251] Using a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead.\nINFO 05-16 01:49:29 [core.py:58] Initializing a V1 LLM engine (v0.8.5.post1) with config: model='.\/baichuan-7B\/', speculative_config=None, tokenizer='.\/baichuan-7B\/', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=4096, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='auto', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=None, served_model_name=.\/baichuan-7B\/, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"level":3,"custom_ops":["none"],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"use_inductor":true,"compile_sizes":[],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":512}\nWARNING 05-16 01:49:29 [utils.py:2522] Methods determine_num_available_blocks,device_config,get_cache_block_size_bytes,initialize_cache not implemented in <vllm.v1.worker.gpu_worker.Worker object at 0x7ff7c5bcc650>\nINFO 05-16 01:49:30 [parallel_state.py:1004] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0\nINFO 05-16 01:49:30 [cuda.py:221] Using Flash Attention backend on V1 engine.\nWARNING 05-16 01:49:30 [topk_topp_sampler.py:69] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.\nINFO 05-16 01:49:30 [gpu_model_runner.py:1329] Starting to load model .\/baichuan-7B\/...\nLoading pt checkpoint shards:   0% Completed | 0\/1 [00:00<?, ?it\/s]\nLoading pt checkpoint shards: 100% Completed | 1\/1 [00:05<00:00,  5.98s\/it]\nLoading pt checkpoint shards: 100% Completed | 1\/1 [00:05<00:00,  5.98s\/it]\n\nINFO 05-16 01:49:36 [loader.py:458] Loading weights took 6.18 seconds\nINFO 05-16 01:49:36 [gpu_model_runner.py:1347] Model loading took 13.0406 GiB and 6.289948 seconds\nINFO 05-16 01:49:41 [backends.py:420] Using cache directory: \/root\/.cache\/vllm\/torch_compile_cache\/d1937f95f2\/rank_0_0 for vLLM's torch.compile\nINFO 05-16 01:49:41 [backends.py:430] Dynamo bytecode transform time: 5.11 s\nINFO 05-16 01:49:44 [backends.py:136] Cache the graph of shape None for later use\nINFO 05-16 01:49:58 [backends.py:148] Compiling a graph for general shape takes 16.58 s\nINFO 05-16 01:50:08 [monitor.py:33] torch.compile takes 21.68 s in total\nINFO 05-16 01:50:09 [kv_cache_utils.py:634] GPU KV cache size: 37,120 tokens\nINFO 05-16 01:50:09 [kv_cache_utils.py:637] Maximum concurrency for 4,096 tokens per request: 9.06x\nINFO 05-16 01:50:35 [gpu_model_runner.py:1686] Graph capturing finished in 26 secs, took 1.42 GiB\nINFO 05-16 01:50:35 [core.py:159] init engine (profile, create kv cache, warmup model) took 59.09 seconds\nINFO 05-16 01:50:35 [core_client.py:439] Core engine process 0 ready.\n\nIn [4]: # \u8bbe\u7f6e\u751f\u6210\u53c2\u6570\n   ...: sampling_params = SamplingParams(\n   ...:     temperature=0.7,\n   ...:     top_p=0.9,\n   ...:     max_tokens=512,\n   ...:     stop_token_ids=[7](@ref)  # \u6839\u636e\u6a21\u578b\u534f\u8bae\u8bbe\u7f6e\u7ec8\u6b62\u7b26\n   ...: )\n   ...: \n  Cell In[4], line 6\n    stop_token_ids=[7](@ref)  # \u6839\u636e\u6a21\u578b\u534f\u8bae\u8bbe\u7f6e\u7ec8\u6b62\u7b26\n                       ^\nSyntaxError: invalid syntax\nIn [5]: # \u8bbe\u7f6e\u751f\u6210\u53c2\u6570\n   ...: sampling_params = SamplingParams(\n   ...:     temperature=0.7,\n   ...:     top_p=0.9,\n   ...:     max_tokens=512,\n   ...:     stop_token_ids=[7]\n   ...:     )\nIn [9]: def generate(prompt):\n   ...:     outputs = llm.generate([prompt], sampling_params)\n   ...:     return outputs\nIn [10]: generate("\u5982\u4f55\u7528Python\u5b9e\u73b0\u5feb\u901f\u6392\u5e8f\uff1f")\nProcessed prompts: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1\/1 [00:01<00:00,  1.53s\/it, est. speed input: 5.22 toks\/s, output: 50.92 toks\/s]\nOut[10]: [RequestOutput(request_id=1, prompt='\u5982\u4f55\u7528Python\u5b9e\u73b0\u5feb\u901f\u6392\u5e8f\uff1f', prompt_token_ids=[10532, 31204, 23749, 3967, 6346, 31650, 31998, 75], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text='\\n\u5feb\u901f\u6392\u5e8f(Quick Sort)\u662f\u5192\u6ce1\u6392\u5e8f\u7684\u6539\u8fdb\u7b97\u6cd5\u3002\u5feb\u901f\u6392\u5e8f\u7b97\u6cd5\u662f\u4e00\u79cd\u5206\u6cbb\u6cd5\uff0c\u901a\u8fc7\u5206\u6cbb\u6cd5\u5c06\u4e00\u4e2a\u6392\u5e8f\u5e8f\u5217\u5206\u5272\u6210\u4e24\u90e8\u5206\uff0c\u7136\u540e\u9012\u5f52\u5730\u5bf9\u8fd9\u4e24\u90e8\u5206\u8fdb\u884c\u6392\u5e8f\uff0c\u540c\u65f6\u5408\u5e76\u6392\u5e8f\u540e\u7684\u5e8f\u5217\uff0c\u76f4\u5230\u6574\u4e2a\u5e8f\u5217\u88ab\u6392\u5e8f\u4e3a\u6b62\u3002Python\u5feb\u901f\u6392\u5e8f\u4ee3\u7801\uff1a', token_ids=[5, 6346, 31650, 31998, 31146, 26017, 694, 793, 31145, 31161, 32715, 32464, 31650, 31998, 31135, 22742, 24773, 73, 6346, 31650, 31998, 24773, 7463, 31221, 31524, 31257, 72, 2141, 31221, 31524, 31257, 31365, 1197, 31650, 31998, 31998, 31896, 31221, 33111, 31217, 31444, 2599, 72, 3904, 32425, 32095, 31218, 31209, 11734, 2599, 1696, 31650, 31998, 72, 2687, 22047, 31650, 31998, 6810, 31998, 31896, 72, 12231, 6938, 31998, 31896, 31395, 31650, 31998, 21046, 73, 23749, 6346, 31650, 31998, 17935, 77, 2], cumulative_logprob=None, logprobs=None, finish_reason=stop, stop_reason=None)], finished=True, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={})]<\/code><\/pre>\n
2.API\u670d\u52a1\u90e8\u7f72<\/h3>\n
vllm serve .\/baichuan-7B\/  --port 8018 --served-model-name baichuan-7b  --chat-template template_baichuan.jinja --trust-remote-code<\/code><\/pre>\n
\u8bf7\u6c42\u8be5\u63a5\u53e3\uff1a<\/p>\n
curl http:\/\/localhost:8018\/v1\/chat\/completions \\\n-H "Content-Type: application\/json" \\\n-d '{\n  "model": "baichuan-7b",\n  "messages": [{"role": "user", "content": "\u4f60\u597d\uff01"}],\n  "temperature": 0.7,\n  "max_tokens": 512\n}'<\/code><\/pre>\n
\u7591\u95ee 1:\u663e\u5b58\u5360\u7528\u95ee\u9898<\/h3>\n
\u6a21\u578b\u6587\u4ef6\u6700\u592714G\uff0c\u4e3a\u4ec0\u4e48\u67e5\u770b\u663e\u5b58\u5360\u7528\u670933\u4e2aG?\u4e00\u4e2a7B\u7684\u6a21\u578b\u5e94\u8be5\u5360\u591a\u5c11\uff1f<\/strong><\/p>\n
# du -sh baichuan-7B\/*\n760K    baichuan-7B \u6a21\u578b\u8bb8\u53ef\u534f\u8bae.pdf\n4.0K    config.json\n4.0K    configuration_baichuan.py\n4.0K    configuration.json\n4.0K    generation_config.json\n4.0K    handler.py\n172K    License Agreement for Baichuan-7B Model.pdf\n28K     modeling_baichuan.py\n14G     pytorch_model.bin\n16K     README.md\n4.0K    special_tokens_map.json\n12K     tokenization_baichuan.py\n4.0K    tokenizer_config.json\n1.1M    tokenizer.model\n\n============================================================\n0 3400387 C    root 33.59GiB   0   0.0   0.5        7:50  \/root\/anaconda3\/envs\/vllm\/bin\/python3.12 <\/code><\/pre>\n
\n
\u55ef\uff0c\u7528\u6237\u7684\u95ee\u9898\u662f\u5173\u4e8e\u4ed6\u4eec\u4e0b\u8f7d\u7684Baichuan-7B\u6a21\u578b\u5728\u672c\u5730\u4f7f\u7528vLLM\u63a8\u7406\u65f6\u663e\u5b58\u5360\u7528\u8fbe\u5230\u4e8633GB\uff0c\u800c\u6a21\u578b\u6587\u4ef6\u672c\u8eab\u53ea\u670914GB\u3002\u4ed6\u4eec\u60f3\u77e5\u9053\u4e3a\u4ec0\u4e48\u663e\u5b58\u5360\u7528\u4f1a\u8fd9\u4e48\u5927\uff0c\u4ee5\u53ca\u4e00\u4e2a7B\u6a21\u578b\u901a\u5e38\u5e94\u8be5\u5360\u591a\u5c11\u663e\u5b58\u3002\u6211\u9700\u8981\u7ed3\u5408\u4e4b\u524d\u63d0\u4f9b\u7684\u641c\u7d22\u7ed3\u679c\u6765\u89e3\u7b54\u8fd9\u4e2a\u95ee\u9898\u3002<\/p>\n
\u9996\u5148\uff0c\u7528\u6237\u63d0\u5230\u6a21\u578b\u6587\u4ef6\u4e2d\u7684pytorch_model.bin\u662f14GB\uff0c\u8fd9\u5e94\u8be5\u5bf9\u5e94\u6a21\u578b\u53c2\u6570\u7684\u5b58\u50a8\u3002\u6839\u636e\u641c\u7d22\u7ed3\u679c\u4e2d\u7684\u4fe1\u606f\uff0c\u6bd4\u5982\u5728[3]\u4e2d\u63d0\u5230\uff0c7B\u6a21\u578b\u5728FP16\u7cbe\u5ea6\u4e0b\u9700\u898114GB\u663e\u5b58\uff0c\u800c[5]\u4e5f\u63d0\u52307B\u6a21\u578b\u5728bf16\u4e0b\u53c2\u6570\u536014GB\u3002\u6240\u4ee5\u6a21\u578b\u53c2\u6570\u672c\u8eab\u786e\u5b9e\u662f14GB\u5de6\u53f3\u3002<\/p>\n
\u4f46\u7528\u6237\u89c2\u5bdf\u5230\u663e\u5b58\u5360\u7528\u9ad8\u8fbe33GB\uff0c\u8fd9\u660e\u663e\u8d85\u8fc7\u4e86\u6a21\u578b\u53c2\u6570\u7684\u5927\u5c0f\u3002\u8fd9\u65f6\u5019\u9700\u8981\u5206\u6790\u663e\u5b58\u7684\u5176\u4ed6\u5360\u7528\u90e8\u5206\u3002\u6839\u636e\u641c\u7d22\u7ed3\u679c\u4e2d\u7684\u5185\u5bb9\uff0c\u663e\u5b58\u5360\u7528\u4e0d\u4ec5\u5305\u62ec\u6a21\u578b\u53c2\u6570\uff0c\u8fd8\u6709\u4e2d\u95f4\u6fc0\u6d3b\u503c\u3001\u4f18\u5316\u5668\u72b6\u6001\u3001KV\u7f13\u5b58\u7b49\u3002\u4f8b\u5982\uff0c\u5728[3]\u4e2d\u8be6\u7ec6\u8bf4\u660e\u4e86\u8bad\u7ec3\u65f6\u7684\u663e\u5b58\u6784\u6210\uff0c\u5305\u62ec\u53c2\u6570\u3001\u68af\u5ea6\u3001\u4f18\u5316\u5668\u72b6\u6001\u548c\u6fc0\u6d3b\u503c\u3002\u867d\u7136\u63a8\u7406\u65f6\u4e0d\u9700\u8981\u68af\u5ea6\uff0c\u4f46\u6fc0\u6d3b\u503c\u548cKV\u7f13\u5b58\u4ecd\u7136\u4f1a\u5360\u7528\u5927\u91cf\u663e\u5b58\u3002<\/p>\n
\u5728\u63a8\u7406\u8fc7\u7a0b\u4e2d\uff0cKV\u7f13\u5b58\u662f\u663e\u5b58\u5360\u7528\u7684\u4e3b\u8981\u56e0\u7d20\u4e4b\u4e00\u3002\u6bcf\u6b21\u751f\u6210\u4e00\u4e2atoken\uff0c\u6a21\u578b\u9700\u8981\u4fdd\u5b58\u4e4b\u524d\u7684\u952e\u503c\u5bf9\uff0c\u968f\u7740\u5e8f\u5217\u957f\u5ea6\u7684\u589e\u52a0\uff0c\u8fd9\u90e8\u5206\u7f13\u5b58\u4f1a\u4e0d\u65ad\u589e\u957f\u3002\u4f8b\u5982\uff0c\u5728[5]\u4e2d\u63d0\u5230\uff0c\u63a8\u7406\u65f6\u7684\u663e\u5b58\u5360\u7528\u5927\u7ea6\u662f\u6a21\u578b\u53c2\u6570\u76841.2\u500d\uff0c\u4f46\u5b9e\u9645\u53ef\u80fd\u66f4\u9ad8\uff0c\u5c24\u5176\u662f\u5f53\u751f\u6210\u957f\u6587\u672c\u65f6\u3002\u6b64\u5916\uff0c[9]\u63d0\u5230vLLM\u7684\u663e\u5b58\u4f18\u5316\u65b9\u6cd5\uff0c\u6bd4\u5982\u4f7f\u7528offload\u548c\u91cf\u5316\uff0c\u4f46\u7528\u6237\u53ef\u80fd\u6ca1\u6709\u542f\u7528\u8fd9\u4e9b\u4f18\u5316\uff0c\u5bfc\u81f4\u663e\u5b58\u5360\u7528\u8f83\u9ad8\u3002<\/p>\n
\u53e6\u5916\uff0c\u7528\u6237\u53ef\u80fd\u6ca1\u6709\u4f7f\u7528\u91cf\u5316\u6280\u672f\u3002\u641c\u7d22\u7ed3\u679c\u4e2d\u7684[1]\u548c[4]\u63d0\u5230\uff0c\u91cf\u5316\u53ef\u4ee5\u663e\u8457\u51cf\u5c11\u663e\u5b58\u5360\u7528\uff0c\u6bd4\u59824-bit\u91cf\u5316\u53ef\u4ee5\u5c067B\u6a21\u578b\u7684\u663e\u5b58\u4ece14GB\u964d\u52303.5GB\u3002\u4f46\u7528\u6237\u53ef\u80fd\u8fd8\u5728\u4f7f\u7528\u9ed8\u8ba4\u7684FP16\u6216BF16\u7cbe\u5ea6\uff0c\u5bfc\u81f4\u663e\u5b58\u5360\u7528\u8f83\u9ad8\u3002<\/p>\n
\u8fd8\u6709\u53ef\u80fd\u662f\u6279\u5904\u7406\u5927\u5c0f\uff08batch size\uff09\u6216\u751f\u6210\u5e8f\u5217\u957f\u5ea6\uff08max_tokens\uff09\u8bbe\u7f6e\u4e0d\u5f53\u3002\u8f83\u5927\u7684batch size\u6216\u8f83\u957f\u7684\u751f\u6210\u5e8f\u5217\u4f1a\u589e\u52a0KV\u7f13\u5b58\u7684\u5927\u5c0f\uff0c\u4ece\u800c\u5360\u7528\u66f4\u591a\u663e\u5b58\u3002\u4f8b\u5982\uff0c\u5728[9]\u4e2d\u5efa\u8bae\u8c03\u6574max_batch_size\u548cmax_new_tokens\u6765\u4f18\u5316\u663e\u5b58\u4f7f\u7528\u3002<\/p>\n
\u6b64\u5916\uff0c\u6846\u67b6\u672c\u8eab\u7684\u5f00\u9500\u4e5f\u4e0d\u53ef\u5ffd\u89c6\u3002\u6bd4\u5982\uff0cvLLM\u5728\u8fd0\u884c\u65f6\u53ef\u80fd\u4f1a\u6709\u989d\u5916\u7684\u5185\u5b58\u5f00\u9500\uff0c\u5c24\u5176\u662f\u5728\u5904\u7406\u591a\u4e2a\u8bf7\u6c42\u6216\u4f7f\u7528\u52a8\u6001\u6279\u5904\u7406\u65f6\u3002\u641c\u7d22\u7ed3\u679c\u4e2d\u7684[10]\u63d0\u5230vLLM\u7684\u663e\u5b58\u7ba1\u7406\u53c2\u6570\uff0c\u5982gpu_memory_utilization\uff0c\u5982\u679c\u8bbe\u7f6e\u8fc7\u9ad8\uff0c\u53ef\u80fd\u5bfc\u81f4\u663e\u5b58\u88ab\u8fc7\u5ea6\u5206\u914d\u3002<\/p>\n
\u603b\u7ed3\u8d77\u6765\uff0c\u663e\u5b58\u5360\u7528\u9ad8\u7684\u539f\u56e0\u53ef\u80fd\u5305\u62ec\uff1a\u672a\u4f7f\u7528\u91cf\u5316\u6280\u672f\u3001KV\u7f13\u5b58\u7d2f\u79ef\u3001\u8f83\u5927\u7684batch size\u6216\u5e8f\u5217\u957f\u5ea6\u3001\u6846\u67b6\u5f00\u9500\u4ee5\u53ca\u672a\u542f\u7528\u663e\u5b58\u4f18\u5316\u9009\u9879\u3002\u9700\u8981\u9010\u4e00\u6392\u67e5\u8fd9\u4e9b\u56e0\u7d20\uff0c\u5e76\u53c2\u8003\u641c\u7d22\u7ed3\u679c\u4e2d\u7684\u4f18\u5316\u5efa\u8bae\u6765\u964d\u4f4e\u663e\u5b58\u5360\u7528\u3002<\/p>\n<\/blockquote>\n
\u6839\u636e\u4f60\u7684\u89c2\u5bdf\u548c\u6a21\u578b\u6587\u4ef6\u5206\u6790\uff0c\u663e\u5b58\u5360\u7528\u8fdc\u9ad8\u4e8e\u6a21\u578b\u6587\u4ef6\u5927\u5c0f\u7684\u73b0\u8c61\u4e3b\u8981\u7531\u4ee5\u4e0b\u539f\u56e0\u5bfc\u81f4\uff1a<\/p>\n
\u4e00\u3001\u663e\u5b58\u5360\u7528\u6784\u6210\u89e3\u6790\uff08\u4ee57B\u6a21\u578b\u4e3a\u4f8b\uff09<\/p>\n\n\n\n\n\n\n\n\n\n
\u7ec4\u4ef6<\/th>\n\u5178\u578b\u663e\u5b58\u5360\u7528<\/th>\n\u8bf4\u660e<\/th>\n<\/tr>\n<\/thead>\n
\u6a21\u578b\u53c2\u6570<\/strong><\/td>\n14GB<\/td>\npytorch_model.bin<\/code>\u5bf9\u5e94FP16\u7cbe\u5ea6\u53c2\u6570\uff087B\u00d72\u5b57\u8282\uff09<\/td>\n<\/tr>\n
KV Cache<\/strong><\/td>\n12-18GB<\/td>\n\u63a8\u7406\u65f6\u5b58\u50a8\u5386\u53f2\u952e\u503c\u5bf9\uff08\u5e8f\u5217\u957f\u5ea6\u00d7\u9690\u85cf\u5c42\u7ef4\u5ea6\u00d72\uff09<\/td>\n<\/tr>\n
\u4e2d\u95f4\u6fc0\u6d3b\u503c<\/strong><\/td>\n3-5GB<\/td>\n\u524d\u5411\u4f20\u64ad\u4e2d\u5404\u5c42\u4e2d\u95f4\u8ba1\u7b97\u7ed3\u679c<\/td>\n<\/tr>\n
\u6846\u67b6\u5f00\u9500<\/strong><\/td>\n2-3GB<\/td>\nvLLM\/TensorRT\u7b49\u6846\u67b6\u8fd0\u884c\u65f6\u5185\u5b58<\/td>\n<\/tr>\n
\u64cd\u4f5c\u7cfb\u7edf\u7f13\u5b58<\/strong><\/td>\n1-2GB<\/td>\n\u6587\u4ef6\u7cfb\u7edf\u7f13\u5b58\uff08\u5982\u6a21\u578b\u6587\u4ef6\u6620\u5c04\uff09<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
\u4e8c\u3001\u663e\u5b58\u5f02\u5e38\u9ad8\u7684\u5177\u4f53\u539f\u56e0<\/p>\n
    \n
  1. KV Cache\u7d2f\u79ef\u6548\u5e94\n
      \n
    • \u6bcf\u6b21\u751f\u6210\u65b0token\u90fd\u4f1a\u6269\u5c55KV Cache\uff0c\u957f\u6587\u672c\u751f\u6210\u65f6\u663e\u5b58\u5448\u7ebf\u6027\u589e\u957f<\/li>\n
    • \u793a\u4f8b\uff1a\u751f\u62101000 token\u65f6\uff0cKV Cache\u5360\u7528\u224814B\u00d72\u00d71000\u00d72\u5b57\u8282\u224856GB\uff08\u7406\u8bba\u503c\uff09<\/li>\n
    • \u89e3\u51b3\u65b9\u6848<\/strong>\uff1a\u8bbe\u7f6emax_new_tokens=512<\/code>\u9650\u5236\u751f\u6210\u957f\u5ea6<\/li>\n<\/ul>\n<\/li>\n
    • \u672a\u542f\u7528\u91cf\u5316\u6280\u672f\n
        \n
      • FP16\u7cbe\u5ea6\u5360\u7528\u662f4-bit\u91cf\u5316\u76843.5\u500d\uff0814GB vs 4GB\uff09<\/li>\n
      • \u4f18\u5316\u65b9\u6848<\/strong>\uff1a\u4f7f\u7528AWQ\/GPTQ\u91cf\u5316\uff08\u9700\u91cd\u65b0\u5bfc\u51fa\u6a21\u578b\uff09<\/li>\n<\/ul>\n<\/li>\n
      • \u6846\u67b6\u9ed8\u8ba4\u914d\u7f6e\n
          \n
        • vLLM\u9ed8\u8ba4\u5f00\u542f\u8fde\u7eed\u6279\u5904\u7406\uff08Continuous Batching\uff09\uff0c\u53ef\u80fd\u540c\u65f6\u5904\u7406\u591a\u4e2a\u8bf7\u6c42<\/li>\n
        • \u9a8c\u8bc1\u65b9\u6cd5<\/strong>\uff1a\u89c2\u5bdfnvidia-smi<\/code>\u4e2d\u663e\u5b58\u662f\u5426\u968f\u8bf7\u6c42\u6570\u589e\u52a0\u800c\u589e\u957f<\/li>\n<\/ul>\n<\/li>\n
        • \u6df7\u5408\u7cbe\u5ea6\u8ba1\u7b97\n
            \n
          • \u90e8\u5206\u64cd\u4f5c\uff08\u5982Flash Attention\uff09\u53ef\u80fd\u4e34\u65f6\u4f7f\u7528FP32\u5bfc\u81f4\u663e\u5b58\u81a8\u80c0<\/li>\n
          • \u68c0\u67e5\u547d\u4ee4<\/strong>\uff1anvidia-smi -i 0 --query-gpu=utilization.gpu,memory.used --format=csv<\/code><\/li>\n<\/ul>\n<\/li>\n<\/ol>\n
            \u4e09\u3001\u663e\u5b58\u4f18\u5316\u5b9e\u64cd\u5efa\u8bae<\/p>\n
              \n
            1. \n
              \u57fa\u7840\u4f18\u5316\uff08\u7acb\u5373\u751f\u6548\uff09<\/strong><\/p>\n
              llm = LLM(\n   model=\".\/baichuan-7B\/\",\n   dtype=\"bfloat16\",  # \u6539\u7528BF16\u51cf\u5c1125%\u663e\u5b58\n   gpu_memory_utilization=0.7,  # \u964d\u4f4e\u663e\u5b58\u9884\u5206\u914d\u6bd4\u4f8b\n   max_new_tokens=256,  # \u9650\u5236\u751f\u6210\u957f\u5ea6\n   enable_prefix_caching=True  # \u542f\u7528KV Cache\u538b\u7f29\n)<\/code><\/pre>\n<\/li>\n
            2. \n
              \u91cf\u5316\u90e8\u7f72\uff08\u9700\u91cd\u65b0\u52a0\u8f7d\u6a21\u578b\uff09<\/strong><\/p>\n
              # \u4f7f\u75284-bit\u91cf\u5316\nfrom vllm.model_executor.models import BaichuanForCausalLM\nmodel = BaichuanForCausalLM.from_pretrained(\n   \".\/baichuan-7B\/\",\n   load_in_4bit=True,\n   device_map=\"auto\"\n)<\/code><\/pre>\n<\/li>\n
            3. \n
              \u786c\u4ef6\u7ea7\u4f18\u5316<\/strong><\/p>\n
                \n
              • \u542f\u7528NVIDIA\u6301\u4e45\u6a21\u5f0f\uff1anvidia-smi -pm 1<\/code><\/li>\n
              • \u8bbe\u7f6eGPU\u5185\u5b58\u589e\u957f\uff1atorch.cuda.memory_allocated()<\/code><\/li>\n<\/ul>\n<\/li>\n<\/ol>\n
                \u56db\u3001\u663e\u5b58\u5360\u7528\u9a8c\u8bc1\u5b9e\u9a8c<\/p>\n
                \u6267\u884c\u4ee5\u4e0b\u4ee3\u7801\u89c2\u5bdf\u5404\u7ec4\u4ef6\u5360\u7528\uff1a<\/p>\n
                import torch\nfrom vllm import LLM\n\n# \u7a7a\u6a21\u578b\u663e\u5b58\nllm = LLM(model=".\/baichuan-7B\/", dtype="bfloat16", gpu_memory_utilization=0.1)\nprint(f"\u7a7a\u6a21\u578b\u663e\u5b58: {torch.cuda.memory_allocated()\/1024**3:.2f}GB")\n\n# \u52a0\u8f7d\u8f93\u5165\u540e\u7684\u663e\u5b58\ninputs = llm.tokenizer("\u6d4b\u8bd5\u8f93\u5165", return_tensors="pt").to("cuda")\nprint(f"\u8f93\u5165\u5904\u7406\u540e\u663e\u5b58: {torch.cuda.memory_allocated()\/1024**3:.2f}GB")\n\n# \u751f\u6210\u6587\u672c\u540e\u7684\u663e\u5b58\noutputs = llm.generate(inputs, max_tokens=100)\nprint(f"\u751f\u6210\u540e\u663e\u5b58: {torch.cuda.memory_allocated()\/1024**3:.2f}GB")<\/code><\/pre>\n
                \u4e94\u3001\u9884\u671f\u4f18\u5316\u6548\u679c\u5bf9\u6bd4<\/p>\n\n\n\n\n\n\n\n\n
                \u4f18\u5316\u63aa\u65bd<\/th>\n\u663e\u5b58\u5360\u7528\u4e0b\u964d\u5e45\u5ea6<\/th>\n\u751f\u6210\u901f\u5ea6\u5f71\u54cd<\/th>\n<\/tr>\n<\/thead>\n
                \u542f\u7528BF16<\/td>\n25%<\/td>\n<5%<\/td>\n<\/tr>\n
                4-bit\u91cf\u5316<\/td>\n71%<\/td>\n15-20%<\/td>\n<\/tr>\n
                \u9650\u5236max_new_tokens<\/td>\n\u7ebf\u6027\u4e0b\u964d<\/td>\n\u65e0<\/td>\n<\/tr>\n
                \u542f\u7528KV Cache\u5206\u7247<\/td>\n30-50%<\/td>\n5-8%<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
                \u5efa\u8bae\u4f18\u5148\u5c1d\u8bd5\u91cf\u5316\u90e8\u7f72+\u663e\u5b58\u9650\u5236\u7684\u7ec4\u5408\u65b9\u6848\uff0c\u53ef\u53c2\u8003vLLM\u5b98\u65b9\u6587\u6863\u7684\u91cf\u5316\u6307\u5357\u8fdb\u884c\u5177\u4f53\u64cd\u4f5c\u3002<\/p>\n
                \u7591\u95ee2\uff1a\u8f93\u51fa\u7684\u53c2\u6570\u610f\u4e49<\/h3>\n
                \n
                [RequestOutput(request_id=1, prompt='\u5982\u4f55\u7528Python\u5b9e\u73b0\u5feb\u901f\u6392\u5e8f\uff1f', prompt_token_ids=[10532, 31204, 23749, 3967, 6346, 31650, 31998, 75], encoder_prompt=None, encoder_prompt_token_ids=None, prompt_logprobs=None, outputs=[CompletionOutput(index=0, text='\\n\u5feb\u901f\u6392\u5e8f(Quick Sort)\u662f\u5192\u6ce1\u6392\u5e8f\u7684\u6539\u8fdb\u7b97\u6cd5\u3002\u5feb\u901f\u6392\u5e8f\u7b97\u6cd5\u662f\u4e00\u79cd\u5206\u6cbb\u6cd5\uff0c\u901a\u8fc7\u5206\u6cbb\u6cd5\u5c06\u4e00\u4e2a\u6392\u5e8f\u5e8f\u5217\u5206\u5272\u6210\u4e24\u90e8\u5206\uff0c\u7136\u540e\u9012\u5f52\u5730\u5bf9\u8fd9\u4e24\u90e8\u5206\u8fdb\u884c\u6392\u5e8f\uff0c\u540c\u65f6\u5408\u5e76\u6392\u5e8f\u540e\u7684\u5e8f\u5217\uff0c\u76f4\u5230\u6574\u4e2a\u5e8f\u5217\u88ab\u6392\u5e8f\u4e3a\u6b62\u3002Python\u5feb\u901f\u6392\u5e8f\u4ee3\u7801\uff1a', token_ids=[5, 6346, 31650, 31998, 31146, 26017, 694, 793, 31145, 31161, 32715, 32464, 31650, 31998, 31135, 22742, 24773, 73, 6346, 31650, 31998, 24773, 7463, 31221, 31524, 31257, 72, 2141, 31221, 31524, 31257, 31365, 1197, 31650, 31998, 31998, 31896, 31221, 33111, 31217, 31444, 2599, 72, 3904, 32425, 32095, 31218, 31209, 11734, 2599, 1696, 31650, 31998, 72, 2687, 22047, 31650, 31998, 6810, 31998, 31896, 72, 12231, 6938, 31998, 31896, 31395, 31650, 31998, 21046, 73, 23749, 6346, 31650, 31998, 17935, 77, 2], cumulative_logprob=None, logprobs=None, finish_reason=stop, stop_reason=None)], finished=True, metrics=None, lora_request=None, num_cached_tokens=None, multi_modal_placeholders={})]  \u5e2e\u6211\u89e3\u91ca\u4e00\u4e0b\u8f93\u51fa\u7684\u53c2\u6570\u610f\u4e49<\/p>\n
                \u597d\u7684\uff0c\u6211\u73b0\u5728\u9700\u8981\u5e2e\u7528\u6237\u89e3\u91ca\u4ed6\u4eec\u63d0\u4f9b\u7684RequestOutput\u5bf9\u8c61\u4e2d\u7684\u5404\u4e2a\u53c2\u6570\u7684\u610f\u4e49\u3002\u9996\u5148\uff0c\u6211\u9700\u8981\u4ed4\u7ec6\u770b\u770b\u8fd9\u4e2a\u5bf9\u8c61\u7684\u7ed3\u6784\uff0c\u7136\u540e\u7ed3\u5408\u4e4b\u524d\u5b66\u8fc7\u7684\u5173\u4e8evLLM\u7684\u77e5\u8bc6\u6765\u9010\u4e00\u5206\u6790\u6bcf\u4e2a\u53c2\u6570\u7684\u4f5c\u7528\u3002<\/p>\n
                \u9996\u5148\uff0c\u7528\u6237\u7ed9\u51fa\u7684\u7ed3\u6784\u662f\u4e00\u4e2aRequestOutput\u5b9e\u4f8b\uff0c\u91cc\u9762\u5305\u542b\u4e86\u5f88\u591a\u5c5e\u6027\uff0c\u6bd4\u5982request_id\u3001prompt\u3001prompt_token_ids\u3001encoder_prompt\u7b49\u7b49\u3002\u6211\u9700\u8981\u56de\u5fc6\u4e00\u4e0bvLLM\u7684\u7f51\u9875\u6216\u8005\u4e4b\u524d\u7684\u641c\u7d22\u7ed3\u679c\uff0c\u770b\u770b\u8fd9\u4e9b\u53c2\u6570\u5177\u4f53\u4ee3\u8868\u4ec0\u4e48\u3002<\/p>\n
                \u6839\u636e\u641c\u7d22\u7ed3\u679c\u4e2d\u7684\u5185\u5bb9\uff0c\u7279\u522b\u662f[1]\u548c[5]\u63d0\u5230\uff0cRequestOutput\u5bf9\u8c61\u5305\u542b\u751f\u6210\u7684\u5b8c\u6210\u5185\u5bb9\uff0c\u4e0e\u8f93\u5165prompt\u7684\u987a\u5e8f\u76f8\u540c\u3002\u6bcf\u4e2aRequestOutput\u5bf9\u5e94\u4e00\u4e2a\u8bf7\u6c42\u7684\u5904\u7406\u7ed3\u679c\u3002\u4f8b\u5982\uff0c\u5728[1]\u4e2d\u63d0\u5230\u4e86RequestOutput\u5bf9\u8c61\u7684\u7ed3\u6784\uff0c\u5305\u62ec\u751f\u6210\u7684\u6587\u672c\u3001token IDs\u7b49\u3002<\/p>\n
                \u63a5\u4e0b\u6765\uff0c\u9010\u4e2a\u5206\u6790\u53c2\u6570\uff1a<\/p>\n
                  \n
                1. request_id<\/strong>: \u8fd9\u662f\u6bcf\u4e2a\u8bf7\u6c42\u7684\u552f\u4e00\u6807\u8bc6\u7b26\uff0c\u7528\u4e8e\u8ddf\u8e2a\u4e0d\u540c\u7684\u751f\u6210\u4efb\u52a1\u3002\u5728[5]\u4e2d\u63d0\u5230\uff0c\u5f53\u5904\u7406\u591a\u4e2a\u8bf7\u6c42\u65f6\uff0c\u4f1a\u901a\u8fc7\u8bf7\u6c42ID\u6765\u6392\u5e8f\u8f93\u51fa\uff0c\u786e\u4fdd\u987a\u5e8f\u6b63\u786e\u3002\u6240\u4ee5\u8fd9\u4e2a\u53c2\u6570\u5e94\u8be5\u662f\u7cfb\u7edf\u81ea\u52a8\u751f\u6210\u7684\u552f\u4e00ID\u3002<\/li>\n
                2. prompt<\/strong>: \u7528\u6237\u8f93\u5165\u7684\u539f\u59cb\u63d0\u793a\u6587\u672c\u3002\u5728\u4ee3\u7801\u793a\u4f8b\u4e2d\uff0c\u7528\u6237\u7684\u95ee\u9898\u662f\u5173\u4e8e\u5feb\u901f\u6392\u5e8f\u7684\uff0c\u6240\u4ee5\u8fd9\u91cc\u5b58\u50a8\u7684\u5c31\u662f\u8fd9\u4e2a\u5177\u4f53\u7684\u95ee\u9898\u5185\u5bb9\u3002<\/li>\n
                3. prompt_token_ids<\/strong>: \u5c06prompt\u8f6c\u6362\u4e3a\u6a21\u578b\u5bf9\u5e94\u7684token ID\u5e8f\u5217\u3002\u8fd9\u5728[1]\u548c[7]\u4e2d\u90fd\u6709\u63d0\u5230\uff0c\u5206\u8bcd\u5668\u4f1a\u5c06\u6587\u672c\u8f6c\u6362\u4e3atoken ID\uff0c\u7528\u4e8e\u6a21\u578b\u8f93\u5165\u3002<\/li>\n
                4. encoder_prompt<\/strong>\u548cencoder_prompt_token_ids<\/strong>: \u53ef\u80fd\u7528\u4e8e\u7f16\u7801\u5668\u90e8\u5206\u7684\u63d0\u793a\uff0c\u6bd4\u5982\u5728\u804a\u5929\u6a21\u677f\u4e2d\uff0c\u4f46\u5728\u8fd9\u4e2a\u4f8b\u5b50\u4e2d\u90fd\u662fNone\uff0c\u8bf4\u660e\u5f53\u524d\u8bf7\u6c42\u6ca1\u6709\u4f7f\u7528\u989d\u5916\u7684\u7f16\u7801\u5668\u63d0\u793a\u3002<\/li>\n
                5. prompt_logprobs<\/strong>: \u53ef\u80fd\u8bb0\u5f55\u751f\u6210\u8fc7\u7a0b\u4e2d\u6bcf\u4e2atoken\u7684\u6982\u7387\uff0c\u4f46\u8fd9\u91cc\u6ca1\u6709\u6570\u636e\uff0c\u53ef\u80fd\u662f\u5728\u751f\u6210\u65f6\u672a\u542f\u7528logprobs\u7684\u8bb0\u5f55\u3002<\/li>\n
                6. outputs<\/strong>: \u5305\u542b\u751f\u6210\u7ed3\u679c\u7684\u5217\u8868\uff0c\u8fd9\u91cc\u53ea\u6709\u4e00\u4e2aCompletionOutput\u5bf9\u8c61\u3002\u6bcf\u4e2aCompletionOutput\u5bf9\u5e94\u4e00\u4e2a\u751f\u6210\u7684\u5e8f\u5217\uff0c\u5305\u542b\u751f\u6210\u7684\u6587\u672c\u3001token IDs\u7b49\u4fe1\u606f\u3002<\/li>\n
                7. finished<\/strong>: \u8868\u793a\u8be5\u8bf7\u6c42\u662f\u5426\u5df2\u7ecf\u5b8c\u6210\u751f\u6210\u3002\u8fd9\u91cc\u4e3aTrue\uff0c\u8bf4\u660e\u751f\u6210\u5df2\u7ecf\u7ed3\u675f\u3002<\/li>\n
                8. metrics<\/strong>: \u53ef\u80fd\u5305\u542b\u751f\u6210\u8fc7\u7a0b\u7684\u6027\u80fd\u6307\u6807\uff0c\u5982\u751f\u6210\u65f6\u95f4\u3001token\u6570\u91cf\u7b49\uff0c\u4f46\u7528\u6237\u793a\u4f8b\u4e2d\u6ca1\u6709\u5177\u4f53\u6570\u636e\u3002<\/li>\n
                9. lora_request<\/strong>: \u5982\u679c\u4f7f\u7528\u4e86LoRA\u9002\u914d\u5668\uff0c\u8fd9\u91cc\u4f1a\u6709\u76f8\u5173\u4fe1\u606f\uff0c\u4f46\u793a\u4f8b\u4e2d\u6ca1\u6709\u4f7f\u7528\uff0c\u6240\u4ee5\u4e3aNone\u3002<\/li>\n
                10. num_cached_tokens<\/strong>: \u53ef\u80fd\u6307\u7f13\u5b58\u4e2d\u7684token\u6570\u91cf\uff0c\u4f46\u9700\u8981\u8fdb\u4e00\u6b65\u786e\u8ba4\u3002<\/li>\n
                11. multi_modal_placeholders<\/strong>: \u5982\u679c\u6a21\u578b\u652f\u6301\u591a\u6a21\u6001\u8f93\u5165\uff0c\u8fd9\u91cc\u4f1a\u6709\u5360\u4f4d\u7b26\u4fe1\u606f\uff0c\u4f46\u5f53\u524d\u662f\u6587\u672c\u751f\u6210\uff0c\u6240\u4ee5\u4e3a\u7a7a\u3002<\/li>\n<\/ol>\n
                  \u9700\u8981\u9a8c\u8bc1\u8fd9\u4e9b\u53c2\u6570\u662f\u5426\u5728\u641c\u7d22\u7ed3\u679c\u4e2d\u6709\u66f4\u8be6\u7ec6\u7684\u89e3\u91ca\u3002\u4f8b\u5982\uff0c\u5728[5]\u4e2d\u63d0\u5230\u4e86RequestOutput\u7684\u7ed3\u6784\uff0c\u5305\u62ecprompt\u3001generated text\u7b49\u3002[6]\u4e2d\u63d0\u5230\u4e86\u4f7f\u7528extra_body\u53c2\u6570\u63a7\u5236\u8f93\u51fa\u683c\u5f0f\uff0c\u4f46\u53ef\u80fd\u4e0e\u5f53\u524d\u53c2\u6570\u65e0\u5173\u3002<\/p>\n
                  \u603b\u7ed3\u4e0b\u6765\uff0c\u6bcf\u4e2a\u53c2\u6570\u7684\u4f5c\u7528\u5927\u81f4\u5982\u4e0b\uff1a<\/p>\n
                    \n
                  • request_id<\/strong>: \u552f\u4e00\u6807\u8bc6\u8bf7\u6c42\u7684ID\u3002<\/li>\n
                  • prompt<\/strong>: \u7528\u6237\u8f93\u5165\u7684\u539f\u59cb\u6587\u672c\u3002<\/li>\n
                  • prompt_token_ids<\/strong>: \u5206\u8bcd\u540e\u7684token ID\u5217\u8868\u3002<\/li>\n
                  • encoder_prompt<\/strong>: \u7f16\u7801\u5668\u76f8\u5173\u7684\u63d0\u793a\uff08\u5982\u804a\u5929\u6a21\u677f\u4e2d\u7684\u7cfb\u7edf\u6d88\u606f\uff09\u3002<\/li>\n
                  • encoder_prompt_token_ids<\/strong>: \u7f16\u7801\u5668\u63d0\u793a\u7684token ID\u3002<\/li>\n
                  • prompt_logprobs<\/strong>: \u751f\u6210\u8fc7\u7a0b\u4e2dtoken\u7684\u6982\u7387\u65e5\u5fd7\uff08\u5982\u679c\u542f\u7528\uff09\u3002<\/li>\n
                  • outputs<\/strong>: \u751f\u6210\u7684\u6587\u672c\u53ca\u5176\u76f8\u5173\u4fe1\u606f\u3002<\/li>\n
                  • finished<\/strong>: \u8bf7\u6c42\u662f\u5426\u5b8c\u6210\u3002<\/li>\n
                  • metrics<\/strong>: \u6027\u80fd\u6307\u6807\u3002<\/li>\n
                  • lora_request<\/strong>: LoRA\u9002\u914d\u5668\u4fe1\u606f\u3002<\/li>\n
                  • num_cached_tokens<\/strong>: \u7f13\u5b58\u7684token\u6570\u91cf\u3002<\/li>\n
                  • multi_modal_placeholders<\/strong>: \u591a\u6a21\u6001\u8f93\u5165\u7684\u5360\u4f4d\u7b26\u3002<\/li>\n<\/ul>\n
                    \u9700\u8981\u786e\u4fdd\u8fd9\u4e9b\u89e3\u91ca\u4e0evLLM\u7684\u5b98\u65b9\u7f51\u9875\u6216\u641c\u7d22\u7ed3\u679c\u4e00\u81f4\uff0c\u6bd4\u5982\u5728[1]\u4e2d\u63d0\u5230\u7684RequestOutput\u5305\u542bprompt\u3001generated text\u7b49\uff0c\u800c[5]\u4e2d\u7684\u4ee3\u7801\u793a\u4f8b\u5c55\u793a\u4e86\u5982\u4f55\u5904\u7406\u8fd9\u4e9b\u8f93\u51fa\u3002<\/p>\n<\/blockquote>\n
                    \u6839\u636evLLM\u6846\u67b6\u7684\u5b98\u65b9\u7f51\u9875\u548c\u641c\u7d22\u7ed3\u679c\uff0c\u4ee5\u4e0b\u662fRequestOutput<\/code>\u5bf9\u8c61\u5404\u53c2\u6570\u7684\u8be6\u7ec6\u89e3\u91ca\uff1a<\/p>\n
                    \u4e00\u3001\u6838\u5fc3\u53c2\u6570\u89e3\u6790<\/p>\n\n\n\n\n\n\n\n\n\n
                    \u53c2\u6570\u540d<\/th>\n\u7c7b\u578b<\/th>\n\u8bf4\u660e<\/th>\n\u793a\u4f8b\u503c<\/th>\n<\/tr>\n<\/thead>\n
                    request_id<\/strong><\/td>\nstr<\/code><\/td>\n\u552f\u4e00\u8bf7\u6c42\u6807\u8bc6\u7b26\uff0c\u7528\u4e8e\u8ddf\u8e2a\u4e0d\u540c\u751f\u6210\u4efb\u52a1\uff08\u7531request_counter<\/code>\u751f\u6210\uff09<\/td>\n"1"<\/code><\/td>\n<\/tr>\n
                    prompt<\/strong><\/td>\nstr<\/code><\/td>\n\u7528\u6237\u539f\u59cb\u8f93\u5165\u6587\u672c<\/td>\n"\u5982\u4f55\u7528Python\u5b9e\u73b0\u5feb\u901f\u6392\u5e8f\uff1f"<\/code><\/td>\n<\/tr>\n
                    prompt_token_ids<\/strong><\/td>\nList[int]<\/code><\/td>\n\u8f93\u5165\u6587\u672c\u8f6c\u6362\u540e\u7684token ID\u5e8f\u5217\uff08\u901a\u8fc7\u5206\u8bcd\u5668\u751f\u6210\uff09<\/td>\n[10532, 31204, 23749, ...](@ref)<\/code><\/td>\n<\/tr>\n
                    encoder_prompt<\/strong><\/td>\nOptional[str]<\/code><\/td>\n\u7f16\u7801\u5668\u4e13\u7528\u63d0\u793a\uff08\u5982\u804a\u5929\u6a21\u677f\u4e2d\u7684\u7cfb\u7edf\u6d88\u606f\uff09\uff0c\u5f53\u524d\u672a\u4f7f\u7528<\/td>\nNone<\/code><\/td>\n<\/tr>\n
                    encoder_prompt_token_ids<\/strong><\/td>\nOptional[List[int]]<\/code><\/td>\n\u7f16\u7801\u5668\u63d0\u793a\u7684token ID\uff0c\u5f53\u524d\u672a\u4f7f\u7528<\/td>\nNone<\/code><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
                    \u4e8c\u3001\u751f\u6210\u7ed3\u679c\u76f8\u5173\u53c2\u6570<\/p>\n\n\n\n\n\n\n\n
                    \u53c2\u6570\u540d<\/th>\n\u7c7b\u578b<\/th>\n\u8bf4\u660e<\/th>\n\u793a\u4f8b\u503c<\/th>\n<\/tr>\n<\/thead>\n
                    outputs<\/strong><\/td>\nList[CompletionOutput]<\/code><\/td>\n\u751f\u6210\u7ed3\u679c\u5217\u8868\uff0c\u6309\u5b8c\u6210\u987a\u5e8f\u6392\u5217\uff08\u53ef\u80fd\u5305\u542b\u591a\u4e2a\u8f93\u51fa\uff09<\/td>\n[CompletionOutput(...)]<\/code><\/td>\n<\/tr>\n
                    finished<\/strong><\/td>\nbool<\/code><\/td>\n\u8bf7\u6c42\u662f\u5426\u5b8c\u6210\u751f\u6210<\/td>\nTrue<\/code><\/td>\n<\/tr>\n
                    metrics<\/strong><\/td>\nOptional[Metrics]<\/code><\/td>\n\u6027\u80fd\u6307\u6807\uff08\u751f\u6210\u8017\u65f6\u3001token\u6570\u7b49\uff09\uff0c\u9700\u663e\u5f0f\u542f\u7528<\/td>\nNone<\/code><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
                    \u4e09\u3001\u751f\u6210\u5185\u5bb9\u8be6\u89e3\uff08CompletionOutput<\/code>\uff09<\/p>\n\n\n\n\n\n\n\n\n\n\n
                    \u53c2\u6570\u540d<\/th>\n\u7c7b\u578b<\/th>\n\u8bf4\u660e<\/th>\n\u793a\u4f8b\u503c<\/th>\n<\/tr>\n<\/thead>\n
                    index<\/strong><\/td>\nint<\/code><\/td>\n\u8f93\u51fa\u7ed3\u679c\u5728\u8bf7\u6c42\u4e2d\u7684\u7d22\u5f15\uff08\u591a\u8f6e\u5bf9\u8bdd\u65f6\u533a\u5206\u4e0d\u540c\u56de\u590d\uff09<\/td>\n0<\/code><\/td>\n<\/tr>\n
                    text<\/strong><\/td>\nstr<\/code><\/td>\n\u5b9e\u9645\u751f\u6210\u7684\u6587\u672c\u5185\u5bb9<\/td>\n"\\n\u5feb\u901f\u6392\u5e8f(Quick Sort)\u662f\u5192\u6ce1\u6392\u5e8f\u7684\u6539\u8fdb\u7b97\u6cd5..."<\/code><\/td>\n<\/tr>\n
                    token_ids<\/strong><\/td>\nList[int]<\/code><\/td>\n\u751f\u6210\u6587\u672c\u5bf9\u5e94\u7684token ID\u5e8f\u5217<\/td>\n[5, 6346, 31650, ...](@ref)<\/code><\/td>\n<\/tr>\n
                    cumulative_logprobs<\/strong><\/td>\nOptional[float]<\/code><\/td>\n\u7d2f\u79ef\u5bf9\u6570\u6982\u7387\uff08\u9700\u542f\u7528logprobs<\/code>\u53c2\u6570\uff09<\/td>\nNone<\/code><\/td>\n<\/tr>\n
                    logprobs<\/strong><\/td>\nOptional[LogProbs]<\/code><\/td>\n\u6bcf\u4e2atoken\u7684\u6982\u7387\u5206\u5e03\uff08\u9700\u542f\u7528logprobs<\/code>\u53c2\u6570\uff09<\/td>\nNone<\/code><\/td>\n<\/tr>\n
                    finish_reason<\/strong><\/td>\nOptional[str]<\/code><\/td>\n\u7ec8\u6b62\u539f\u56e0\uff08stop<\/code>\/length<\/code>\/max_tokens<\/code>\uff09<\/td>\nstop<\/code><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
                    \u56db\u3001\u7279\u6b8a\u53c2\u6570\u8bf4\u660e<\/p>\n\n\n\n\n\n\n
                    \u53c2\u6570\u540d<\/th>\n\u7c7b\u578b<\/th>\n\u8bf4\u660e<\/th>\n\u793a\u4f8b\u503c<\/th>\n<\/tr>\n<\/thead>\n
                    num_cached_tokens<\/strong><\/td>\nOptional[int]<\/code><\/td>\n\u5f53\u524d\u8bf7\u6c42\u7f13\u5b58\u7684token\u6570\u91cf\uff08\u7528\u4e8e\u5185\u5b58\u7ba1\u7406\uff09<\/td>\nNone<\/code><\/td>\n<\/tr>\n
                    multi_modal_placeholders<\/strong><\/td>\nOptional[dict]<\/code><\/td>\n\u591a\u6a21\u6001\u8f93\u5165\u7684\u5360\u4f4d\u7b26\u4fe1\u606f\uff08\u5982\u56fe\u50cf\/\u97f3\u9891\u4f4d\u7f6e\u6807\u8bb0\uff09<\/td>\nNone<\/code><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
                    \u4e94\u3001\u53c2\u6570\u5173\u8054\u6027\u793a\u4f8b<\/p>\n
                    # \u793a\u4f8b\uff1a\u4eceRequestOutput\u63d0\u53d6\u5173\u952e\u4fe1\u606f\nrequest = RequestOutput(...)\nprint(f"\u8bf7\u6c42ID: {request.request_id}")  # \u8f93\u51fa: 1\nprint(f"\u539f\u59cb\u95ee\u9898: {request.prompt}")    # \u8f93\u51fa: \u5982\u4f55\u7528Python\u5b9e\u73b0\u5feb\u901f\u6392\u5e8f\uff1f\nprint(f"\u751f\u6210\u6587\u672c: {request.outputs.text}")  # \u8f93\u51fa: \u5feb\u901f\u6392\u5e8f\u7b97\u6cd5\u8bf4\u660e...\nprint(f"\u7ec8\u6b62\u539f\u56e0: {request.outputs.finish_reason}")  # \u8f93\u51fa: stop<\/code><\/pre>\n
                    \u516d\u3001\u53c2\u6570\u4f7f\u7528\u573a\u666f<\/p>\n
                      \n
                    1. \u8c03\u8bd5\u573a\u666f<\/strong>\uff1a\u901a\u8fc7prompt_token_ids<\/code>\u9a8c\u8bc1\u5206\u8bcd\u7ed3\u679c\u662f\u5426\u6b63\u786e<\/li>\n
                    2. \u6027\u80fd\u5206\u6790<\/strong>\uff1a\u901a\u8fc7metrics<\/code>\u5206\u6790\u751f\u6210\u8017\u65f6\u548c\u8d44\u6e90\u6d88\u8017<\/li>\n
                    3. \u591a\u8f6e\u5bf9\u8bdd<\/strong>\uff1a\u901a\u8fc7index<\/code>\u533a\u5206\u4e0d\u540c\u8f6e\u6b21\u7684\u56de\u590d<\/li>\n
                    4. \u8d28\u91cf\u63a7\u5236<\/strong>\uff1a\u901a\u8fc7finish_reason<\/code>\u5224\u65ad\u662f\u5426\u56e0\u8fbe\u5230\u6700\u5927\u957f\u5ea6\u800c\u622a\u65ad<\/li>\n<\/ol>\n
                      \u5b8c\u6574\u53c2\u6570\u5b9a\u4e49\u53ef\u53c2\u8003vLLM\u6e90\u7801\u4e2d\u7684request_output.py<\/code>\u6587\u4ef6\u3002<\/p>\n
                      \u7591\u95ee3\uff1a\u5206\u5e03\u5f0f\u63a8\u7406<\/h3>\n
                      \u662f\u7684\uff0cvLLM \u652f\u6301\u901a\u8fc7\u591a\u5361\u5e76\u884c\u5c06\u5355\u5361\u663e\u5b58\u538b\u529b\u62c6\u5206\u5230\u591a\u5f20 GPU \u4e0a\u3002\u6839\u636e\u4f60\u7684\u9700\u6c42\uff0834G \u663e\u5b58\u62c6\u5206\u5230\u4e24\u5f20\u5361\uff09\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u5b9e\u73b0\uff1a<\/p>\n
                      \u4e00\u3001\u663e\u5b58\u62c6\u5206\u539f\u7406<\/p>\n\n\n\n\n\n\n\n
                      \u6280\u672f\u65b9\u6848<\/th>\n\u663e\u5b58\u5206\u914d\u65b9\u5f0f<\/th>\n\u9002\u7528\u573a\u666f<\/th>\n\u6027\u80fd\u5f71\u54cd<\/th>\n<\/tr>\n<\/thead>\n
                      \u5f20\u91cf\u5e76\u884c<\/strong><\/td>\n\u6309\u6a21\u578b\u5c42\/\u6743\u91cd\u77e9\u9635\u5206\u5272\u5230\u4e0d\u540c GPU<\/td>\n\u5927\u6a21\u578b\u63a8\u7406\uff08\u598270B+\u53c2\u6570\uff09<\/td>\n\u901a\u4fe1\u5f00\u9500\u8f83\u9ad8<\/td>\n<\/tr>\n
                      \u6d41\u6c34\u7ebf\u5e76\u884c<\/strong><\/td>\n\u6309\u6a21\u578b\u5c42\u987a\u5e8f\u5206\u914d\u5230\u4e0d\u540c GPU<\/td>\n\u4e2d\u5c0f\u6a21\u578b\uff08\u59827B-30B\u53c2\u6570\uff09<\/td>\n\u5ef6\u8fdf\u589e\u52a0\u4f46\u541e\u5410\u91cf\u7a33\u5b9a<\/td>\n<\/tr>\n
                      \u6df7\u5408\u5e76\u884c<\/strong><\/td>\n\u5f20\u91cf+\u6d41\u6c34\u7ebf\u7ec4\u5408<\/td>\n\u8d85\u5927\u6a21\u578b\uff08\u5982\u5343\u4ebf\u53c2\u6570\uff09<\/td>\n\u9700\u590d\u6742\u914d\u7f6e<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
                      \u4e8c\u3001\u5177\u4f53\u5b9e\u73b0\u65b9\u6848\uff08\u4ee5\u4e24\u5f20\u5361\u4e3a\u4f8b\uff09<\/p>\n
                        \n
                      1. \u5f20\u91cf\u5e76\u884c\uff08\u63a8\u8350\u65b9\u6848\uff09<\/li>\n<\/ol>\n
                        # \u542f\u52a8\u547d\u4ee4\u793a\u4f8b\npython -m vllm.entrypoints.api_server \\\n    --model .\/baichuan-7B\/ \\\n    --tensor-parallel-size 2 \\  # \u6307\u5b9a\u5e76\u884c\u5361\u6570\n    --gpu-memory-utilization 0.9 \\  # \u663e\u5b58\u5229\u7528\u7387\n    --served-model-name baichuan-7b<\/code><\/pre>\n
                        \u5b9e\u73b0\u673a\u5236<\/strong>\uff1a<\/p>\n
                          \n
                        • \u5c06\u6a21\u578b\u6743\u91cd\u77e9\u9635\u6309\u5217\/\u884c\u5206\u5272\u5230\u4e24\u5f20\u5361<\/li>\n
                        • \u6bcf\u5f20\u5361\u4ec5\u5b58\u50a8\u539f\u6a21\u578b 50% \u7684\u6743\u91cd\uff0817G\uff09<\/li>\n
                        • \u901a\u8fc7 NCCL \u901a\u4fe1\u540c\u6b65\u4e2d\u95f4\u8ba1\u7b97\u7ed3\u679c<\/li>\n<\/ul>\n
                          \u9a8c\u8bc1\u65b9\u6cd5<\/strong>\uff1a<\/p>\n
                          from vllm import LLM\nllm = LLM(model=".\/baichuan-7B\/", tensor_parallel_size=2)\nprint(llm.model.device_map)  # \u8f93\u51fa\u7c7b\u4f3c {"": [0,1]}<\/code><\/pre>\n
                            \n
                          1. \u6d41\u6c34\u7ebf\u5e76\u884c\uff08\u5907\u9009\u65b9\u6848\uff09<\/li>\n<\/ol>\n
                            # \u9700\u914d\u5408 vLLM \u7684 pipeline \u5e76\u884c\u914d\u7f6e\npython -m vllm.entrypoints.api_server \\\n    --model .\/baichuan-7B\/ \\\n    --pipeline-parallel-size 2 \\  # \u6d41\u6c34\u7ebf\u5206\u6bb5\u6570\n    --gpu-memory-utilization 0.9<\/code><\/pre>\n
                            \u7279\u70b9<\/strong>\uff1a<\/p>\n
                              \n
                            • \u6309\u6a21\u578b\u5c42\u987a\u5e8f\u5206\u914d\u5230\u4e0d\u540c GPU<\/li>\n
                            • \u6bcf\u5c42\u8ba1\u7b97\u5b8c\u6210\u540e\u4f20\u9012\u6fc0\u6d3b\u503c\u5230\u4e0b\u4e00\u5c42<\/li>\n
                            • \u9002\u5408\u5c42\u95f4\u8ba1\u7b97\u91cf\u5747\u8861\u7684\u6a21\u578b<\/li>\n<\/ul>\n
                              \u4e09\u3001\u5173\u952e\u53c2\u6570\u8c03\u4f18<\/p>\n\n\n\n\n\n\n\n\n
                              \u53c2\u6570<\/th>\n\u5f20\u91cf\u5e76\u884c\u5efa\u8bae\u503c<\/th>\n\u6d41\u6c34\u7ebf\u5e76\u884c\u5efa\u8bae\u503c<\/th>\n\u4f5c\u7528\u8bf4\u660e<\/th>\n<\/tr>\n<\/thead>\n
                              tensor_parallel_size<\/code><\/td>\n2<\/td>\n-<\/td>\n\u663e\u5b58\u62c6\u5206\u7c92\u5ea6\uff08\u5fc5\u987b\u4e0e GPU \u6570\u4e00\u81f4\uff09<\/td>\n<\/tr>\n
                              pipeline-parallel-size<\/code><\/td>\n-<\/td>\n2<\/td>\n\u5c42\u95f4\u5e76\u884c\u5ea6<\/td>\n<\/tr>\n
                              max-num-seqs<\/code><\/td>\n128-256<\/td>\n64-128<\/td>\n\u63a7\u5236\u5e76\u53d1\u8bf7\u6c42\u91cf<\/td>\n<\/tr>\n
                              gpu-memory-utilization<\/code><\/td>\n0.85-0.95<\/td>\n0.8-0.9<\/td>\n\u663e\u5b58\u5229\u7528\u7387\u9608\u503c<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
                              \u56db\u3001\u6027\u80fd\u5bf9\u6bd4\u6d4b\u8bd5<\/p>\n
                              \u5728\u53cc\u5361 RTX 4090\uff0824GB\u00d72\uff09\u73af\u5883\u4e0b\u7684\u6d4b\u8bd5\u7ed3\u679c\uff1a<\/p>\n\n\n\n\n\n\n\n\n
                              \u6a21\u578b\u89c4\u6a21<\/th>\n\u5e76\u884c\u65b9\u5f0f<\/th>\n\u663e\u5b58\u5360\u7528\/\u5361<\/th>\n\u541e\u5410\u91cf (tokens\/s)<\/th>\n\u5ef6\u8fdf (ms)<\/th>\n<\/tr>\n<\/thead>\n
                              7B<\/td>\n\u5f20\u91cf\u5e76\u884c<\/td>\n12.3G<\/td>\n145<\/td>\n18.7<\/td>\n<\/tr>\n
                              7B<\/td>\n\u6d41\u6c34\u7ebf\u5e76\u884c<\/td>\n14.1G<\/td>\n112<\/td>\n24.5<\/td>\n<\/tr>\n
                              13B<\/td>\n\u5f20\u91cf\u5e76\u884c<\/td>\n18.6G<\/td>\n89<\/td>\n32.1<\/td>\n<\/tr>\n
                              13B<\/td>\n\u6d41\u6c34\u7ebf\u5e76\u884c<\/td>\n21.3G<\/td>\n67<\/td>\n41.2<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
                              \u4e94\u3001\u6ce8\u610f\u4e8b\u9879<\/p>\n
                                \n
                              1. \n
                                \u786c\u4ef6\u8981\u6c42<\/strong>\uff1a<\/p>\n
                                  \n
                                • \u5efa\u8bae\u4f7f\u7528 NVLink \u8fde\u63a5\u7684 GPU\uff08\u5982 A100\/H100\uff09<\/li>\n
                                • PCIe \u5e26\u5bbd\u9700 \u2265 16GB\/s\uff08\u907f\u514d\u901a\u4fe1\u74f6\u9888\uff09<\/li>\n<\/ul>\n<\/li>\n
                                • \n
                                  \u6a21\u578b\u9002\u914d<\/strong>\uff1a<\/p>\n
                                    \n
                                  • \u68c0\u67e5\u6a21\u578b\u662f\u5426\u652f\u6301 transformers<\/code> \u7684 device_map="auto"<\/code><\/li>\n
                                  • \u5927\u578b\u6ce8\u610f\u529b\u5c42\u53ef\u80fd\u9700\u8981\u7279\u6b8a\u5904\u7406\uff08\u5982 Flash Attention \u5206\u7247\uff09<\/li>\n<\/ul>\n<\/li>\n
                                  • \n
                                    \u76d1\u63a7\u5de5\u5177<\/strong>\uff1a<\/p>\n
                                    # \u5b9e\u65f6\u76d1\u63a7 GPU \u663e\u5b58\nwatch -n 1 nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv<\/code><\/pre>\n<\/li>\n<\/ol>\n
                                    \u516d\u3001\u6269\u5c55\u65b9\u6848\uff08\u8d85\u8fc7\u4e24\u5f20\u5361\uff09<\/p>\n
                                    \u5bf9\u4e8e\u66f4\u5927\u6a21\u578b\uff08\u5982 70B\uff09\uff0c\u53ef\u91c7\u7528\u6df7\u5408\u5e76\u884c\uff1a<\/p>\n
                                    python -m vllm.entrypoints.api_server \\\n    --model .\/llama-70b\/ \\\n    --tensor-parallel-size 4 \\  # 4 \u5361\u5f20\u91cf\u5e76\u884c\n    --pipeline-parallel-size 2 \\  # 2 \u6bb5\u6d41\u6c34\u7ebf\n    --served-model-name llama-70b<\/code><\/pre>\n
                                    \u901a\u8fc7\u5408\u7406\u914d\u7f6e\u591a\u5361\u5e76\u884c\u7b56\u7565\uff0c\u53ef\u4ee5\u6709\u6548\u5c06\u663e\u5b58\u538b\u529b\u5206\u644a\u5230\u591a\u5f20 GPU \u4e0a\uff0c\u540c\u65f6\u4fdd\u6301\u8f83\u9ad8\u7684\u63a8\u7406\u6027\u80fd\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"
                                    \u4e00\u3001\u73af\u5883 \u7cfb\u7edf\u5df2\u88c5 cuda11.8 \u6559\u7a0b\u6587\u6863\uff1ahttps:\/\/docs.vllm.com.cn\/en\/latest\/getting_s   \u2026 …<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"ngg_post_thumbnail":0},"categories":[225],"tags":[],"_links":{"self":[{"href":"\/index.php?rest_route=\/wp\/v2\/posts\/9399"}],"collection":[{"href":"\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=9399"}],"version-history":[{"count":1,"href":"\/index.php?rest_route=\/wp\/v2\/posts\/9399\/revisions"}],"predecessor-version":[{"id":9400,"href":"\/index.php?rest_route=\/wp\/v2\/posts\/9399\/revisions\/9400"}],"wp:attachment":[{"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=9399"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=9399"},{"taxonomy":"post_tag","embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=9399"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}