vLLM

docker pull vllm/vllm-openai:latest

# 단일 GPU 기준 Qwen3 모델을 vLLM 서버로 실행합니다.
docker run --gpus all \
  -p 8000:8000 \
  --ipc=host \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  vllm/vllm-openai:latest \
    Qwen/Qwen3-8B-Instruct \
    --host 0.0.0.0 \
    --port 8000 \
    --dtype bfloat16 \
    --max-model-len 8192 \
    --gpu-memory-utilization 0.90 \
    --seed 42

# compose.yaml
services:
  vllm:
    image: vllm/vllm-openai:latest
    ports:
      - "8000:8000"
    ipc: host
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    gpus: all
    command: >
      Qwen/Qwen3-8B-Instruct
      --host 0.0.0.0
      --port 8000
      --dtype bfloat16
      --max-model-len 8192
      --gpu-memory-utilization 0.90
      --seed 42

# 서버 시작
docker compose up -d

# 로그 확인
docker compose logs -f vllm

# 서버 종료
docker compose down

services:
  vllm:
    image: vllm/vllm-openai:latest
    ports:
      - "8000:8000"
    ipc: host
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    gpus: all
    command: >
      Qwen/Qwen3-8B-Instruct
      --host 0.0.0.0
      --port 8000
      --dtype bfloat16
      --max-model-len 8192
      --gpu-memory-utilization 0.92
      --max-num-seqs 32
      --max-num-batched-tokens 4096
      --enable-chunked-prefill
      --async-scheduling
      --performance-mode throughput
      --seed 42

services:
  vllm:
    image: vllm/vllm-openai:latest
    ports:
      - "8000:8000"
    ipc: host
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    gpus:
      - count: 2
    command: >
      Qwen/Qwen3-8B-Instruct
      --tensor-parallel-size 2
      --host 0.0.0.0
      --port 8000
      --dtype bfloat16
      --max-model-len 8192
      --gpu-memory-utilization 0.90
      --max-num-seqs 64
      --max-num-batched-tokens 8192
      --performance-mode throughput
      --seed 42

services:
  vllm:
    image: vllm/vllm-openai:latest
    ports:
      - "8000:8000"
    ipc: host
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    gpus: all
    command: >
      Qwen/Qwen3-8B-Instruct
      --tensor-parallel-size 2
      --data-parallel-size 2
      --api-server-count 2
      --host 0.0.0.0
      --port 8000
      --dtype bfloat16
      --max-model-len 8192
      --gpu-memory-utilization 0.90
      --max-num-seqs 128
      --max-num-batched-tokens 16384
      --enable-chunked-prefill
      --async-scheduling
      --performance-mode throughput

services:
  vllm:
    image: vllm/vllm-openai:latest
    ports:
      - "8000:8000"
    ipc: host
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    gpus: all
    command: >
      Qwen/Qwen3-8B-Instruct
      --host 0.0.0.0
      --port 8000
      --dtype float16
      --max-model-len 4096
      --kv-cache-memory-bytes 10G
      --cpu-offload-gb 8
      --swap-space 16

# OpenAI 호환 Chat Completions API 호출
curl http://127.0.0.1:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "Qwen/Qwen3-8B-Instruct",
    "messages": [
      {"role": "system", "content": "당신은 딥러닝 튜터입니다."},
      {"role": "user", "content": "vLLM의 장점을 3가지로 설명해줘."}
    ],
    "temperature": 0.0,
    "max_tokens": 256
  }'

# OpenAI SDK를 vLLM 엔드포인트에 연결
from openai import OpenAI

client = OpenAI(
    base_url="http://127.0.0.1:8000/v1",
    api_key="EMPTY",  # vLLM 로컬 서버는 임의 문자열 사용 가능
)

# 재현 가능성을 위해 temperature를 0으로 설정
response = client.chat.completions.create(
    model="Qwen/Qwen3-8B-Instruct",
    messages=[
        {"role": "system", "content": "당신은 딥러닝 튜터입니다."},
        {"role": "user", "content": "Qwen3를 서비스할 때 주의점을 알려줘."},
    ],
    temperature=0.0,
    max_tokens=256,
)

print(response.choices[0].message.content)

# 2개 GPU를 사용해 Qwen3 대형 모델 서빙
docker run --gpus '"device=0,1"' \
  -p 8000:8000 \
  --ipc=host \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  vllm/vllm-openai:latest \
    Qwen/Qwen3-32B-Instruct \
    --tensor-parallel-size 2 \
    --host 0.0.0.0 \
    --port 8000

ollama pull qwen3:8b
OLLAMA_NUM_PARALLEL=8 ollama serve

docker run --gpus all \
  -p 8000:8000 \
  --ipc=host \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  vllm/vllm-openai:latest \
    Qwen/Qwen3-8B-Instruct \
    --host 0.0.0.0 \
    --port 8000 \
    --dtype bfloat16 \
    --max-model-len 8192 \
    --gpu-memory-utilization 0.90 \
    --seed 42

import argparse
import asyncio
import statistics
import time

import httpx

PROMPT = "vLLM과 Ollama의 동시 요청 처리 방식 차이를 3문장으로 설명해줘."

async def one_request(client: httpx.AsyncClient, target: str, max_tokens: int) -> float:
    if target == "vllm":
        url = "http://127.0.0.1:8000/v1/chat/completions"
        payload = {
            "model": "Qwen/Qwen3-8B-Instruct",
            "messages": [{"role": "user", "content": PROMPT}],
            "temperature": 0.0,
            "max_tokens": max_tokens,
        }
    else:
        url = "http://127.0.0.1:11434/api/generate"
        payload = {
            "model": "qwen3:8b",
            "prompt": PROMPT,
            "stream": False,
            "options": {"temperature": 0.0, "num_predict": max_tokens},
        }

    t0 = time.perf_counter()
    resp = await client.post(url, json=payload, timeout=180.0)
    resp.raise_for_status()
    return (time.perf_counter() - t0) * 1000.0

async def run_bench(target: str, concurrency: int, total_requests: int, max_tokens: int):
    sem = asyncio.Semaphore(concurrency)
    latencies = []

    async with httpx.AsyncClient() as client:
        async def worker():
            async with sem:
                return await one_request(client, target, max_tokens)

        start = time.perf_counter()
        results = await asyncio.gather(
            *[worker() for _ in range(total_requests)],
            return_exceptions=True,
        )
        elapsed = time.perf_counter() - start

    for r in results:
        if not isinstance(r, Exception):
            latencies.append(r)

    success = len(latencies)
    rps = success / elapsed if elapsed > 0 else 0.0
    p50 = statistics.median(latencies) if latencies else 0.0
    p95 = (
        statistics.quantiles(latencies, n=100)[94]
        if len(latencies) >= 20
        else (max(latencies) if latencies else 0.0)
    )

    print(
        f"[{target}] c={concurrency:>2} | success={success:>3}/{total_requests:<3} "
        f"| req/s={rps:>6.2f} | p50={p50:>7.1f}ms | p95={p95:>7.1f}ms"
    )

async def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--target", choices=["ollama", "vllm"], required=True)
    parser.add_argument("--concurrency", type=int, nargs="+", default=[1, 4, 8, 16, 32])
    parser.add_argument("--total-requests", type=int, default=64)
    parser.add_argument("--max-tokens", type=int, default=128)
    args = parser.parse_args()

    for c in args.concurrency:
        await run_bench(args.target, c, args.total_requests, args.max_tokens)

if __name__ == "__main__":
    asyncio.run(main())

pip install -U httpx

python benchmark_concurrency.py --target ollama --concurrency 1 4 8 16 32 --total-requests 64

python benchmark_concurrency.py --target vllm --concurrency 1 4 8 16 32 --total-requests 64

services:
  vllm-8b:
    image: vllm/vllm-openai:latest
    ports:
      - "8000:8000"
    ipc: host
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    gpus:
      - count: 1
    command: >
      Qwen/Qwen3-8B-Instruct
      --host 0.0.0.0
      --port 8000

  vllm-32b:
    image: vllm/vllm-openai:latest
    ports:
      - "8001:8000"
    ipc: host
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    gpus:
      - count: 2
    command: >
      Qwen/Qwen3-32B-Instruct
      --tensor-parallel-size 2
      --host 0.0.0.0
      --port 8000

문서 저장소 → 벡터 임베딩 → 벡터 DB → 검색기 → 프롬프트 구성 → vLLM API → 최종 응답