LM/vLLM

Verify environments in container

docker run -it --rm \
  --device=/dev/dri \
  --group-add video \
  -v $HOME/.cache/huggingface/hub:/root/.cache/huggingface/hub \
  -v /dev/dri/by-path:/dev/dri/by-path \
  --entrypoint bash \
  vllm/vllm-openai-xpu:latest

Command	Result
lsb_release -a	No LSB modules are available. Distributor ID: Ubuntu Description: Ubuntu 24.04.4 LTS Release: 24.04 Codename: noble
xpu-smi	+----------------------------------------------+----------------------------+------------------------+ \| Intel XPU-SMI v2.1 Driver: 9913356BC2F8B14798CAB3D Level Zero: 1.32.0 \| +----------------------------------------------+----------------------------+------------------------+ \| GPU Name Persistence-M \| Bus-Id Disp.A \| Volatile Uncorr. ECC \| \| Fan Temp Pwr:Usage/Cap \| Memory-Usage \| GPU-Util Compute M. \| +----------------------------------------------+----------------------------+------------------------+ \| 0 Intel(R) Arc(TM) Pro B Off \| 0000:03:00.0 Off \| Disabled \| \| 70 Graphics \| \| \| \| 0% N/A 0W / 230W \| 42MiB / 32656MiB \| N/A Default \| +----------------------------------------------+----------------------------+------------------------+ +-------+-------+--------+----------------+----------------------------------------------------------+ \| Processes: \| \| GPU PID Type Process Name GPU Memory Usage \| +-------+-------+--------+----------------+----------------------------------------------------------+ \| No running GPU processes found. \| +-------+-------+--------+----------------+----------------------------------------------------------+
which python which vllm	/opt/venv/bin/python /opt/venv/bin/vllm
python -V	Python 3.12.3
pip list	Package Version Editable project location 12:20 [237/1957] ---------------------------------------- ------------- ------------------------------------- absl-py 2.5.0 accelerate 1.14.0 agent-detector 2.0.0 aiohappyeyeballs 2.7.1 aiohttp 3.14.3 aiosignal 1.4.0 albucore 0.0.24 albumentations 2.0.8 annotated-doc 0.0.5 annotated-types 0.8.0 anthropic 0.122.0 anyio 4.14.2 apache-tvm-ffi 0.1.13.post3 arctic_inference 0.2.0 astor 0.8.1 attrs 26.1.0 audioread 3.1.0 auto-round-lib 0.14.2 blake3 1.0.9 blobfile 3.2.0 bm25s 0.3.10 bounded-pool-executor 0.0.3 cachetools 7.1.7 cbor2 6.1.4 certifi 2026.7.22 cffi 2.1.1 chardet 6.0.0.post1 charset-normalizer 3.5.1 chz 0.4.0 click 8.4.2 cloudpickle 3.1.2 cmake 4.4.3 colorama 0.4.6 compressed-tensors 0.17.0 coverage 7.15.4 cryptography 50.0.0 DataProperty 1.1.1 datasets 5.0.1 decorator 5.3.1 defusedxml 0.7.1 depyf 0.20.0 detect-installer 0.1.0 dill 0.4.1 distro 1.9.0 dnspython 2.8.0 docker 7.2.0 docopt 0.6.2 docstring_parser 0.18.0 dpcpp-cpp-rt 2026.0.0 einops 0.8.2 email-validator 2.3.0 evaluate 0.4.6 12:20 [184/1957] fastapi 0.136.3 fastapi-cli 0.0.32 fastapi-cloud-cli 0.23.0 fastar 0.11.0 filelock 3.32.3 frozenlist 1.8.0 fsspec 2026.6.0 googleapis-common-protos 1.75.1 gpt-oss 0.0.9 graphql-core 3.2.11 grpcio 1.83.0 grpcio-tools 1.83.1 h11 0.16.0 harfile 0.5.0 hf-xet 1.6.0 html2text 2025.4.15 httpcore 1.0.9 httpcore2 2.10.0 httptools 0.8.0 httpx 0.28.1 httpx2 2.10.0 huggingface_hub 1.28.0 hypothesis 6.165.10 hypothesis-graphql 0.13.1 hypothesis-jsonschema 0.23.1 idna 3.18 ijson 3.5.1 impi-rt 2021.18.0 iniconfig 2.3.0 intel-cmplr-lib-rt 2026.0.0 intel-cmplr-lib-ur 2026.0.0 intel-cmplr-lic-rt 2026.0.0 intel-opencl-rt 2026.0.0 intel-openmp 2026.0.0 intel-pti 0.17.0 intel-sycl-rt 2026.0.0 interegular 0.3.3 Jinja2 3.1.6 jiter 0.16.0 jiwer 4.0.0 jmespath 1.1.0 joblib 1.5.3 jsonschema 4.26.0 jsonschema_rs 0.49.9 jsonschema-specifications 2025.9.1 jupyter_client 8.9.1 jupyter_core 5.9.1 lark 1.2.2 lazy-loader 0.5 librosa 1.0.0 llguidance 1.7.6 llvmlite 0.47.0 lm_eval 0.4.12 12:20 [131/1957] lm-format-enforcer 0.11.3 loguru 0.7.3 lxml 6.1.1 markdown-it-py 4.2.0 MarkupSafe 3.0.3 mbstrdecoder 1.1.5 mcp 2.0.0 mcp-types 2.0.0 mdurl 0.1.2 mistral_common 1.11.7 mkl 2026.0.0 model-hosting-container-standards 0.1.16 modelscope 1.37.1 more-itertools 11.1.0 mpmath 1.3.0 msgpack 1.2.1 msgspec 0.21.1 mteb 2.19.4 multidict 6.7.1 multiprocess 0.70.19 nanobind 3.0.1 narwhals 2.24.0 networkx 3.6.1 ninja 1.13.0 nixl 1.3.2 nixl-cu12 1.3.2 nltk 3.10.3 num2words 0.5.14 numba 0.65.0 numpy 2.3.5 oneccl 2022.0.0 oneccl-devel 2022.0.0 onemkl-license 2026.0.0 onemkl-sycl-blas 2026.0.0 onemkl-sycl-dft 2026.0.0 onemkl-sycl-lapack 2026.0.0 onemkl-sycl-rng 2026.0.0 onemkl-sycl-sparse 2026.0.0 openai 3.2.0 openai-harmony 0.0.8 opencv-python-headless 5.0.0.93 opentelemetry-api 1.44.0 opentelemetry-exporter-otlp 1.44.0 opentelemetry-exporter-otlp-proto-common 1.44.0 opentelemetry-exporter-otlp-proto-grpc 1.44.0 opentelemetry-exporter-otlp-proto-http 1.44.0 opentelemetry-proto 1.44.0 opentelemetry-sdk 1.44.0 opentelemetry-semantic-conventions 0.65b0 opentelemetry-semantic-conventions-ai 0.5.1 outlines_core 0.2.14 packaging 26.3 pandas 3.0.5 partial-json-parser 0.2.1.1.post7 pathvalidate 3.3.1 pillow 12.3.0 pip 26.2.1 platformdirs 4.11.3 pluggy 1.6.0 polars 1.43.2 polars-runtime-32 1.43.2 pooch 1.9.0 portalocker 4.1.0 pqdm 0.2.0 prometheus_client 0.26.0 prometheus-fastapi-instrumentator 8.1.0 propcache 0.5.2 protobuf 7.35.1 psutil 7.2.2 py-cpuinfo 9.0.0 pyarrow 25.0.1 pybase64 1.5.0 pycountry 26.2.16 pycparser 3.0 pycryptodomex 3.23.0 pydantic 2.13.4 pydantic_core 2.46.4 pydantic-extra-types 2.11.1 pydantic-settings 2.15.0 pyelftools 0.33 Pygments 2.21.0 PyJWT 2.13.0 pyrate-limiter 4.4.0 PyStemmer 3.1.0 pytablewriter 1.2.1 pytest 9.1.1 pytest-asyncio 1.4.0 pytest-cov 7.1.0 pytest-forked 1.7.5 pytest-rerunfailures 16.6 pytest-shard 0.1.2 pytest-timeout 2.4.0 python-dateutil 2.9.0.post0 python-dotenv 1.2.3 python-json-logger 4.2.0 python-multipart 0.0.32 pytrec_eval-terrier 0.5.10 pytz 2026.3.post1 PyYAML 6.0.3 pyzes 0.1.1 pyzmq 27.1.0 RapidFuzz 3.14.5 ray 2.58.0 referencing 0.37.0 regex 2026.7.19 12:20 [26/1957] requests 2.34.2 rich 15.0.0 rich-toolkit 0.20.3 rignore 0.8.1 rouge_score 0.1.2 rpds-py 2026.6.3 sacrebleu 2.6.0 safetensors 0.8.0 schemathesis 4.24.3 scikit-learn 1.9.0 scipy 1.18.0 semantic-version 2.10.0 sentence-transformers 5.7.0 sentencepiece 0.2.1 sentry-sdk 2.68.0 setproctitle 1.3.7 setuptools 80.10.2 setuptools-rust 1.13.0 setuptools-scm 10.2.3 shellingham 1.5.4 simsimd 6.5.16 six 1.17.0 sniffio 1.3.1 sortedcontainers 2.4.0 soundfile 0.14.0 soxr 1.1.0 sqlitedict 2.1.0 sse-starlette 3.4.8 starlette 1.6.0 starlette-testclient 0.4.1 stringzilla 5.1.2 structlog 26.1.0 supervisor 4.3.0 sympy 1.14.0 tabledata 1.3.5 tabulate 0.10.0 tbb 2023.0.0 tblib 3.2.2 tcmlib 1.5.0 tcolorpy 0.1.7 tenacity 9.1.4 termcolor 3.3.0 threadpoolctl 3.6.0 tiktoken 0.14.0 timm 1.0.28 tokenizers 0.22.2 torch 2.13.0+xpu torchaudio 2.11.0+xpu torchcodec 0.16.0+cpu torchvision 0.28.0+xpu tornado 6.5.8 tqdm 4.70.0 traitlets 5.16.1 transformers 5.15.0 triton 3.7.2+xpu triton-xpu 3.7.2 truststore 0.10.4 typepy 1.3.5 typer 0.27.1 typing_extensions 4.16.0 typing-inspection 0.4.4 umf 1.1.0 urllib3 2.7.0 uvicorn 0.52.3 uvloop 0.22.1 vcs-versioning 2.3.4 vllm 0.29.0+xpu vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils vllm-xpu-kernels 0.1.14.1 watchfiles 1.2.0 websockets 17.0.1 Werkzeug 3.1.8 wheel 0.48.0 word2number 1.1 xgrammar 0.2.3 xxhash 4.0.1 yarl 1.24.5

compose (Works!)

services:
  vllm-xpu:
    image: vllm/vllm-openai-xpu:latest
    container_name: vllm-xpu-qwen38
    privileged: true
    network_mode: host
    shm_size: "32g"

    devices:
      - /dev/dri:/dev/dri

    volumes:
      - ${HOME}/.cache/huggingface:/root/.cache/huggingface

    environment:
      - HF_HOME=/root/.cache/huggingface
      - HF_TOKEN=${HF_TOKEN:-}
      - no_proxy=localhost,127.0.0.1
      - ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE
      - ZE_AFFINITY_MASK=0
      - VLLM_WORKER_MULTIPROC_METHOD=spawn
      - PYTORCH_ALLOC_CONF=expandable_segments:True
      - VLLM_XPU_ENABLE_XPU_GRAPH=1

    ports:
      - "9000:9000"

    entrypoint: ["vllm", "serve"]
    command:
      - "SergiioB/Qwen3.8-27B-GPTQ-Int4-sym-G128-MTP-BF16"
      - "--served-model-name=qwen38"
      - "--port=9000"
      - "--host=0.0.0.0"
      - "--quantization=gptq"
      - "--dtype=float16"
      - "--max-model-len=131072"
      - "--gpu-memory-utilization=0.88"
      - "--kv-cache-dtype=fp8"
      - "--max-num-seqs=1"
      - "--max-num-batched-tokens=8192"
      - "--trust-remote-code"
      - "--enable-auto-tool-choice"
      - "--tool-call-parser=qwen3_xml"
      - "--reasoning-parser=qwen3"
      - "--limit-mm-per-prompt={\"image\":10}"
      - "--speculative-config={\"method\":\"mtp\",\"num_speculative_tokens\":4}"

    restart: no

sudo usermod -aG docker $USER
docker compose up

docker.io

sudo apt install docker.io
sudo docker pull intel/vllm

docker run -t -d --shm-size 10g --net=host --ipc=host --privileged \
  -v /dev/dri/by-path:/dev/dri/by-path --name=vllm-test \
  --device /dev/dri:/dev/dri --entrypoint=/bin/bash intel/vllm:latest

source

uv venv --python 3.12 --seed --managed-python
source .venv/bin/activate

git clone https://github.com/vllm-project/vllm.git
cd vllm
pip install --upgrade pip
pip install -v -r requirements/xpu.txt

pip uninstall -y triton triton-xpu
pip install triton-xpu==3.7.2 --extra-index-url https://download.pytorch.org/whl/xpu

VLLM_TARGET_DEVICE=xpu pip install --no-build-isolation -e . -v

# check device
python -c "import torch; print(torch.xpu.is_available()); print(torch.xpu.device_count())"

# download model
hf download ciocan/gemma-4-E4B-it-W4A16

# Run
vllm serve unsloth/gemma-4-E4B-it-qat-GGUF --device xpu

LM/vLLM

Contents

Verify environments in container

compose (Works!)

docker.io

source

TODO

Navigation menu

LM/vLLM

Verify environments in container

compose (Works!)

docker.io

source

TODO

Navigation menu

Search