Multi-node serving on Dynamo

# EKS g6.8xlarge, one NVIDIA L4 per node. A single L4 can't hold the 14B model,
# so the gang spans two nodes. This node has enough memory to hold the weights as
# they load, and the 100 GB disk holds the vLLM image.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceClass
metadata:
  name: l4-1x-g6-8xlarge
spec:
  description: "EKS g6.8xlarge, 1x NVIDIA L4"
  provisioning:
    provider: EKS
    eks:
      instanceType: g6.8xlarge
      diskSizeGb: 100
      accelerator:
        type: nvidia-l4
        count: 1
  devices:
  - name: gpu
    claim: DRA
    driver: gpu.nvidia.com
    deviceClassName: gpu.nvidia.com
    count: 1
    attributes:
      architecture: { string: Ada Lovelace }
    capacity:
      memory: { value: "23034Mi" }   # L4's real reported VRAM (not the nominal 24GB)

# An EKS cluster running the Dynamo serving stack, with a two-node L4 pool so a
# gang can span both nodes. spec.stack: Dynamo installs Grove and the KAI
# Scheduler, which gang-schedule the leader and worker together and compose them
# as a Grove PodCliqueSet, and ModelExpress, which serves cached weights to the
# gang and moves them between replicas peer-to-peer.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceCluster
metadata:
  name: eks-us-east-dynamo
  labels:
    modelplane.ai/region: us-east
    modelplane.ai/stack: dynamo
spec:
  stack: Dynamo
  cluster:
    source: EKS
    eks:
      region: us-east-1
  nodePools:
  - name: gpu-l4
    className: l4-1x-g6-8xlarge
    nodeCount: 2
    minNodeCount: 2
    maxNodeCount: 2
    zones:
    - us-east-1b

# The shared read-write-many cache the gang serves from, hydrated once from
# Hugging Face. Both gang pods mount it and read weights from it over EFS,
# instead of each pulling its own copy. Qwen2.5-14B is open, so it needs no
# token. Its FP16 weights are about 29 GB, so sizeGiB leaves headroom. The
# clusterSelector stages it only on the Dynamo cluster, not the getting started
# tour's, and the deployment follows it there.
apiVersion: modelplane.ai/v1alpha1
kind: ModelCache
metadata:
  name: qwen2-5-14b
  namespace: ml-team
spec:
  source: HuggingFace
  huggingFace:
    repo: Qwen/Qwen2.5-14B-Instruct
    sizeGiB: 40
  clusterSelector:
    matchLabels:
      modelplane.ai/stack: dynamo

# Qwen2.5-14B served across two L4 nodes as a gang. The FP16 weights (~29 GB)
# don't fit one L4's 23 GB, so the engine is a Leader + Worker gang,
# pipeline-parallel across two g6.8xlarge nodes with one L4 each. Both pods mount
# the shared ModelCache.
#
# The cluster runs the Dynamo stack, so Grove and the KAI Scheduler gang-schedule
# the two pods, and Modelplane composes them as a Grove PodCliqueSet.
# $(MODELPLANE_LEADER_ADDRESS) resolves to the leader on Dynamo, but
# $(MODELPLANE_RANK) isn't injected there yet (modelplaneai/modelplane#418), so
# each command sets its own --node-rank: 0 on the leader, and
# $$((GROVE_PCLQ_POD_INDEX + 1)) on the worker ($$ escapes past Kubernetes,
# leaving $((...)) for the shell to evaluate).
#
# Notes on the engine flags:
#   --pipeline-parallel-size=2 splits the model across the two nodes;
#     --tensor-parallel-size=1 keeps one GPU per node. Pipeline parallelism sends
#     only activations between nodes, so it stays light on the network.
#   --distributed-executor-backend=mp is vLLM's native multiprocessing multi-node
#     path; vllm/vllm-openai:v0.23.0 no longer ships Ray.
#   --load-format modelexpress loads weights through the ModelExpress server the
#     Dynamo stack runs. The first replica seeds from the cache and publishes
#     itself; later replicas pull their weights peer-to-peer from a replica that
#     already has them. The vLLM image doesn't ship the loader, so pip install it
#     first. --load-format=runai_streamer is the alternative that always reads the
#     cache directly, on any stack.
#   --max-model-len=8192 caps context so the KV cache fits alongside the weights.
apiVersion: modelplane.ai/v1alpha1
kind: ModelDeployment
metadata:
  name: qwen2-5-14b
  namespace: ml-team
spec:
  replicas: 1
  template:
    spec:
      modelCacheRef:
        name: qwen2-5-14b
      engines:
      - name: qwen
        members:
        - role: Leader
          nodeSelector:
            devices:
            - name: gpu
              count: 1
              selectors:
              - cel: |
                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.23.0
                command:
                - /bin/sh
                - -c
                - >-
                  pip install --index-url https://pypi.nvidia.com modelexpress &&
                  exec vllm serve Qwen/Qwen2.5-14B-Instruct
                  --served-model-name=$(MODELPLANE_SERVED_MODEL_NAME)
                  --tensor-parallel-size=1
                  --pipeline-parallel-size=2
                  --distributed-executor-backend=mp
                  --nnodes=2 --node-rank=0
                  --master-addr=$(MODELPLANE_LEADER_ADDRESS)
                  --load-format modelexpress
                  --max-model-len=8192
                  --gpu-memory-utilization=0.90
                  --port=8000
        - role: Worker
          worker:
            nodes: 1
          nodeSelector:
            devices:
            - name: gpu
              count: 1
              selectors:
              - cel: |
                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.23.0
                command:
                - /bin/sh
                - -c
                - >-
                  pip install --index-url https://pypi.nvidia.com modelexpress &&
                  exec vllm serve Qwen/Qwen2.5-14B-Instruct
                  --served-model-name=$(MODELPLANE_SERVED_MODEL_NAME)
                  --tensor-parallel-size=1
                  --pipeline-parallel-size=2
                  --distributed-executor-backend=mp
                  --nnodes=2 --node-rank=$$((GROVE_PCLQ_POD_INDEX + 1))
                  --master-addr=$(MODELPLANE_LEADER_ADDRESS)
                  --headless
                  --load-format modelexpress
                  --max-model-len=8192
                  --gpu-memory-utilization=0.90
                  --port=8000

# Exposes the gang as one model a caller can name. Modelplane composes one
# ModelEndpoint per replica, labeled modelplane.ai/deployment: qwen2-5-14b, so
# this selector reaches it. Name the model,
# ml-team/qwen2-5-14b, in a request to an InferenceGateway that serves it.
apiVersion: modelplane.ai/v1alpha1
kind: ModelService
metadata:
  name: qwen2-5-14b
  namespace: ml-team
spec:
  endpoints:
  - name: qwen2-5-14b
    selector:
      matchLabels:
        modelplane.ai/deployment: qwen2-5-14b