RunPod vs Modal: which serverless GPU platform should you choose?

import runpod
import os
from openai import OpenAI

# RunPod requires pre-configuring a GPU pod template in the UI,
# then managing it via API or SDK.

# 1. Start a pod (or use existing)
pod = runpod.create_pod(
    cloud_type="SECURE_CLOUD",
    gpu_count=1,
    volume_in_gb=40,
    container_disk_in_gb=20,
    min_vcpu_count=4,
    min_memory_in_gb=20,
    gpu_class_id="NVIDIA_A100",
    name="vllm-inference"
)
pod_id = pod['id']

# 2. Wait for pod to be ready and get connection details
while True:
    pod_status = runpod.get_pod(pod_id)
    if pod_status['runtime']['ports']:
        port = pod_status['runtime']['ports'][0]['publicIp']
        break
    time.sleep(5)

# 3. Send inference request via OpenAI-compatible endpoint
# (assumes vLLM is running inside the pod: must be in Dockerfile or startup script)
client = OpenAI(
    base_url=f"http://{port}:8000}/v1",
    api_key="not-needed"
)

response = client.chat.completions.create(
    model="meta-llama/Llama-2-7b-hf",
    messages=[{"role": "user", "content": "Explain quantum computing"}],
    max_tokens=100
)

print(response.choices[0].message.content)

# 4. Stop pod to save costs
runpod.stop_pod(pod_id)

import modal
from openai import OpenAI

# Modal wraps the entire deployment in a Python function with decorators
app = modal.App(name="vllm-inference")

# Define the model and requirements
model_name = "meta-llama/Llama-2-7b-hf"
vllm_image = (
    modal.Image.debian_slim()
    .pip_install(
        "vllm==0.4.2",
        "openai",
        "torch",
        "transformers"
    )
)

@app.cls(
    image=vllm_image,
    gpu="A100",  # Automatic GPU allocation per invocation
    timeout=600,
    concurrency_limit=1  # One request per GPU instance
)
class VLLMInference:
    def __enter__(self):
        """Modal automatically calls this on cold start."""
        from vllm import LLM
        self.llm = LLM(model=model_name)
    
    @modal.method()
    def generate(self, prompt: str) -> str:
        """Inference method: Modal handles HTTP routing."""
        from vllm import SamplingParams
        params = SamplingParams(max_tokens=100, temperature=0.7)
        output = self.llm.generate(prompt, params)
        return output[0].outputs[0].text

# Deploy and invoke
@app.function()
def run_inference():
    model = VLLMInference()
    result = model.generate.remote("Explain quantum computing")
    return result

if __name__ == "__main__":
    # Deploy: modal deploy inference.py
    # Or invoke directly:
    result = run_inference.remote()
    print(result)