Modal vs Runpod: serverless GPU vs managed cloud compute

from modal import App, Image, gpu
import os

app = App(name="llm-inference")

image = Image.debian_slim().pip_install(
    "vllm==0.4.0",
    "pydantic"
)

@app.cls(
    gpu="H100",
    image=image,
    concurrency_limit=10
)
class LLMModel:
    def __enter__(self):
        from vllm import LLM
        self.model = LLM("meta-llama/Llama-2-7b-hf")
    
    @app.method()
    def infer(self, prompt: str) -> str:
        # Modal handles scaling: multiple instances created based on traffic
        output = self.model.generate([prompt], max_tokens=100)
        return output[0].outputs[0].text

@app.function()
def api():
    from fastapi import FastAPI
    web_app = FastAPI()
    
    @web_app.post("/inference")
    async def inference(prompt: str):
        result = LLMModel().infer.call(prompt)
        return {"response": result}
    
    return web_app

if __name__ == "__main__":
    app.serve()

# Runpod: Create pod via UI/API first, then deploy handler to reserved instance
import runpod
import json
from vllm import LLM
import os

# Initialize model on pod startup (persists across requests)
model = LLM("meta-llama/Llama-2-7b-hf")

def inference_handler(event):
    # Runpod serverless endpoint handler
    prompt = event["input"]["prompt"]
    
    # Model already loaded in GPU memory; no cold start per request
    output = model.generate([prompt], max_tokens=100)
    
    return {"response": output[0].outputs[0].text}

# For reserved pods: deploy via Dockerfile + pod creation
# For serverless pods: this handler runs on Runpod infrastructure
runpod.serverless.start({"handler": inference_handler})

# Manual setup required:
# 1. Create pod: runpod-cli create --gpu-name "H100" --docker-image "custom"
# 2. Configure networking: SSH tunnel or ngrok proxy
# 3. Scale: manually create multiple pod instances