Multi-region ML deployment strategy

# DVC multi-region storage config (.dvc/config)
['remote "us-east-1"']
    url = s3://ml-models-us-east-1/dvc-store
    region = us-east-1

['remote "eu-central-1"']
    url = s3://ml-models-eu-central-1/dvc-store
    region = eu-central-1

['remote "asia-southeast-1"']
    url = s3://ml-models-ap-southeast-1/dvc-store
    region = ap-southeast-1

[core]
    remote = us-east-1
    autostage = true

# dvc.yaml: train once, push to all regions
stages:
  train:
    cmd: python train.py --output model.pkl
    deps:
      - train.py
      - data/train.csv
    outs:
      - model.pkl:
          hash: md5
          md5: abc123def456
          size: 524288000

# Kubernetes deployment (k8s-us-east-1.yaml)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ml-inference-us-east-1
  namespace: ml-serving
spec:
  replicas: 3
  selector:
    matchLabels:
      app: ml-inference
      region: us-east-1
  template:
    metadata:
      labels:
        app: ml-inference
        region: us-east-1
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - ml-inference
                topologyKey: kubernetes.io/hostname
      containers:
        - name: inference
          image: ml-inference:v1.2.3
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8000
              name: http
          env:
            - name: DVC_REMOTE
              value: s3://ml-models-us-east-1/dvc-store
            - name: MODEL_VERSION
              value: "v1.2.3"
            - name: REGION
              value: us-east-1
            - name: AWS_REGION
              value: us-east-1
            - name: MLFLOW_REGISTRY_URI
              value: https://mlflow-central.example.com
          resources:
            requests:
              memory: "2Gi"
              cpu: "1000m"
            limits:
              memory: "4Gi"
              cpu: "2000m"
          livenessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /ready
              port: 8000
            initialDelaySeconds: 10
            periodSeconds: 5
          volumeMounts:
            - name: model-cache
              mountPath: /models
      volumes:
        - name: model-cache
          emptyDir:
            sizeLimit: 2Gi
      serviceAccountName: ml-inference
---
apiVersion: v1
kind: Service
metadata:
  name: ml-inference-svc-us-east-1
  namespace: ml-serving
spec:
  type: ClusterIP
  ports:
    - port: 80
      targetPort: 8000
      name: http
  selector:
    app: ml-inference
    region: us-east-1

# Route 53 health checks + multi-region routing (AWS)
# CloudFormation snippet (use Terraform in prod)
AWSTemplateFormatVersion: '2010-09-09'
Resources:
  MLInferenceHealthCheckUSEast1:
    Type: AWS::Route53::HealthCheck
    Properties:
      Type: HTTPS
      ResourcePath: /health
      FullyQualifiedDomainName: inference-api-us-east-1.example.com
      Port: 443
      RequestInterval: 30
      FailureThreshold: 3
      MeasureLatency: true
      HealthCheckTags:
        - Key: Name
          Value: ml-inference-us-east-1

  MLInferenceHealthCheckEUCentral1:
    Type: AWS::Route53::HealthCheck
    Properties:
      Type: HTTPS
      ResourcePath: /health
      FullyQualifiedDomainName: inference-api-eu-central-1.example.com
      Port: 443
      RequestInterval: 30
      FailureThreshold: 3
      MeasureLatency: true
      HealthCheckTags:
        - Key: Name
          Value: ml-inference-eu-central-1

  MLInferenceMultiRegionDNS:
    Type: AWS::Route53::RecordSet
    Properties:
      HostedZoneId: Z123ABC456
      Name: inference-api.example.com
      Type: A
      SetIdentifier: weighted-routing
      GeoLocation:
        CountryCode: US
      AliasTarget:
        HostedZoneId: Z35SXDOTRQ7X7K
        DNSName: inference-api-us-east-1.example.com
        EvaluateTargetHealth: true
      Weight: 50
  MLInferenceMultiRegionDNSEU:
    Type: AWS::Route53::RecordSet
    Properties:
      HostedZoneId: Z123ABC456
      Name: inference-api.example.com
      Type: A
      SetIdentifier: weighted-routing-eu
      GeoLocation:
        ContinentCode: EU
      AliasTarget:
        HostedZoneId: Z32O12XQLNTSW2
        DNSName: inference-api-eu-central-1.example.com
        EvaluateTargetHealth: true
      Weight: 50

# MLflow model registry sync script (run every 5 min via K8s CronJob)
apiVersion: batch/v1
kind: CronJob
metadata:
  name: mlflow-registry-sync
  namespace: ml-ops
spec:
  schedule: "*/5 * * * *"
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: mlflow-sync
          containers:
            - name: sync
              image: python:3.11-slim
              command:
                - /bin/bash
                - -c
                - |
                  pip install mlflow boto3 google-cloud-storage -q
                  python /scripts/sync_registry.py
              env:
                - name: MLFLOW_TRACKING_URI
                  value: https://mlflow-central.example.com
                - name: REGIONS
                  value: "us-east-1,eu-central-1,ap-southeast-1"
              volumeMounts:
                - name: sync-script
                  mountPath: /scripts
          volumes:
            - name: sync-script
              configMap:
                name: mlflow-sync-script
                defaultMode: 0755
          restartPolicy: OnFailure