재현 가능한 ML을 위한 스토리지와 버전 관리

Git commit SHA
  └─ training image tag + OCI digest
       └─ dataset URI + checksum/catalog version
            └─ lockfile hash
                 └─ RayJob name/UID
                      └─ MLflow run ID
                           └─ registered model version
                                └─ serving artifact URI
                                     └─ GitOps commit

import os

import mlflow

REQUIRED = (
    "GIT_COMMIT_SHA",
    "TRAINING_IMAGE",
    "DATASET_URI",
    "DATASET_VERSION",
    "LOCKFILE_SHA256",
)

missing = [name for name in REQUIRED if not os.getenv(name)]
if missing:
    raise RuntimeError(f"missing provenance values: {', '.join(missing)}")

if mlflow.active_run() is None:
    raise RuntimeError("provenance tags must be logged inside the active training run")

mlflow.set_tags(
    {
        "source.git_commit": os.environ["GIT_COMMIT_SHA"],
        "runtime.training_image": os.environ["TRAINING_IMAGE"],
        "data.uri": os.environ["DATASET_URI"],
        "data.version": os.environ["DATASET_VERSION"],
        "runtime.lockfile_sha256": os.environ["LOCKFILE_SHA256"],
        "kubernetes.rayjob": os.getenv("RAY_JOB_NAME", "local"),
    }
)

s3://ml-datasets/digits/
├── manifests/
│   └── 2026-07-14.json
└── objects/
    └── sha256-7f4c...c21.parquet

{
  "dataset": "digits",
  "created_at": "2026-07-14T00:00:00Z",
  "object_uri": "s3://ml-datasets/digits/objects/sha256-7f4c...c21.parquet",
  "sha256": "7f4c...c21",
  "row_count": 1797,
  "schema_version": "1",
  "label_column": "target"
}

kubectl get storageclass

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: ray-data
  namespace: ml-training
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: nfs-csi
  resources:
    requests:
      storage: 20Gi

kubectl apply -f ray-data-pvc.yaml
kubectl wait --for=jsonpath='{.status.phase}'=Bound \
  pvc/ray-data -n ml-training --timeout=120s

# RayJob의 rayClusterSpec 아래에서 필요한 부분만 발췌
headGroupSpec:
  template:
    spec:
      volumes:
        - name: ray-data
          persistentVolumeClaim:
            claimName: ray-data
      containers:
        - name: ray-head
          volumeMounts:
            - name: ray-data
              mountPath: /mnt/datasets
workerGroupSpecs:
  - groupName: workers
    replicas: 2
    minReplicas: 2
    maxReplicas: 2
    rayStartParams: {}
    template:
      spec:
        volumes:
          - name: ray-data
            persistentVolumeClaim:
              claimName: ray-data
        containers:
          - name: ray-worker
            volumeMounts:
              - name: ray-data
                mountPath: /mnt/datasets

test -n "${MINIO_ACCESS_KEY:-}" || { echo "MINIO_ACCESS_KEY is required"; exit 1; }
test -n "${MINIO_SECRET_KEY:-}" || { echo "MINIO_SECRET_KEY is required"; exit 1; }

kubectl create secret generic ml-s3-credentials \
  --namespace ml-training \
  --from-literal=AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" \
  --from-literal=AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" \
  --from-literal=AWS_DEFAULT_REGION="${MINIO_REGION:-us-east-1}" \
  --dry-run=client -o yaml | kubectl apply -f -

env:
  - name: MLFLOW_TRACKING_URI
    value: http://mlflow.mlflow.svc.cluster.local:5000
  - name: MLFLOW_S3_ENDPOINT_URL
    value: https://minio.lab.example.com
envFrom:
  - secretRef:
      name: ml-s3-credentials

apiVersion: v1
kind: Secret
metadata:
  name: kserve-s3-credentials
  namespace: ml-serving
  annotations:
    serving.kserve.io/s3-endpoint: minio.minio.svc.cluster.local:9000
    serving.kserve.io/s3-usehttps: "0"
    serving.kserve.io/s3-region: us-east-1
    serving.kserve.io/s3-useanoncredential: "false"
type: Opaque
stringData:
  AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY}
  AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY}
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kserve-model-reader
  namespace: ml-serving
secrets:
  - name: kserve-s3-credentials

mlflow>=3
scikit-learn>=1.5
ray[default]

[project]
name = "digits-trainer"
version = "0.1.0"
requires-python = ">=3.11,<3.12"
dependencies = [
  "boto3",
  "mlflow",
  "ray[default]",
  "scikit-learn",
]

[dependency-groups]
dev = ["pytest", "ruff"]

uv lock
uv sync --frozen
uv lock --check
git add pyproject.toml uv.lock

ARG UV_IMAGE
ARG RAY_BASE_IMAGE

FROM ${UV_IMAGE} AS uv
FROM ${RAY_BASE_IMAGE}

USER root
COPY --from=uv /uv /uvx /bin/
RUN mkdir -p /app && chown 1000:100 /app

WORKDIR /app

COPY --chown=1000:100 pyproject.toml uv.lock ./

USER 1000
RUN uv sync --frozen --no-dev --no-install-project

COPY --chown=1000:100 src ./src
ENV PATH="/app/.venv/bin:${PATH}" \
    PYTHONPATH=/app \
    PYTHONDONTWRITEBYTECODE=1

CMD ["python", "-m", "src.train"]

from mlflow.models import infer_signature

signature = infer_signature(x_train, model.predict(x_train))

mlflow.sklearn.log_model(
    sk_model=model,
    artifact_path="model",
    signature=signature,
    input_example=x_train[:2],
    registered_model_name="digits-classifier",
)

# 잠금 상태가 최신인지
uv lock --check

# 저장 직후 같은 잠금 환경에서 다시 읽고 예측할 수 있는지
uv run pytest -q tests/test_model_roundtrip.py

# tests/test_model_roundtrip.py
import mlflow.sklearn
from sklearn.datasets import load_digits
from sklearn.ensemble import RandomForestClassifier

def test_model_roundtrip(tmp_path):
    dataset = load_digits()
    trained_model = RandomForestClassifier(n_estimators=5, random_state=42)
    trained_model.fit(dataset.data, dataset.target)

    model_path = tmp_path / "model"
    mlflow.sklearn.save_model(trained_model, str(model_path))
    restored = mlflow.sklearn.load_model(str(model_path))
    batch = dataset.data[:2]
    assert restored.predict(batch).shape == (2,)

# PVC가 실제로 Bound인가
kubectl get pvc -n ml-training ray-data

# Ray head와 worker 모두 같은 볼륨을 갖는가
kubectl get pod -n ml-training -l ray.io/cluster \
  -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .spec.volumes[*]}{.name}{","}{end}{"\n"}{end}'

# Secret 값이 아니라 키 이름만 확인한다
kubectl get secret -n ml-training ml-s3-credentials \
  -o jsonpath='{range $k,$v := .data}{$k}{"\n"}{end}'

# MLflow run에 계보 태그가 있는지 API/UI에서 확인한다
# source.git_commit, runtime.training_image, data.uri, data.version, runtime.lockfile_sha256

kubectl describe pvc -n ml-training ray-data
kubectl get storageclass nfs-csi -o yaml
kubectl get events -n ml-training --sort-by=.lastTimestamp