Commit에서 Cluster까지: GitLab, Harbor, Argo CD end-to-end 실습

git --version
helm version
kubectl version --client
kubectl get gateway -n nginx-gateway platform-gateway
kubectl get applications.argoproj.io -n argocd

kubectl create namespace student-01 --dry-run=client -o yaml | kubectl apply -f -
kubectl label namespace student-01 gateway-access=true --overwrite

student-01/student-01-web-code
platform/student-01-web-deploy

git clone https://gitlab.lab.example.com/student-01/student-01-web-code.git
git clone https://gitlab.lab.example.com/platform/student-01-web-deploy.git

CI push:       HARBOR_ROBOT_USER / HARBOR_ROBOT_PASSWORD
Cluster pull:  HARBOR_PULL_USER  / HARBOR_PULL_PASSWORD

student-01-web-deploy/
├── argocd/
│   ├── application-dev.yaml
│   └── project.yaml
└── charts/
    └── web/
        ├── Chart.yaml
        ├── values.yaml
        ├── values-dev.yaml
        └── templates/
            ├── _helpers.tpl
            ├── deployment.yaml
            ├── httproute.yaml
            └── service.yaml

cd student-01-web-deploy
mkdir -p argocd charts/web/templates

apiVersion: v2
name: web
description: Web application deployed by GitOps
type: application
version: 0.1.0
appVersion: "1.0.0"

replicaCount: 1

image:
  repository: harbor.lab.example.com/mlops-training/student-01-web
  tag: "REPLACED_BY_CI"
  pullPolicy: IfNotPresent

imagePullSecrets:
  - name: harbor-pull

service:
  port: 80

gateway:
  enabled: true
  name: platform-gateway
  namespace: nginx-gateway
  sectionName: https
  hostname: web-dev.lab.example.com

resources:
  requests:
    cpu: 50m
    memory: 64Mi
  limits:
    cpu: 250m
    memory: 128Mi

replicaCount: 1

image:
  repository: harbor.lab.example.com/mlops-training/student-01-web
  tag: "REPLACED_BY_CI"

gateway:
  hostname: web-dev.lab.example.com

{{- define "web.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{- define "web.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}

{{- define "web.selectorLabels" -}}
app.kubernetes.io/name: {{ include "web.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end -}}

{{- define "web.labels" -}}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
{{ include "web.selectorLabels" . }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end -}}

apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "web.fullname" . }}
  labels:
    {{- include "web.labels" . | nindent 4 }}
spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      {{- include "web.selectorLabels" . | nindent 6 }}
  template:
    metadata:
      labels:
        {{- include "web.selectorLabels" . | nindent 8 }}
    spec:
      imagePullSecrets:
        {{- toYaml .Values.imagePullSecrets | nindent 8 }}
      containers:
        - name: web
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
          readinessProbe:
            httpGet:
              path: /
              port: http
            initialDelaySeconds: 2
            periodSeconds: 5
          livenessProbe:
            httpGet:
              path: /
              port: http
            initialDelaySeconds: 10
            periodSeconds: 10
          resources:
            {{- toYaml .Values.resources | nindent 12 }}

apiVersion: v1
kind: Service
metadata:
  name: {{ include "web.fullname" . }}
  labels:
    {{- include "web.labels" . | nindent 4 }}
spec:
  type: ClusterIP
  selector:
    {{- include "web.selectorLabels" . | nindent 4 }}
  ports:
    - name: http
      port: {{ .Values.service.port }}
      targetPort: http
      protocol: TCP

{{- if .Values.gateway.enabled }}
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: {{ include "web.fullname" . }}
  labels:
    {{- include "web.labels" . | nindent 4 }}
spec:
  parentRefs:
    - name: {{ .Values.gateway.name }}
      namespace: {{ .Values.gateway.namespace }}
      sectionName: {{ .Values.gateway.sectionName }}
  hostnames:
    - {{ .Values.gateway.hostname | quote }}
  rules:
    - matches:
        - path:
            type: PathPrefix
            value: /
      backendRefs:
        - name: {{ include "web.fullname" . }}
          port: {{ .Values.service.port }}
{{- end }}

apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
  name: web-training
  namespace: argocd
spec:
  description: Web training applications
  sourceRepos:
    - https://gitlab.lab.example.com/platform/student-01-web-deploy.git
  destinations:
    - server: https://kubernetes.default.svc
      namespace: student-01
  namespaceResourceWhitelist:
    - group: apps
      kind: Deployment
    - group: ""
      kind: Service
    - group: gateway.networking.k8s.io
      kind: HTTPRoute

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: student-01-web-dev
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io
spec:
  project: web-training
  source:
    repoURL: https://gitlab.lab.example.com/platform/student-01-web-deploy.git
    targetRevision: main
    path: charts/web
    helm:
      releaseName: web
      valueFiles:
        - values-dev.yaml
  destination:
    server: https://kubernetes.default.svc
    namespace: student-01
  syncPolicy:
    automated:
      enabled: true
      prune: true
      selfHeal: true

helm lint charts/web -f charts/web/values-dev.yaml
helm template web charts/web -f charts/web/values-dev.yaml > /tmp/web-rendered.yaml
kubectl apply --dry-run=client -f /tmp/web-rendered.yaml
git add .
git commit -m "feat: add web Helm chart and Argo CD application"
git branch -M main
git push -u origin main

read -r -p "GitLab Deploy Token username: " GITLAB_REPO_USER
read -r -s -p "GitLab Deploy Token: " GITLAB_REPO_TOKEN
printf '\n'
export GITLAB_REPO_USER GITLAB_REPO_TOKEN

envsubst <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Secret
metadata:
  name: student-01-web-deploy-repo
  namespace: argocd
  labels:
    argocd.argoproj.io/secret-type: repository
stringData:
  type: git
  url: https://gitlab.lab.example.com/platform/student-01-web-deploy.git
  username: ${GITLAB_REPO_USER}
  password: ${GITLAB_REPO_TOKEN}
EOF

export HARBOR_REGISTRY=harbor.lab.example.com
read -r -p "Harbor pull robot username: " HARBOR_PULL_USER
read -r -s -p "Harbor pull robot secret: " HARBOR_PULL_PASSWORD
printf '\n'
export HARBOR_PULL_USER HARBOR_PULL_PASSWORD

HARBOR_AUTH="$(printf '%s:%s' "$HARBOR_PULL_USER" "$HARBOR_PULL_PASSWORD" | base64 | tr -d '\n')"
DOCKER_CONFIG_JSON="$(printf '{"auths":{"%s":{"auth":"%s"}}}' "$HARBOR_REGISTRY" "$HARBOR_AUTH" | base64 | tr -d '\n')"
export DOCKER_CONFIG_JSON

envsubst <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Namespace
metadata:
  name: student-01
  labels:
    gateway-access: "true"
---
apiVersion: v1
kind: Secret
metadata:
  name: harbor-pull
  namespace: student-01
type: kubernetes.io/dockerconfigjson
data:
  .dockerconfigjson: ${DOCKER_CONFIG_JSON}
EOF

unset HARBOR_AUTH DOCKER_CONFIG_JSON

cd student-01-web-deploy
kubectl apply -f argocd/project.yaml
kubectl apply -f argocd/application-dev.yaml

kubectl -n argocd get application student-01-web-dev

cd ../student-01-web-code

<!doctype html>
<html lang="ko">
  <head>
    <meta charset="utf-8" />
    <title>Commit to Cluster</title>
  </head>
  <body>
    <h1>Commit to Cluster v1</h1>
    <p>GitLab CI, Harbor, Helm, Argo CD</p>
  </body>
</html>

ARG NGINX_BASE_IMAGE
FROM ${NGINX_BASE_IMAGE}

COPY index.html /usr/share/nginx/html/index.html

stages:
  - test
  - build
  - update-config

workflow:
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

variables:
  HARBOR_REGISTRY: harbor.lab.example.com
  HARBOR_PROJECT: mlops-training
  IMAGE_NAME: student-01-web
  IMAGE_REPOSITORY: "${HARBOR_REGISTRY}/${HARBOR_PROJECT}/${IMAGE_NAME}"
  DEPLOY_REPO_HOST: gitlab.lab.example.com
  DEPLOY_REPO_PATH: platform/student-01-web-deploy
  DEPLOY_REPO_BRANCH: main
  DEPLOY_VALUES_PATH: charts/web/values-dev.yaml

test-content:
  stage: test
  image: alpine:3.21
  script:
    - test -s index.html
    - grep -q '<title>Commit to Cluster</title>' index.html
    - grep -q 'Commit to Cluster' index.html

build-and-push:
  stage: build
  image:
    name: "${BUILDKIT_IMAGE}"
    entrypoint: [""]
  needs:
    - test-content
  variables:
    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
  before_script:
    - mkdir -p "$HOME/.docker"
    - HARBOR_AUTH="$(printf '%s:%s' "$HARBOR_ROBOT_USER" "$HARBOR_ROBOT_PASSWORD" | base64 | tr -d '\n')"
    - printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$HARBOR_REGISTRY" "$HARBOR_AUTH" > "$HOME/.docker/config.json"
  script:
    - |
      buildctl-daemonless.sh build \
        --frontend dockerfile.v0 \
        --local context=. \
        --local dockerfile=. \
        --opt "build-arg:NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \
        --output "type=image,name=${IMAGE_REPOSITORY}:${CI_COMMIT_SHA},push=true"
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

update-deploy-repo:
  stage: update-config
  image: alpine:3.21
  needs:
    - build-and-push
  variables:
    GIT_STRATEGY: none
  before_script:
    - apk add --no-cache git yq
    - git config --global user.name "GitLab CI"
    - git config --global user.email "gitlab-ci@noreply.lab.example.com"
    - |
      cat > /tmp/git-askpass <<'EOF'
      #!/bin/sh
      case "$1" in
        *Username*) printf '%s\n' "$DEPLOY_REPO_USERNAME" ;;
        *)          printf '%s\n' "$DEPLOY_REPO_TOKEN" ;;
      esac
      EOF
      chmod 700 /tmp/git-askpass
      export GIT_ASKPASS=/tmp/git-askpass
      export GIT_TERMINAL_PROMPT=0
  script:
    - git clone --branch "$DEPLOY_REPO_BRANCH" "https://${DEPLOY_REPO_HOST}/${DEPLOY_REPO_PATH}.git" deploy-repo
    - cd deploy-repo
    - export IMAGE_TAG="$CI_COMMIT_SHA"
    - yq -i '.image.repository = strenv(IMAGE_REPOSITORY) | .image.tag = strenv(IMAGE_TAG)' "$DEPLOY_VALUES_PATH"
    - git add "$DEPLOY_VALUES_PATH"
    - |
      if git diff --cached --quiet; then
        echo "Desired state already points to ${IMAGE_TAG}"
      else
        git commit -m "deploy(dev): ${IMAGE_NAME} ${CI_COMMIT_SHORT_SHA}"
        git push origin "HEAD:${DEPLOY_REPO_BRANCH}"
      fi
  after_script:
    - rm -f /tmp/git-askpass
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

git add index.html Dockerfile .gitlab-ci.yml
git commit -m "feat: add first web page and GitOps pipeline"
git branch -M main
git push -u origin main

test-content → build-and-push → update-deploy-repo

git rev-parse HEAD

Projects → mlops-training → Repositories → student-01-web

cd ../student-01-web-deploy
git pull --ff-only
grep -A2 '^image:' charts/web/values-dev.yaml
git log -1 --oneline

image:
  repository: harbor.lab.example.com/mlops-training/student-01-web
  tag: "<APP_COMMIT>"

kubectl -n argocd get application student-01-web-dev \
  -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status,REVISION:.status.sync.revision

NAME                 SYNC     HEALTHY   REVISION
student-01-web-dev   Synced   Healthy   <deploy repository commit SHA>

kubectl -n student-01 rollout status deployment/web --timeout=180s
kubectl -n student-01 get pod,service,httproute
kubectl -n student-01 get deployment web \
  -o jsonpath='{.spec.template.spec.containers[0].image}{"\n"}'

harbor.lab.example.com/mlops-training/student-01-web:<APP_COMMIT>

kubectl -n student-01 get httproute web -o jsonpath='{range .status.parents[*].conditions[*]}{.type}={.status} {.reason}{"\n"}{end}'

kubectl get secret -n nginx-gateway lab-wildcard-tls \
  -o jsonpath='{.data.tls\.crt}' | base64 -d > /tmp/lab-wildcard.crt
curl --fail --silent --show-error \
  --cacert /tmp/lab-wildcard.crt \
  https://web-dev.lab.example.com
rm -f /tmp/lab-wildcard.crt

Commit to Cluster v1

cd ../student-01-web-code
sed -i 's/Commit to Cluster v1/Commit to Cluster v2/' index.html
git add index.html
git commit -m "feat: update page to v2"
git push

before_script:
  - REG_HOST="${HARBOR_REGISTRY%%/*}"
  - mkdir -p "$HOME/.config/buildkit/certs/$REG_HOST"
  - cp "$HARBOR_CA_CERT" "$HOME/.config/buildkit/certs/$REG_HOST/ca.pem"
  - |
      printf '[registry."%s"]\n  ca = ["%s"]\n' \
        "$REG_HOST" "$HOME/.config/buildkit/certs/$REG_HOST/ca.pem" \
        > "$HOME/.config/buildkit/buildkitd.toml"

kubectl -n argocd get secret student-01-web-deploy-repo \
  -o jsonpath='{.metadata.labels.argocd\.argoproj\.io/secret-type}{"\n"}'

kubectl -n student-01 describe pod -l app.kubernetes.io/instance=web
kubectl -n student-01 get secret harbor-pull
kubectl -n student-01 get deployment web \
  -o jsonpath='{.spec.template.spec.imagePullSecrets[*].name}{"\n"}'

kubectl -n student-01 get endpointslice -l kubernetes.io/service-name=web
kubectl -n student-01 describe httproute web
kubectl -n nginx-gateway describe gateway platform-gateway