Rekayasa GitOps Multi-Cluster Skala Enterprise: Konvergensi ArgoCD vs Flux v2, Zero-Trust Supply-Chain via Sigstore Cosign, dan Modular Terragrunt IaC

========================================================================================
                      ARGOCD CONTROLLER RECONCILIATION ENGINE
========================================================================================
[ Git Repo (Source of Truth) ] ── (Webhook / 3m Resync) ──► [ argocd-repo-server ]
                                                                   │
                                                                   ▼ (Kustomize / Helm Build)
                                                          [ Manifest Memory Stream ]
                                                                   │
┌───────────────────────── argocd-application-controller ─────────▼─────────────────────┐
│                                                                                       │
│  [ SharedIndexInformer (Kube-API Cache) ] ◄── (WATCH) ── [ Target K8s Clusters ]      │
│                     │                                                                 │
│                     ▼                                                                 │
│       [ Three-Way Diffing Engine ] ◄─── (Target Manifests vs Live State vs Last-Applied)
│                     │                                                                 │
│                     ├── Identical? ───────► Status: Synced (Emit Prometheus Metrics)  │
│                     └── Out of Sync?                                                  │
│                              │                                                        │
│                              ▼                                                        │
│                 [ Sync Task Execution Engine ]                                        │
│                 - Phase PreSync (Hooks & Migrations)                                  │
│                 - Apply Sync Waves (Wave 0 -> Wave 1 -> Wave n)                       │
│                 - Dynamic Client Server-Side Apply (SSA)                              │
│                 - Phase PostSync (Health Check Validation)                            │
└───────────────────────────────────────────────────────────────────────────────────────┘

[ Developer Git Commit ] 
       │
       ▼
[ Hardened CI Runner (Rootless Buildah / Kaniko) ]
       │
       ├──► 1. Build OCI Container (Deterministic Layer Hash sha256:7a9f...)
       ├──► 2. Generate SBOM (Software Bill of Materials) via Syft
       ├──► 3. Vulnerability Scan (Grype) ──► Fail on Critical CVE
       │
       ▼
[ OCI Registry Push (Subnet RFC 5737: 198.51.100.50:5000) ]
       │
       ├──► 4. Cosign Sign Digest (Keyless Sigstore OIDC / Private Key)
       ├──► 5. Cosign Attest SBOM (In-toto Predicate Format)
       └──► 6. Rekor Transparency Log Append
                               │
                               ▼
┌────────────────────── Kubernetes Cluster Edge ──────────────────────────────────────┐
│                                                                                     │
│  [ GitOps Engine Sync ] ──► [ Kube-Apiserver (VIP: 198.51.100.10:6443) ]            │
│                                       │                                             │
│                                       ▼ (AdmissionReview HTTPS Webhook)             │
│                        [ Kyverno Admission Controller ]                             │
│                                       │                                             │
│              ┌────────────────────────┴────────────────────────┐                    │
│              │                                                 │                    │
│              ▼                                                 ▼                    │
│   [ Verify Cosign Signature ]                       [ Verify In-Toto SBOM ]         │
│   - Check Rekor Log & Fulcio Cert                  - Validate Vulnerability Report  │
│   - Match Public Key PEM payload                   - Assert CycloneDX Schema 1.4    │
│              │                                                 │                    │
│              └────────────────────────┬────────────────────────┘                    │
│                                       │                                             │
│                  ┌────────────────────┴────────────────────┐                        │
│                  ▼                                         ▼                        │
│          [ Signature Match ]                       [ Invalid / Tampered ]           │
│          Allow Pod Creation                        Reject Pod (403 Forbidden)       │
└─────────────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────────────┐
│                    HASHICORP VAULT CLUSTER (198.51.100.20:8200)                     │
│                    Path: secret/data/production/financial/payment                   │
│                    Payload: { "DB_PASS": "Encrypted#Synth99" }                      │
└──────────────────────────────────────────┬──────────────────────────────────────────┘
                                           │
                                           │ TLS 1.3 / Projected ServiceAccount JWT
                                           ▼
┌─────────────────────────────────── KUBERNETES WORKLOAD CLUSTER ─────────────────────┐
│                                                                                     │
│   [ Git Repository ]                                                                │
│          │ (ArgoCD Sync)                                                            │
│          ▼                                                                          │
│   [ ExternalSecret CRD ]                                                            │
│     - Spec: storeRef: enterprise-vault-backend                                      │
│             remoteRef: key=production/financial/payment, property=DB_PASS           │
│          │                                                                          │
│          ▼                                                                          │
│   [ External Secrets Operator (ESO) Controller ]                                    │
│          │                                                                          │
│          ├── 1. Request Vault Token via K8s Auth (`/v1/auth/kubernetes/login`)      │
│          ├── 2. Fetch Decrypted Secret Payload over Mutual TLS                      │
│          ├── 3. Render Native K8s Secret (`kind: Secret`)                           │
│          └── 4. Rotate Secret secara periodik (interval: 1h)                        │
│                     │                                                               │
│                     ▼                                                               │
│   [ Native Kubernetes Secret: "payment-db-secret" ]                                 │
│          ▲                                                                          │
│          └── (Mounted as volume / injected envFrom) ── [ Microservice Pods ]        │
└─────────────────────────────────────────────────────────────────────────────────────┘

# /infrastructure/gitops/argocd/applicationset-matrix-enterprise.yaml
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: fleet-microservices-matrix
  namespace: argocd
  labels:
    tier: production
    managed-by: gitops-control-plane
spec:
  goTemplate: true
  goTemplateOptions: ["missingkey=error"]
  strategy:
    type: RollingSync
    rollingSync:
      steps:
        - matchExpressions:
            - key: environment
              operator: In
              values:
                - staging
          maxUpdate: 100%
        - matchExpressions:
            - key: environment
              operator: In
              values:
                - production
          maxUpdate: 25% # Canary deployment antar cluster bertahap
  generators:
    - matrix:
        generators:
          # Generator 1: Memetakan struktur direktori microservices di Git
          - git:
              repoURL: https://git.example.net/enterprise-core/cloud-platform-manifests.git
              revision: HEAD
              directories:
                - path: apps/services/*
          # Generator 2: Memilih target cluster berdasarkan metadata labels
          - clusters:
              selector:
                matchLabels:
                  fleet.argoproj.io/tier: tier-1-production
              values:
                region: '{{ index .metadata.labels "topology.kubernetes.io/region" }}'
                environment: '{{ index .metadata.labels "environment" }}'
  template:
    metadata:
      name: '{{ .path.basename }}-{{ .name }}'
      namespace: argocd
      labels:
        app.kubernetes.io/name: '{{ .path.basename }}'
        env: '{{ .values.environment }}'
        region: '{{ .values.region }}'
    spec:
      project: default
      source:
        repoURL: https://git.example.net/enterprise-core/cloud-platform-manifests.git
        targetRevision: HEAD
        path: '{{ .path.path }}/overlays/{{ .values.environment }}'
        kustomize:
          images:
            - 'release-image=198.51.100.50:5000/microservices/{{ .path.basename }}:v2.4.1'
      destination:
        server: '{{ .server }}'
        namespace: '{{ .path.basename }}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
          allowEmpty: false
        syncOptions:
          - CreateNamespace=true
          - PrunePropagationPolicy=foreground
          - PruneLast=true
          - ApplyOutOfSyncOnly=true
          - ServerSideApply=true
        retry:
          limit: 5
          backoff:
            duration: 10s
            factor: 2
            maxDuration: 3m
      ignoreDifferences:
        - group: apps
          kind: Deployment
          jsonPointers:
            - /spec/replicas # Membiarkan HorizontalPodAutoscaler mengontrol replikasi
        - group: ""
          kind: Service
          jsonPointers:
            - /spec/clusterIP
            - /spec/ports/0/nodePort

# /infrastructure/gitops/policies/kyverno-verify-supply-chain.yaml
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: enforce-supply-chain-signature-sbom
  annotations:
    policies.kyverno.io/title: "Verify Container Signature & SBOM Attestation"
    policies.kyverno.io/category: "Zero-Trust Supply Chain"
    policies.kyverno.io/severity: "critical"
spec:
  validationFailureAction: Enforce
  background: false
  failurePolicy: Fail
  webhookTimeoutSeconds: 30
  rules:
    - name: verify-cosign-image-signature
      match:
        any:
          - resources:
              kinds:
                - Pod
              namespaces:
                - production-*
                - financial-*
      exclude:
        any:
          - resources:
              namespaces:
                - kube-system
                - kyverno
                - monitoring
      verifyImages:
        - imageReferences:
            - "198.51.100.50:5000/microservices/*"
          attestors:
            - entries:
                - keys:
                    publicKeys: |
                      -----BEGIN PUBLIC KEY-----
                      MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE24F7sX3Bv6jCqR4E/rU2sK8QpQZf
                      G3s9e4R6sYw/5jZqH9kE3tP4Q7uL1nB9w0yV3xR8oF7kL5sN4aB9cD1eFg==
                      -----END PUBLIC KEY-----
          attestations:
            - predicateType: https://cyclonedx.org/schema/1.4
              conditions:
                - all:
                    - key: "{{ cyclonedx.metadata.component.type }}"
                      operator: Equals
                      value: "application"
    - name: block-root-and-privileged-containers
      match:
        any:
          - resources:
              kinds:
                - Pod
              namespaces:
                - production-*
      validate:
        message: "Menjalankan container sebagai root atau privileged dilarang pada tier produksi!"
        pattern:
          spec:
            =(securityContext):
              =(runAsNonRoot): true
            containers:
              - securityContext:
                  allowPrivilegeEscalation: false
                  readOnlyRootFilesystem: true
                  capabilities:
                    drop:
                      - ALL

# /infrastructure/gitops/secrets/eso-vault-store.yaml
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: enterprise-vault-backend
spec:
  provider:
    vault:
      server: "https://vault.internal.example.net:8200"
      path: "secret"
      version: "v2"
      auth:
        kubernetes:
          mountPath: "kubernetes"
          role: "k8s-platform-workload-role"
          serviceAccountRef:
            name: eso-vault-auth-sa
            namespace: external-secrets
      caProvider:
        type: ConfigMap
        name: internal-root-ca
        key: ca.crt
        namespace: external-secrets
---
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: payment-gateway-credentials
  namespace: payment-system
spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: enterprise-vault-backend
  target:
    name: payment-live-secret
    creationPolicy: Owner
    template:
      engineVersion: v2
      metadata:
        labels:
          security.enterprise.net/classification: pci-dss-regulated
      data:
        DATABASE_URL: "postgresql://{{ .DB_USER }}:{{ .DB_PASS }}@192.0.2.30:5432/transact?sslmode=verify-full"
        STRIPE_SECRET_KEY: "{{ .STRIPE_KEY }}"
  data:
    - secretKey: DB_USER
      remoteRef:
        key: production/financial/payment
        property: username
    - secretKey: DB_PASS
      remoteRef:
        key: production/financial/payment
        property: password
    - secretKey: STRIPE_KEY
      remoteRef:
        key: production/financial/payment
        property: stripe_live_token

# /infrastructure/live/terragrunt.hcl (Root Terragrunt Configuration)
locals {
  path_segments = split("/", path_relative_to_include())
  environment   = local.path_segments[0] # e.g. "prod", "staging"
  region        = local.path_segments[1] # e.g. "ap-southeast-1"
  component     = local.path_segments[2] # e.g. "vpc", "eks", "gitops"

  global_vars = yamldecode(file(find_in_parent_folders("global_vars.yaml")))
  env_vars    = yamldecode(file("${get_parent_terragrunt_dir()}/environments/${local.environment}.yaml"))
}

# Dynamic Provider Injection ke seluruh child modules
generate "provider" {
  path      = "provider_override.tf"
  if_exists = "overwrite_terragrunt"
  contents  = <<EOF
terraform {
  required_version = ">= 1.8.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.50"
    }
  }
}

provider "aws" {
  region = "${local.region}"
  default_tags {
    tags = {
      Environment = "${local.environment}"
      ManagedBy   = "Terragrunt"
      CostCenter  = "${local.env_vars.cost_center}"
      Repository  = "https://git.example.net/enterprise-core/infrastructure-live"
    }
  }
}
EOF
}

# Remote State Backend Configuration (S3 + DynamoDB State Lock)
remote_state {
  backend = "s3"
  generate = {
    path      = "backend.tf"
    if_exists = "overwrite_terragrunt"
  }
  config = {
    bucket         = "enterprise-tf-state-${local.global_vars.account_id}-${local.region}"
    key            = "${path_relative_to_include()}/terraform.tfstate"
    region         = local.region
    encrypt        = true
    dynamodb_table = "enterprise-tf-state-locks"
    s3_bucket_tags = {
      Owner       = "DevOps Core Team"
      PurgePolicy = "Never"
    }
  }
}

inputs = merge(
  local.global_vars,
  local.env_vars,
  {
    environment = local.environment
    region      = local.region
  }
)

# /infrastructure/live/prod/ap-southeast-1/eks/terragrunt.hcl
include "root" {
  path = find_in_parent_folders()
}

terraform {
  source = "git::git@git.example.net:terraform-modules/eks.git?ref=v20.8.5"
}

dependency "vpc" {
  config_path = "../vpc"
  mock_outputs = {
    vpc_id          = "vpc-00000000000000000"
    private_subnets = ["subnet-00000001", "subnet-00000002", "subnet-00000003"]
  }
  mock_outputs_allowed_terraform_commands = ["validate", "plan"]
}

inputs = {
  cluster_name    = "enterprise-k8s-prod-sg"
  cluster_version = "1.30"

  vpc_id     = dependency.vpc.outputs.vpc_id
  subnet_ids = dependency.vpc.outputs.private_subnets

  cluster_endpoint_public_access  = false
  cluster_endpoint_private_access = true

  enable_cluster_creator_admin_permissions = false
  authentication_mode                      = "API_AND_CONFIG_MAP"

  eks_managed_node_groups = {
    core_system = {
      min_size       = 3
      max_size       = 10
      desired_size   = 3
      instance_types = ["m6i.2xlarge"]
      capacity_type  = "ON_DEMAND"
      labels = {
        tier = "platform-core"
      }
    }
  }
}

#!/usr/bin/env bash
# ==============================================================================
# verify-supply-chain.sh — Zero-Trust Supply-Chain Attestation Validator
# Author: La Nina
# ==============================================================================
set -euo pipefail

IMAGE_REF="${1:-198.51.100.50:5000/microservices/payment-engine:v2.4.1}"
COSIGN_KEY_FILE="${2:-/etc/cosign/cosign.pub}"

echo "======================================================================"
echo "    ZERO-TRUST SUPPLY-CHAIN ATTESTATION & SECURITY AUDITOR"
echo "    Target OCI Artifact: ${IMAGE_REF}"
echo "======================================================================"

TMP_DIR=$(mktemp -d)
trap 'rm -rf "${TMP_DIR}"' EXIT

# 1. Verifikasi Cosign Cryptographic Signature
echo "[STEP 1] Memvalidasi signature kriptografis image..."
if [ -f "${COSIGN_KEY_FILE}" ]; then
    if cosign verify --key "${COSIGN_KEY_FILE}" "${IMAGE_REF}" > "${TMP_DIR}/cosign_verify.json" 2>&1; then
        echo "  [OK] Signature valid terotentikasi oleh: ${COSIGN_KEY_FILE}"
    else
        echo "  [FATAL] SIGNATURE INVALID! Citra kontainer dimanipulasi atau tidak sah!"
        exit 1
    fi
fi

# 2. Verifikasi In-toto SBOM Attestation
echo "[STEP 2] Memeriksa attestation format CycloneDX..."
if [ -f "${COSIGN_KEY_FILE}" ]; then
    if cosign verify-attestation --key "${COSIGN_KEY_FILE}" --type cyclonedx "${IMAGE_REF}" > "${TMP_DIR}/attestation.json" 2>&1; then
        echo "  [OK] Attestation CycloneDX SBOM terverifikasi sah."
    else
        echo "  [WARN] Attestation SBOM tidak ditemukan atau tidak match."
    fi
fi

echo "======================================================================"
echo "[SUCCESS] Citra OCI memenuhi seluruh syarat keamanan Zero-Trust."
echo "======================================================================"

#!/usr/bin/env bash
# ==============================================================================
# benchmark-gitops-perf.sh — High-Concurrency Admission & Convergence Benchmark
# Author: La Nina
# ==============================================================================
set -euo pipefail

NAMESPACE="benchmark-gitops-$(date +%s)"
ITERATIONS="${1:-20}"
CONCURRENCY="${2:-5}"

echo "======================================================================"
echo "    KUBERNETES GITOPS RECONCILIATION & ADMISSION BENCHMARK"
echo "    Namespace: ${NAMESPACE} | Iterasi: ${ITERATIONS} | Concurrency: ${CONCURRENCY}"
echo "======================================================================"

kubectl create namespace "${NAMESPACE}"
trap 'kubectl delete namespace "${NAMESPACE}" --wait=false >/dev/null 2>&1 || true' EXIT

START_TIME=$(date +%s%3N)

for i in $(seq 1 "${ITERATIONS}"); do
    cat <<EOF | kubectl apply -f - >/dev/null &
apiVersion: apps/v1
kind: Deployment
metadata:
  name: bench-app-${i}
  namespace: ${NAMESPACE}
spec:
  replicas: 1
  selector:
    matchLabels:
      app: bench-app-${i}
  template:
    metadata:
      labels:
        app: bench-app-${i}
    spec:
      containers:
        - name: pause
          image: 198.51.100.50:5000/microservices/pause:3.9
          resources:
            limits: { cpu: 20m, memory: 32Mi }
            requests: { cpu: 5m, memory: 16Mi }
EOF
    if (( i % CONCURRENCY == 0 )); then wait; fi
done
wait

END_APPLY=$(date +%s%3N)
APPLY_DURATION=$(( END_APPLY - START_TIME ))

echo "[METRIC] Selesai menerapkan ${ITERATIONS} deployments dalam ${APPLY_DURATION} ms"

# Tunggu konvergensi pod
while true; do
    READY_COUNT=$(kubectl get pods -n "${NAMESPACE}" --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l || echo 0)
    if [ "${READY_COUNT}" -ge "${ITERATIONS}" ]; then break; fi
    sleep 1
done

END_WATCH=$(date +%s%3N)
TOTAL_LATENCY=$(( END_WATCH - START_TIME ))

echo "======================================================================"
echo " Total Manifests     : ${ITERATIONS}"
echo " Client Apply Latency: ${APPLY_DURATION} ms"
echo " Total Convergence   : ${TOTAL_LATENCY} ms"
echo " Rata-rata per-pod   : $(( TOTAL_LATENCY / ITERATIONS )) ms / pod"
echo "======================================================================"