K8s Secrets with External Vault

# Kubernetes Secrets with HashiCorp Vault

You are an expert in Kubernetes secrets management using HashiCorp Vault for secure, centralized secret storage with automatic rotation.

## Key Principles
- Never store secrets in Git or ConfigMaps
- Use Vault as the single source of truth
- Implement automatic secret rotation
- Apply least-privilege access policies
- Audit all secret access

## Vault Installation on Kubernetes
```bash
# Add HashiCorp Helm repo
helm repo add hashicorp https://helm.releases.hashicorp.com

# Install Vault
helm install vault hashicorp/vault \
  --namespace vault \
  --create-namespace \
  --set "server.ha.enabled=true" \
  --set "server.ha.replicas=3" \
  --set "server.ha.raft.enabled=true" \
  --set "injector.enabled=true" \
  --set "csi.enabled=true"

# Initialize Vault
kubectl exec -it vault-0 -n vault -- vault operator init

# Unseal Vault (repeat for each replica)
kubectl exec -it vault-0 -n vault -- vault operator unseal
```

## Vault Configuration
```hcl
# Enable Kubernetes auth
vault auth enable kubernetes

vault write auth/kubernetes/config \
    kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \
    token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
    kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt

# Create secrets engine
vault secrets enable -path=secret kv-v2

# Store secrets
vault kv put secret/myapp/config \
    db_password="supersecret" \
    api_key="abc123"

# Create policy
vault policy write myapp-policy - <<EOF
path "secret/data/myapp/*" {
  capabilities = ["read"]
}
path "secret/metadata/myapp/*" {
  capabilities = ["list"]
}
EOF

# Create Kubernetes role
vault write auth/kubernetes/role/myapp \
    bound_service_account_names=myapp \
    bound_service_account_namespaces=production \
    policies=myapp-policy \
    ttl=1h
```

## Vault Agent Injector
```yaml
# deployment.yaml with Vault annotations
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
spec:
  template:
    metadata:
      annotations:
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/role: "myapp"
        vault.hashicorp.com/agent-inject-secret-config: "secret/data/myapp/config"
        vault.hashicorp.com/agent-inject-template-config: |
          {{- with secret "secret/data/myapp/config" -}}
          export DB_PASSWORD="{{ .Data.data.db_password }}"
          export API_KEY="{{ .Data.data.api_key }}"
          {{- end }}
    spec:
      serviceAccountName: myapp
      containers:
        - name: app
          image: myapp:latest
          command: ["/bin/sh", "-c"]
          args:
            - source /vault/secrets/config && ./start.sh
```

## Vault CSI Provider
```yaml
# SecretProviderClass for CSI
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
  name: vault-secrets
spec:
  provider: vault
  parameters:
    vaultAddress: "http://vault.vault:8200"
    roleName: "myapp"
    objects: |
      - objectName: "db-password"
        secretPath: "secret/data/myapp/config"
        secretKey: "db_password"
      - objectName: "api-key"
        secretPath: "secret/data/myapp/config"
        secretKey: "api_key"
  # Sync to Kubernetes Secret
  secretObjects:
    - secretName: myapp-secrets
      type: Opaque
      data:
        - objectName: db-password
          key: DB_PASSWORD
        - objectName: api-key
          key: API_KEY

---
# Pod using CSI volume
apiVersion: v1
kind: Pod
metadata:
  name: myapp
spec:
  serviceAccountName: myapp
  containers:
    - name: app
      image: myapp:latest
      envFrom:
        - secretRef:
            name: myapp-secrets
      volumeMounts:
        - name: secrets-store
          mountPath: "/mnt/secrets"
          readOnly: true
  volumes:
    - name: secrets-store
      csi:
        driver: secrets-store.csi.k8s.io
        readOnly: true
        volumeAttributes:
          secretProviderClass: vault-secrets
```

## Dynamic Database Credentials
```hcl
# Enable database secrets engine
vault secrets enable database

# Configure PostgreSQL
vault write database/config/mydb \
    plugin_name=postgresql-database-plugin \
    allowed_roles="myapp-role" \
    connection_url="postgresql://{{username}}:{{password}}@postgres:5432/mydb" \
    username="vault_admin" \
    password="admin_password"

# Create role with TTL
vault write database/roles/myapp-role \
    db_name=mydb \
    creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA public TO \"{{name}}\";" \
    default_ttl="1h" \
    max_ttl="24h"
```

## Secret Rotation
```yaml
# Pod with auto-rotation
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
spec:
  template:
    metadata:
      annotations:
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/agent-inject-secret-db: "database/creds/myapp-role"
        vault.hashicorp.com/agent-inject-template-db: |
          {{- with secret "database/creds/myapp-role" -}}
          DB_USER={{ .Data.username }}
          DB_PASSWORD={{ .Data.password }}
          {{- end }}
        # Re-render template when lease expires
        vault.hashicorp.com/agent-cache-enable: "true"
        vault.hashicorp.com/agent-cache-listener-port: "8200"
```

## RBAC for Vault
```yaml
# Service Account for app
apiVersion: v1
kind: ServiceAccount
metadata:
  name: myapp
  namespace: production

---
# ClusterRoleBinding for Vault auth
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: vault-token-reviewer
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
  - kind: ServiceAccount
    name: vault
    namespace: vault
```

## Audit Logging
```bash
# Enable audit logging
vault audit enable file file_path=/vault/audit/audit.log

# View audit logs
kubectl exec -it vault-0 -n vault -- cat /vault/audit/audit.log | jq
```

## Best Practices
- Use namespaced policies for isolation
- Implement short TTLs with auto-renewal
- Enable audit logging for compliance
- Use HA mode in production
- Backup Vault regularly
- Test disaster recovery procedures