# Your operations horror story
# CEO: "We need to deploy the critical bug fix NOW, customers are churning"

# Attempt 1: Manual deployment at 2 AM
$ ssh production-server
production$ git pull origin main
# Merge conflict in critical configuration file
# No automated tests, deploying blind

$ sudo systemctl restart myapp
Job for myapp.service failed because the control process exited with error code.
# Service won't start, no clear error logs

$ sudo journalctl -u myapp
# 50,000 lines of generic logs, needle in haystack
# No structured logging, no error aggregation

# Attempt 2: Emergency rollback
$ git log --oneline
# 47 commits since last known good state
# No release tags, no deployment tracking
# Which commit was actually deployed last?

$ git checkout HEAD~5
$ sudo systemctl restart myapp
# Service starts but database migrations are incompatible
# Data corruption in production database

# Attempt 3: Infrastructure disaster
# Primary database server dies during peak traffic
$ aws rds describe-db-instances --db-instance-identifier prod-db
{
    "DBInstanceStatus": "failed"
}
# No automated failover, no backups tested in 6 months
# Customer data potentially lost forever

# Attempt 4: Monitoring blindness
# Load balancer shows 500 errors for 30 minutes
# First alert comes from angry customer on Twitter
# No proactive monitoring, no alerting
# "Why didn't anyone tell us the site was down?"

# The cascading operations disasters:
# - No CI/CD pipeline, deployments via "git pull and pray"
# - No automated testing, bugs discovered by customers
# - No monitoring, outages discovered via social media
# - No logging strategy, debugging takes hours
# - No disaster recovery, single points of failure everywhere
# - No backup strategy, data loss risk on every failure
# - No change tracking, impossible to identify what broke

# Result: 8-hour outage during Black Friday
# $2M in lost revenue, 30% customer churn
# Engineering team working 16-hour days for weeks
# Company reputation destroyed, acquisition talks canceled
# The brutal truth: Great infrastructure can't save amateur operations

// What happens when operations practices are amateur:
const operationsFailureImpact = {
  deploymentDisasters: {
    problem: "Critical bug fix deployment breaks entire application",
    cause: "No CI/CD pipeline, no automated testing, manual deployments",
    impact: "6-hour outage during peak business hours, revenue loss",
    cost: "$500K in lost sales, 20% customer churn",
  },

  monitoringBlindness: {
    problem: "Performance degradation goes unnoticed for hours",
    cause: "No proactive monitoring, no alerting, reactive debugging",
    impact: "Customers experience slow site, competitors gain market share",
    consequences: "Brand reputation damaged, customer satisfaction plummets",
  },

  disasterRecoveryFailure: {
    problem: "Database corruption during peak season with no recovery plan",
    cause:
      "Untested backups, no disaster recovery procedures, single points of failure",
    impact: "Complete data loss, business operations halt for days",
    reality: "Company closes permanently, all customer data lost forever",
  },

  operationalChaos: {
    problem:
      "Teams spend 80% of time firefighting instead of building features",
    cause: "No automation, no monitoring, no proper deployment processes",
    impact: "Product development stagnates, competitors outpace innovation",
    prevention:
      "Professional operations enable teams to focus on value creation",
  },

  // Perfect infrastructure is worthless when operations
  // lack automation, monitoring, disaster recovery, and reliability practices
};

// Manual deployment vs Professional CI/CD pipeline comparison
const deploymentEvolution = {
  manualDeployment: {
    process: "Developer manually deploys from their laptop",
    testing: "Maybe run a few tests locally if remembered",
    consistency:
      "Every deployment is different, configuration drift guaranteed",
    rollback: "Panic-driven git revert, usually makes things worse",
    visibility: "No one knows what's deployed or when",
    scalability: "One person becomes deployment bottleneck",
    quality: "Bugs discovered by customers in production",
    reliability: "50% chance of deployment causing outage",
  },

  professionalCICD: {
    process: "Automated pipeline triggered by git events",
    testing: "Comprehensive automated test suite runs every time",
    consistency: "Identical deployment process every single time",
    rollback: "One-click automated rollback to any previous version",
    visibility: "Full deployment history and current state tracking",
    scalability: "Multiple teams deploying dozens of times per day",
    quality: "Issues caught in CI/CD before reaching production",
    reliability: "99.9% successful deployments, predictable outcomes",
  },

  theTransformationImpact: [
    "Teams deploy 10x more frequently with higher confidence",
    "Bug detection shifts left, issues caught in minutes not hours",
    "Rollbacks happen in seconds, not emergency all-hands meetings",
    "Developers focus on features, not deployment firefighting",
    "Quality improves dramatically with automated testing",
    "Infrastructure changes become routine, not risky events",
  ],
};

# .github/workflows/production-deployment.yml - Professional CI/CD pipeline
name: Production Deployment Pipeline

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:
    inputs:
      environment:
        description: "Deployment environment"
        required: true
        default: "staging"
        type: choice
        options:
          - staging
          - production

env:
  NODE_VERSION: "18"
  DOCKER_REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  # ========================================
  # Code Quality and Security Analysis
  # ========================================
  code-quality:
    name: Code Quality Analysis
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Needed for SonarCloud

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "npm"

      - name: Install dependencies
        run: |
          npm ci --prefer-offline --no-audit

      - name: Run ESLint with annotations
        run: |
          npx eslint . --format=@microsoft/eslint-formatter-sarif --output-file eslint-results.sarif
          npx eslint . --format=stylish
        continue-on-error: true

      - name: Upload ESLint results to GitHub
        uses: github/codeql-action/upload-sarif@v2
        if: always()
        with:
          sarif_file: eslint-results.sarif

      - name: Run Prettier check
        run: npx prettier --check .

      - name: TypeScript type checking
        run: npx tsc --noEmit

      - name: Security audit
        run: |
          npm audit --audit-level=high
          npx audit-ci --config .auditrc.json

      - name: License compliance check
        run: |
          npx license-checker --onlyAllow "MIT;Apache-2.0;BSD-2-Clause;BSD-3-Clause;ISC" --excludePrivatePackages

      - name: SonarCloud Scan
        uses: SonarSource/sonarcloud-github-action@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

  # ========================================
  # Comprehensive Testing Suite
  # ========================================
  unit-tests:
    name: Unit Tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "npm"

      - name: Install dependencies
        run: npm ci --prefer-offline --no-audit

      - name: Run unit tests with coverage
        run: |
          npm run test:unit -- --coverage --watchAll=false --ci

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage/lcov.info
          flags: unittests
          name: codecov-umbrella

      - name: Comment coverage on PR
        if: github.event_name == 'pull_request'
        uses: codecov/codecov-action@v3

  integration-tests:
    name: Integration Tests
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_DB: testdb
          POSTGRES_USER: testuser
          POSTGRES_PASSWORD: testpass
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432

      redis:
        image: redis:7-alpine
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 6379:6379

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "npm"

      - name: Install dependencies
        run: npm ci --prefer-offline --no-audit

      - name: Wait for services to be ready
        run: |
          timeout 60 bash -c 'until nc -z localhost 5432; do sleep 1; done'
          timeout 60 bash -c 'until nc -z localhost 6379; do sleep 1; done'

      - name: Run database migrations
        run: npm run db:migrate
        env:
          DATABASE_URL: postgresql://testuser:testpass@localhost:5432/testdb
          REDIS_URL: redis://localhost:6379

      - name: Run integration tests
        run: npm run test:integration
        env:
          NODE_ENV: test
          DATABASE_URL: postgresql://testuser:testpass@localhost:5432/testdb
          REDIS_URL: redis://localhost:6379

  e2e-tests:
    name: End-to-End Tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "npm"

      - name: Install dependencies
        run: npm ci --prefer-offline --no-audit

      - name: Build application
        run: npm run build

      - name: Start application for E2E tests
        run: |
          npm start &
          timeout 60 bash -c 'until curl -f http://localhost:3000/health; do sleep 2; done'
        env:
          NODE_ENV: test
          PORT: 3000

      - name: Run Playwright E2E tests
        run: npx playwright test
        env:
          BASE_URL: http://localhost:3000

      - name: Upload Playwright report
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: playwright-report
          path: playwright-report/

  # ========================================
  # Container Security and Building
  # ========================================
  container-security:
    name: Container Security Scan
    runs-on: ubuntu-latest
    needs: [code-quality]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Build Docker image for scanning
        run: |
          docker build -t security-scan:latest .

      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: "security-scan:latest"
          format: "sarif"
          output: "trivy-results.sarif"

      - name: Upload Trivy scan results to GitHub Security
        uses: github/codeql-action/upload-sarif@v2
        if: always()
        with:
          sarif_file: "trivy-results.sarif"

      - name: Run Hadolint Dockerfile linting
        uses: hadolint/hadolint-action@v3.1.0
        with:
          dockerfile: Dockerfile
          format: sarif
          output-file: hadolint-results.sarif

      - name: Upload Hadolint results
        uses: github/codeql-action/upload-sarif@v2
        if: always()
        with:
          sarif_file: hadolint-results.sarif

  build-and-push:
    name: Build and Push Container
    runs-on: ubuntu-latest
    needs: [unit-tests, integration-tests, container-security]
    if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'
    outputs:
      image-digest: ${{ steps.build.outputs.digest }}
      image-tag: ${{ steps.meta.outputs.tags }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Log in to Container Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.DOCKER_REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.DOCKER_REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=ref,event=branch
            type=sha,prefix={{branch}}-
            type=raw,value=latest,enable={{is_default_branch}}

      - name: Build and push Docker image
        id: build
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            BUILDTIME=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
            VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
            REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}

  # ========================================
  # Staging Deployment and Testing
  # ========================================
  deploy-staging:
    name: Deploy to Staging
    runs-on: ubuntu-latest
    needs: [build-and-push]
    if: github.ref == 'refs/heads/main'
    environment:
      name: staging
      url: https://staging.myapp.com
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-west-2

      - name: Setup Kubernetes config
        run: |
          aws eks update-kubeconfig --name myapp-staging-cluster --region us-west-2

      - name: Deploy to staging with Helm
        run: |
          helm upgrade --install myapp-staging ./helm/myapp \
            --namespace staging \
            --create-namespace \
            --set image.repository="${{ env.DOCKER_REGISTRY }}/${{ env.IMAGE_NAME }}" \
            --set image.tag="${{ github.sha }}" \
            --set environment="staging" \
            --set ingress.host="staging.myapp.com" \
            --wait --timeout=10m

      - name: Wait for deployment to be ready
        run: |
          kubectl rollout status deployment/myapp-staging -n staging --timeout=600s

      - name: Run smoke tests against staging
        run: |
          timeout 300 bash -c 'until curl -f https://staging.myapp.com/health; do sleep 10; done'
          npm run test:smoke -- --baseURL=https://staging.myapp.com

      - name: Run load tests against staging
        run: |
          npm run test:load -- --baseURL=https://staging.myapp.com

  # ========================================
  # Production Deployment with Approval
  # ========================================
  deploy-production:
    name: Deploy to Production
    runs-on: ubuntu-latest
    needs: [deploy-staging, e2e-tests]
    if: github.ref == 'refs/heads/main' || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'production')
    environment:
      name: production
      url: https://myapp.com
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_PROD_ROLE_ARN }}
          aws-region: us-west-2

      - name: Setup Kubernetes config
        run: |
          aws eks update-kubeconfig --name myapp-production-cluster --region us-west-2

      - name: Pre-deployment health check
        run: |
          kubectl get nodes
          kubectl get pods -A | grep -E "(Crash|Error|ImagePull)" && exit 1 || true

      - name: Deploy to production with blue-green strategy
        run: |
          # Deploy to inactive environment (green if blue is active)
          ACTIVE_ENV=$(kubectl get service myapp-active -o jsonpath='{.spec.selector.version}' || echo "blue")
          TARGET_ENV=$([ "$ACTIVE_ENV" = "blue" ] && echo "green" || echo "blue")

          echo "Deploying to $TARGET_ENV environment (current active: $ACTIVE_ENV)"

          helm upgrade --install myapp-$TARGET_ENV ./helm/myapp \
            --namespace production \
            --create-namespace \
            --set image.repository="${{ env.DOCKER_REGISTRY }}/${{ env.IMAGE_NAME }}" \
            --set image.tag="${{ github.sha }}" \
            --set environment="production" \
            --set deployment.version="$TARGET_ENV" \
            --set ingress.host="myapp.com" \
            --wait --timeout=15m

      - name: Validate new deployment
        run: |
          TARGET_ENV=$(kubectl get deployment -l version!=active -o jsonpath='{.items[0].metadata.labels.version}')

          # Wait for deployment to be fully ready
          kubectl rollout status deployment/myapp-$TARGET_ENV -n production --timeout=600s

          # Run comprehensive health checks
          kubectl port-forward svc/myapp-$TARGET_ENV 8080:80 -n production &
          PF_PID=$!
          sleep 10

          # Smoke tests
          curl -f http://localhost:8080/health || exit 1
          npm run test:smoke -- --baseURL=http://localhost:8080

          kill $PF_PID

      - name: Switch traffic to new deployment
        run: |
          TARGET_ENV=$(kubectl get deployment -l version!=active -o jsonpath='{.items[0].metadata.labels.version}')

          echo "Switching traffic to $TARGET_ENV"
          kubectl patch service myapp-active -p '{"spec":{"selector":{"version":"'$TARGET_ENV'"}}}'

          # Update labels to mark new environment as active
          kubectl label deployment myapp-$TARGET_ENV version=active --overwrite

          # Mark old environment as inactive
          OLD_ENV=$([ "$TARGET_ENV" = "blue" ] && echo "green" || echo "blue")
          kubectl label deployment myapp-$OLD_ENV version=inactive --overwrite

      - name: Post-deployment monitoring
        run: |
          echo "Monitoring deployment for 5 minutes..."
          for i in {1..30}; do
            if ! curl -f https://myapp.com/health; then
              echo "Health check failed, initiating rollback"
              # Rollback logic would go here
              exit 1
            fi
            sleep 10
          done
          echo "Deployment stable, monitoring successful"

      - name: Cleanup old deployment
        run: |
          OLD_ENV=$(kubectl get deployment -l version=inactive -o jsonpath='{.items[0].metadata.labels.version}')
          if [ -n "$OLD_ENV" ]; then
            echo "Cleaning up old deployment: $OLD_ENV"
            helm uninstall myapp-$OLD_ENV --namespace production || true
          fi

  # ========================================
  # Post-deployment notifications
  # ========================================
  notify-deployment:
    name: Notify Deployment Status
    runs-on: ubuntu-latest
    needs: [deploy-production]
    if: always()
    steps:
      - name: Notify Slack on success
        if: needs.deploy-production.result == 'success'
        uses: 8398a7/action-slack@v3
        with:
          status: success
          channel: "#deployments"
          text: |
            🎉 Production deployment successful!

            **Repository:** ${{ github.repository }}
            **Branch:** ${{ github.ref_name }}
            **Commit:** ${{ github.sha }}
            **Author:** ${{ github.actor }}

            **Deployed to:** https://myapp.com
            **Dashboard:** https://grafana.myapp.com
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

      - name: Notify Slack on failure
        if: needs.deploy-production.result == 'failure'
        uses: 8398a7/action-slack@v3
        with:
          status: failure
          channel: "#alerts"
          text: |
            🚨 Production deployment failed!

            **Repository:** ${{ github.repository }}
            **Branch:** ${{ github.ref_name }}
            **Commit:** ${{ github.sha }}
            **Author:** ${{ github.actor }}

            **Action:** https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

            Please investigate immediately.
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

      - name: Create GitHub release on successful production deploy
        if: needs.deploy-production.result == 'success' && github.ref == 'refs/heads/main'
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: production-${{ github.run_number }}
          release_name: Production Release ${{ github.run_number }}
          body: |
            Automated production release

            **Commit:** ${{ github.sha }}
            **Deployed:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")

            [View deployment workflow](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
          draft: false
          prerelease: false

# .gitlab-ci.yml - Professional GitLab CI/CD pipeline
stages:
  - quality
  - test
  - security
  - build
  - deploy-staging
  - deploy-production
  - monitor

variables:
  DOCKER_REGISTRY: $CI_REGISTRY
  IMAGE_NAME: $CI_REGISTRY_IMAGE
  KUBERNETES_VERSION: "1.28"

# ========================================
# Quality and Security Analysis
# ========================================
code-quality:
  stage: quality
  image: node:18-alpine
  cache:
    paths:
      - node_modules/
  script:
    - npm ci --prefer-offline
    - npm run lint -- --format=junit --output-file=eslint-report.xml
    - npm run prettier:check
    - npx tsc --noEmit
  artifacts:
    reports:
      junit: eslint-report.xml
    paths:
      - eslint-report.xml
    expire_in: 1 week

dependency-security:
  stage: quality
  image: node:18-alpine
  script:
    - npm ci --prefer-offline
    - npm audit --audit-level=high
    - npx audit-ci --config .auditrc.json
    - npx license-checker --onlyAllow "MIT;Apache-2.0;BSD-2-Clause;BSD-3-Clause;ISC"
  allow_failure: false

sonarcloud-check:
  stage: quality
  image: sonarsource/sonar-scanner-cli:latest
  variables:
    SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"
    GIT_DEPTH: "0"
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .sonar/cache
  script:
    - sonar-scanner
  only:
    - main
    - merge_requests

# ========================================
# Comprehensive Testing
# ========================================
unit-tests:
  stage: test
  image: node:18-alpine
  services:
    - postgres:15
    - redis:7-alpine
  variables:
    POSTGRES_DB: testdb
    POSTGRES_USER: testuser
    POSTGRES_PASSWORD: testpass
    REDIS_URL: redis://redis:6379
    DATABASE_URL: postgresql://testuser:testpass@postgres:5432/testdb
  cache:
    paths:
      - node_modules/
  before_script:
    - npm ci --prefer-offline
    - npm run db:migrate
  script:
    - npm run test:unit -- --coverage --ci --watchAll=false
    - npm run test:integration
  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage/cobertura-coverage.xml
      junit: junit.xml
    paths:
      - coverage/
    expire_in: 1 week

e2e-tests:
  stage: test
  image: mcr.microsoft.com/playwright:v1.40.0-focal
  services:
    - postgres:15
    - redis:7-alpine
  variables:
    POSTGRES_DB: testdb
    POSTGRES_USER: testuser
    POSTGRES_PASSWORD: testpass
    DATABASE_URL: postgresql://testuser:testpass@postgres:5432/testdb
    REDIS_URL: redis://redis:6379
  before_script:
    - npm ci --prefer-offline
    - npm run build
    - npm run db:migrate
    - npm start &
    - sleep 30
    - curl -f http://localhost:3000/health
  script:
    - npx playwright test
  artifacts:
    when: always
    paths:
      - playwright-report/
      - test-results/
    expire_in: 1 week

# ========================================
# Security and Container Scanning
# ========================================
container-security:
  stage: security
  image: docker:20.10.16
  services:
    - docker:20.10.16-dind
  variables:
    DOCKER_TLS_CERTDIR: "/certs"
    TRIVY_CACHE_DIR: ".trivycache/"
  before_script:
    - docker build -t $IMAGE_NAME:$CI_COMMIT_SHA .
    - apk add --no-cache curl
    - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
  script:
    - trivy image --exit-code 1 --severity HIGH,CRITICAL --no-progress $IMAGE_NAME:$CI_COMMIT_SHA
    - trivy image --format template --template "@contrib/gitlab.tpl" --output gl-container-scanning-report.json $IMAGE_NAME:$CI_COMMIT_SHA
  cache:
    paths:
      - .trivycache/
  artifacts:
    reports:
      container_scanning: gl-container-scanning-report.json
    expire_in: 1 week

dockerfile-lint:
  stage: security
  image: hadolint/hadolint:latest-alpine
  script:
    - hadolint --format gitlab_codeclimate --failure-threshold warning Dockerfile > hadolint-report.json
  artifacts:
    reports:
      codequality: hadolint-report.json
    expire_in: 1 week

# ========================================
# Build and Registry
# ========================================
build-image:
  stage: build
  image: docker:20.10.16
  services:
    - docker:20.10.16-dind
  variables:
    DOCKER_TLS_CERTDIR: "/certs"
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - |
      # Multi-arch build with buildx
      docker buildx create --use
      docker buildx build \
        --platform linux/amd64,linux/arm64 \
        --tag $IMAGE_NAME:$CI_COMMIT_SHA \
        --tag $IMAGE_NAME:latest \
        --push \
        --build-arg BUILDTIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
        --build-arg VERSION=$CI_COMMIT_TAG \
        --build-arg REVISION=$CI_COMMIT_SHA \
        .
  only:
    - main
    - tags

# ========================================
# Staging Deployment
# ========================================
deploy-staging:
  stage: deploy-staging
  image:
    name: alpine/helm:3.12.0
    entrypoint: [""]
  environment:
    name: staging
    url: https://staging.myapp.com
  before_script:
    - apk add --no-cache curl bash
    - curl -LO "https://dl.k8s.io/release/v$KUBERNETES_VERSION/bin/linux/amd64/kubectl"
    - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
    - echo $KUBE_CONFIG_STAGING | base64 -d > ~/.kube/config
  script:
    - |
      helm upgrade --install myapp-staging ./helm/myapp \
        --namespace staging \
        --create-namespace \
        --set image.repository=$IMAGE_NAME \
        --set image.tag=$CI_COMMIT_SHA \
        --set environment=staging \
        --set ingress.host=staging.myapp.com \
        --wait --timeout=10m
        
      kubectl rollout status deployment/myapp-staging -n staging --timeout=600s

      # Smoke tests
      sleep 30
      curl -f https://staging.myapp.com/health
  only:
    - main

staging-smoke-tests:
  stage: deploy-staging
  image: node:18-alpine
  dependencies:
    - deploy-staging
  script:
    - npm ci --prefer-offline
    - npm run test:smoke -- --baseURL=https://staging.myapp.com
    - npm run test:load -- --baseURL=https://staging.myapp.com --duration=5m
  only:
    - main

# ========================================
# Production Deployment
# ========================================
deploy-production:
  stage: deploy-production
  image:
    name: alpine/helm:3.12.0
    entrypoint: [""]
  environment:
    name: production
    url: https://myapp.com
  before_script:
    - apk add --no-cache curl bash jq
    - curl -LO "https://dl.k8s.io/release/v$KUBERNETES_VERSION/bin/linux/amd64/kubectl"
    - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
    - echo $KUBE_CONFIG_PRODUCTION | base64 -d > ~/.kube/config
  script:
    - |
      # Blue-green deployment strategy
      ACTIVE_ENV=$(kubectl get service myapp-active -o jsonpath='{.spec.selector.version}' 2>/dev/null || echo "blue")
      TARGET_ENV=$([ "$ACTIVE_ENV" = "blue" ] && echo "green" || echo "blue")

      echo "Deploying to $TARGET_ENV (current active: $ACTIVE_ENV)"

      # Deploy to target environment
      helm upgrade --install myapp-$TARGET_ENV ./helm/myapp \
        --namespace production \
        --create-namespace \
        --set image.repository=$IMAGE_NAME \
        --set image.tag=$CI_COMMIT_SHA \
        --set environment=production \
        --set deployment.version=$TARGET_ENV \
        --set ingress.host=myapp.com \
        --wait --timeout=15m

      kubectl rollout status deployment/myapp-$TARGET_ENV -n production --timeout=600s

      # Validation tests
      kubectl port-forward svc/myapp-$TARGET_ENV 8080:80 -n production &
      PF_PID=$!
      sleep 15

      curl -f http://localhost:8080/health || exit 1
      kill $PF_PID

      # Switch traffic
      kubectl patch service myapp-active -p '{"spec":{"selector":{"version":"'$TARGET_ENV'"}}}'
      kubectl label deployment myapp-$TARGET_ENV version=active --overwrite

      # Monitor for 5 minutes
      for i in {1..30}; do
        curl -f https://myapp.com/health || exit 1
        sleep 10
      done

      # Cleanup old deployment
      OLD_ENV=$([ "$TARGET_ENV" = "blue" ] && echo "green" || echo "blue")
      helm uninstall myapp-$OLD_ENV --namespace production || true

  when: manual
  only:
    - main

# ========================================
# Monitoring and Notifications
# ========================================
post-deploy-monitoring:
  stage: monitor
  image: curlimages/curl:latest
  script:
    - |
      # Send deployment notification
      curl -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"✅ Production deployment successful!\n**Project:** $CI_PROJECT_NAME\n**Commit:** $CI_COMMIT_SHA\n**Pipeline:** $CI_PIPELINE_URL\"}" \
        $SLACK_WEBHOOK_URL

      # Trigger monitoring dashboard update
      curl -X POST -H "Authorization: Bearer $GRAFANA_API_KEY" \
        -H "Content-Type: application/json" \
        -d '{"dashboard": {"id": null, "title": "Deployment Tracking", "tags": ["deployment"], "timezone": "browser"}}' \
        $GRAFANA_URL/api/dashboards/db
  dependencies:
    - deploy-production
  when: on_success
  only:
    - main

# terraform/modules/networking/main.tf - Reusable networking module
variable "project_name" {
  description = "Name of the project"
  type        = string
}

variable "environment" {
  description = "Environment (dev, staging, production)"
  type        = string
}

variable "region" {
  description = "AWS region"
  type        = string
}

variable "availability_zones" {
  description = "List of availability zones"
  type        = list(string)
}

variable "vpc_cidr" {
  description = "CIDR block for VPC"
  type        = string
  default     = "10.0.0.0/16"
}

variable "enable_nat_gateway" {
  description = "Enable NAT Gateway for private subnets"
  type        = bool
  default     = true
}

variable "enable_vpn_gateway" {
  description = "Enable VPN Gateway"
  type        = bool
  default     = false
}

locals {
  common_tags = {
    Project     = var.project_name
    Environment = var.environment
    Module      = "networking"
    ManagedBy   = "terraform"
  }

  # Calculate subnet CIDRs automatically
  public_subnet_cidrs  = [for i in range(length(var.availability_zones)) : cidrsubnet(var.vpc_cidr, 8, i)]
  private_subnet_cidrs = [for i in range(length(var.availability_zones)) : cidrsubnet(var.vpc_cidr, 8, i + 10)]
  database_subnet_cidrs = [for i in range(length(var.availability_zones)) : cidrsubnet(var.vpc_cidr, 8, i + 20)]
}

# VPC
resource "aws_vpc" "main" {
  cidr_block           = var.vpc_cidr
  enable_dns_hostnames = true
  enable_dns_support   = true

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-vpc"
  })
}

# Internet Gateway
resource "aws_internet_gateway" "main" {
  vpc_id = aws_vpc.main.id

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-igw"
  })
}

# Public Subnets
resource "aws_subnet" "public" {
  count = length(var.availability_zones)

  vpc_id                  = aws_vpc.main.id
  cidr_block              = local.public_subnet_cidrs[count.index]
  availability_zone       = var.availability_zones[count.index]
  map_public_ip_on_launch = true

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-public-${count.index + 1}"
    Type = "public"
    "kubernetes.io/role/elb" = "1"
  })
}

# Private Subnets
resource "aws_subnet" "private" {
  count = length(var.availability_zones)

  vpc_id            = aws_vpc.main.id
  cidr_block        = local.private_subnet_cidrs[count.index]
  availability_zone = var.availability_zones[count.index]

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-private-${count.index + 1}"
    Type = "private"
    "kubernetes.io/role/internal-elb" = "1"
  })
}

# Database Subnets
resource "aws_subnet" "database" {
  count = length(var.availability_zones)

  vpc_id            = aws_vpc.main.id
  cidr_block        = local.database_subnet_cidrs[count.index]
  availability_zone = var.availability_zones[count.index]

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-database-${count.index + 1}"
    Type = "database"
  })
}

# NAT Gateways (conditional)
resource "aws_eip" "nat" {
  count = var.enable_nat_gateway ? length(var.availability_zones) : 0

  domain = "vpc"

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-nat-eip-${count.index + 1}"
  })
}

resource "aws_nat_gateway" "main" {
  count = var.enable_nat_gateway ? length(var.availability_zones) : 0

  allocation_id = aws_eip.nat[count.index].id
  subnet_id     = aws_subnet.public[count.index].id

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-nat-${count.index + 1}"
  })

  depends_on = [aws_internet_gateway.main]
}

# Route Tables
resource "aws_route_table" "public" {
  vpc_id = aws_vpc.main.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.main.id
  }

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-public-rt"
  })
}

resource "aws_route_table" "private" {
  count = length(var.availability_zones)

  vpc_id = aws_vpc.main.id

  dynamic "route" {
    for_each = var.enable_nat_gateway ? [1] : []
    content {
      cidr_block     = "0.0.0.0/0"
      nat_gateway_id = aws_nat_gateway.main[count.index].id
    }
  }

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-private-rt-${count.index + 1}"
  })
}

resource "aws_route_table" "database" {
  vpc_id = aws_vpc.main.id

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-database-rt"
  })
}

# Route Table Associations
resource "aws_route_table_association" "public" {
  count = length(var.availability_zones)

  subnet_id      = aws_subnet.public[count.index].id
  route_table_id = aws_route_table.public.id
}

resource "aws_route_table_association" "private" {
  count = length(var.availability_zones)

  subnet_id      = aws_subnet.private[count.index].id
  route_table_id = aws_route_table.private[count.index].id
}

resource "aws_route_table_association" "database" {
  count = length(var.availability_zones)

  subnet_id      = aws_subnet.database[count.index].id
  route_table_id = aws_route_table.database.id
}

# Database Subnet Group
resource "aws_db_subnet_group" "main" {
  name       = "${var.project_name}-${var.environment}-db-subnet-group"
  subnet_ids = aws_subnet.database[*].id

  tags = merge(local.common_tags, {
    Name = "${var.project_name}-${var.environment}-db-subnet-group"
  })
}

# VPC Flow Logs
resource "aws_flow_log" "main" {
  iam_role_arn    = aws_iam_role.flow_log.arn
  log_destination = aws_cloudwatch_log_group.vpc_flow_log.arn
  traffic_type    = "ALL"
  vpc_id          = aws_vpc.main.id
}

resource "aws_cloudwatch_log_group" "vpc_flow_log" {
  name              = "/aws/vpc/flowlogs/${var.project_name}-${var.environment}"
  retention_in_days = 30

  tags = local.common_tags
}

resource "aws_iam_role" "flow_log" {
  name = "${var.project_name}-${var.environment}-flow-log-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "vpc-flow-logs.amazonaws.com"
        }
      }
    ]
  })

  tags = local.common_tags
}

resource "aws_iam_role_policy" "flow_log" {
  name = "${var.project_name}-${var.environment}-flow-log-policy"
  role = aws_iam_role.flow_log.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "logs:CreateLogGroup",
          "logs:CreateLogStream",
          "logs:PutLogEvents",
          "logs:DescribeLogGroups",
          "logs:DescribeLogStreams"
        ]
        Resource = "*"
      }
    ]
  })
}

# Outputs
output "vpc_id" {
  description = "ID of the VPC"
  value       = aws_vpc.main.id
}

output "vpc_cidr_block" {
  description = "CIDR block of the VPC"
  value       = aws_vpc.main.cidr_block
}

output "public_subnet_ids" {
  description = "IDs of the public subnets"
  value       = aws_subnet.public[*].id
}

output "private_subnet_ids" {
  description = "IDs of the private subnets"
  value       = aws_subnet.private[*].id
}

output "database_subnet_ids" {
  description = "IDs of the database subnets"
  value       = aws_subnet.database[*].id
}

output "database_subnet_group_name" {
  description = "Name of the database subnet group"
  value       = aws_db_subnet_group.main.name
}

output "internet_gateway_id" {
  description = "ID of the Internet Gateway"
  value       = aws_internet_gateway.main.id
}

output "nat_gateway_ids" {
  description = "IDs of the NAT Gateways"
  value       = aws_nat_gateway.main[*].id
}

# terraform/environments/production/backend.tf - Remote state configuration
terraform {
  backend "s3" {
    bucket         = "myapp-terraform-state-production"
    key            = "infrastructure/production/terraform.tfstate"
    region         = "us-west-2"
    encrypt        = true
    dynamodb_table = "terraform-locks-production"

    # State locking and consistency checking
    skip_credentials_validation = false
    skip_metadata_api_check     = false
    skip_region_validation      = false
    force_path_style           = false
  }

  required_version = ">= 1.5.0"

  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.23"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "~> 2.11"
    }
  }
}

# terraform/environments/production/main.tf - Environment-specific configuration
locals {
  environment = "production"
  region      = "us-west-2"

  # Environment-specific configurations
  cluster_config = {
    version = "1.28"
    node_groups = {
      main = {
        instance_types = ["m5.large", "m5.xlarge"]
        capacity_type  = "ON_DEMAND"
        min_size       = 3
        max_size       = 20
        desired_size   = 5
      }
      spot = {
        instance_types = ["m5.large", "m5.xlarge", "m5a.large", "m5a.xlarge"]
        capacity_type  = "SPOT"
        min_size       = 2
        max_size       = 15
        desired_size   = 3
      }
    }
  }

  database_config = {
    engine_version = "15.4"
    instance_class = "db.r5.xlarge"
    multi_az       = true
    backup_retention = 30
    storage_size   = 500
    max_storage    = 2000
  }

  redis_config = {
    node_type         = "cache.r6g.large"
    num_cache_clusters = 3
    automatic_failover = true
    multi_az          = true
  }
}

data "aws_availability_zones" "available" {
  state = "available"
}

# Networking Module
module "networking" {
  source = "../../modules/networking"

  project_name       = var.project_name
  environment        = local.environment
  region            = local.region
  availability_zones = slice(data.aws_availability_zones.available.names, 0, 3)
  vpc_cidr          = "10.0.0.0/16"
  enable_nat_gateway = true
  enable_vpn_gateway = false
}

# Security Module
module "security" {
  source = "../../modules/security"

  project_name = var.project_name
  environment  = local.environment
  vpc_id       = module.networking.vpc_id
  vpc_cidr     = module.networking.vpc_cidr_block
}

# EKS Cluster Module
module "eks" {
  source = "../../modules/eks"

  project_name          = var.project_name
  environment          = local.environment
  region               = local.region
  kubernetes_version   = local.cluster_config.version

  vpc_id              = module.networking.vpc_id
  subnet_ids          = module.networking.private_subnet_ids
  security_group_ids  = [module.security.cluster_security_group_id]

  node_groups = local.cluster_config.node_groups

  # Add-ons
  enable_cluster_autoscaler = true
  enable_aws_load_balancer_controller = true
  enable_external_dns = true
  enable_cert_manager = true

  tags = {
    Environment = local.environment
    Terraform   = "true"
  }
}

# Database Module
module "database" {
  source = "../../modules/database"

  project_name   = var.project_name
  environment    = local.environment

  vpc_id                = module.networking.vpc_id
  subnet_group_name     = module.networking.database_subnet_group_name
  security_group_ids    = [module.security.database_security_group_id]

  engine_version       = local.database_config.engine_version
  instance_class       = local.database_config.instance_class
  allocated_storage    = local.database_config.storage_size
  max_allocated_storage = local.database_config.max_storage

  multi_az = local.database_config.multi_az
  backup_retention_period = local.database_config.backup_retention

  # Performance Insights
  performance_insights_enabled = true
  performance_insights_retention_period = 7

  # Enhanced Monitoring
  monitoring_interval = 60
  monitoring_role_arn = aws_iam_role.rds_enhanced_monitoring.arn

  tags = {
    Environment = local.environment
    Terraform   = "true"
  }
}

# Enhanced Monitoring IAM Role for RDS
resource "aws_iam_role" "rds_enhanced_monitoring" {
  name = "${var.project_name}-${local.environment}-rds-monitoring-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "monitoring.rds.amazonaws.com"
        }
      }
    ]
  })
}

resource "aws_iam_role_policy_attachment" "rds_enhanced_monitoring" {
  role       = aws_iam_role.rds_enhanced_monitoring.name
  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonRDSEnhancedMonitoringRole"
}

# Redis Module
module "redis" {
  source = "../../modules/redis"

  project_name = var.project_name
  environment  = local.environment

  vpc_id             = module.networking.vpc_id
  subnet_ids         = module.networking.private_subnet_ids
  security_group_ids = [module.security.redis_security_group_id]

  node_type               = local.redis_config.node_type
  num_cache_clusters      = local.redis_config.num_cache_clusters
  automatic_failover_enabled = local.redis_config.automatic_failover
  multi_az_enabled        = local.redis_config.multi_az

  # Security
  at_rest_encryption_enabled = true
  transit_encryption_enabled = true
  auth_token_enabled        = true

  # Backup
  snapshot_retention_limit = 7
  snapshot_window         = "03:00-05:00"

  tags = {
    Environment = local.environment
    Terraform   = "true"
  }
}

# Monitoring Module
module "monitoring" {
  source = "../../modules/monitoring"

  project_name = var.project_name
  environment  = local.environment

  cluster_name = module.eks.cluster_name
  vpc_id       = module.networking.vpc_id

  # SNS Topics for alerts
  create_sns_topics = true
  alert_email = var.alert_email

  # CloudWatch Dashboard
  create_dashboard = true

  # Log retention
  log_retention_days = 30

  tags = {
    Environment = local.environment
    Terraform   = "true"
  }
}

# Outputs
output "cluster_endpoint" {
  description = "EKS cluster endpoint"
  value       = module.eks.cluster_endpoint
  sensitive   = true
}

output "cluster_name" {
  description = "EKS cluster name"
  value       = module.eks.cluster_name
}

output "database_endpoint" {
  description = "RDS database endpoint"
  value       = module.database.endpoint
  sensitive   = true
}

output "redis_endpoint" {
  description = "Redis cluster endpoint"
  value       = module.redis.configuration_endpoint
  sensitive   = true
}

output "vpc_id" {
  description = "VPC ID"
  value       = module.networking.vpc_id
}

#!/bin/bash
# terraform-management.sh - Professional Terraform workflow automation

set -euo pipefail

PROJECT_NAME="${PROJECT_NAME:-myapp}"
TERRAFORM_VERSION="${TERRAFORM_VERSION:-1.5.7}"
AWS_REGION="${AWS_REGION:-us-west-2}"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

log() {
    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}"
}

warn() {
    echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING: $1${NC}"
}

error() {
    echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $1${NC}"
    exit 1
}

check_prerequisites() {
    log "Checking prerequisites..."

    # Check Terraform installation
    if ! command -v terraform &> /dev/null; then
        error "Terraform is not installed"
    fi

    local tf_version=$(terraform version -json | jq -r '.terraform_version')
    if [[ "$tf_version" != "$TERRAFORM_VERSION" ]]; then
        warn "Expected Terraform $TERRAFORM_VERSION, found $tf_version"
    fi

    # Check AWS CLI
    if ! command -v aws &> /dev/null; then
        error "AWS CLI is not installed"
    fi

    # Verify AWS credentials
    if ! aws sts get-caller-identity &> /dev/null; then
        error "AWS credentials not configured or invalid"
    fi

    log "Prerequisites check passed"
}

setup_terraform_backend() {
    local environment="$1"

    log "Setting up Terraform backend for $environment..."

    # Create S3 bucket for state if it doesn't exist
    local bucket_name="${PROJECT_NAME}-terraform-state-${environment}"
    if ! aws s3 ls "s3://$bucket_name" &> /dev/null; then
        log "Creating S3 bucket: $bucket_name"
        aws s3 mb "s3://$bucket_name" --region "$AWS_REGION"

        # Enable versioning
        aws s3api put-bucket-versioning \
            --bucket "$bucket_name" \
            --versioning-configuration Status=Enabled

        # Enable server-side encryption
        aws s3api put-bucket-encryption \
            --bucket "$bucket_name" \
            --server-side-encryption-configuration '{
                "Rules": [{
                    "ApplyServerSideEncryptionByDefault": {
                        "SSEAlgorithm": "AES256"
                    }
                }]
            }'

        # Block public access
        aws s3api put-public-access-block \
            --bucket "$bucket_name" \
            --public-access-block-configuration \
                BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true
    fi

    # Create DynamoDB table for locking if it doesn't exist
    local table_name="terraform-locks-${environment}"
    if ! aws dynamodb describe-table --table-name "$table_name" &> /dev/null; then
        log "Creating DynamoDB table: $table_name"
        aws dynamodb create-table \
            --table-name "$table_name" \
            --attribute-definitions AttributeName=LockID,AttributeType=S \
            --key-schema AttributeName=LockID,KeyType=HASH \
            --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \
            --region "$AWS_REGION"

        # Wait for table to be active
        aws dynamodb wait table-exists --table-name "$table_name"
    fi

    log "Terraform backend setup completed for $environment"
}

terraform_plan() {
    local environment="$1"
    local workspace_dir="terraform/environments/$environment"

    if [[ ! -d "$workspace_dir" ]]; then
        error "Environment directory not found: $workspace_dir"
    fi

    cd "$workspace_dir"

    log "Planning Terraform changes for $environment..."

    # Initialize if needed
    terraform init -upgrade

    # Validate configuration
    terraform validate

    # Format check
    if ! terraform fmt -check -recursive; then
        warn "Terraform files are not properly formatted. Run 'terraform fmt -recursive' to fix."
    fi

    # Security scan with tfsec
    if command -v tfsec &> /dev/null; then
        log "Running security scan with tfsec..."
        tfsec . --format=junit --out=tfsec-report.xml || warn "Security issues found"
    fi

    # Cost estimation with Infracost
    if command -v infracost &> /dev/null && [[ -n "${INFRACOST_API_KEY:-}" ]]; then
        log "Generating cost estimate..."
        infracost breakdown --path . --format json --out-file infracost.json
        infracost diff --path . --format table
    fi

    # Plan with detailed output
    terraform plan \
        -detailed-exitcode \
        -out="tfplan-$(date +%Y%m%d-%H%M%S).plan" \
        -var="project_name=$PROJECT_NAME" \
        -var-file="terraform.tfvars"

    local plan_exit_code=$?

    if [[ $plan_exit_code -eq 1 ]]; then
        error "Terraform plan failed"
    elif [[ $plan_exit_code -eq 2 ]]; then
        log "Terraform plan completed with changes"
    else
        log "Terraform plan completed - no changes"
    fi

    cd - > /dev/null
}

terraform_apply() {
    local environment="$1"
    local workspace_dir="terraform/environments/$environment"
    local auto_approve="${2:-false}"

    if [[ ! -d "$workspace_dir" ]]; then
        error "Environment directory not found: $workspace_dir"
    fi

    cd "$workspace_dir"

    log "Applying Terraform changes for $environment..."

    # Find the latest plan file
    local plan_file=$(ls -t tfplan-*.plan 2>/dev/null | head -n1 || echo "")

    if [[ -z "$plan_file" ]]; then
        warn "No plan file found. Running plan first..."
        terraform plan -var="project_name=$PROJECT_NAME" -var-file="terraform.tfvars"
    fi

    # Apply changes
    local apply_args=(-var="project_name=$PROJECT_NAME" -var-file="terraform.tfvars")

    if [[ "$auto_approve" == "true" ]]; then
        apply_args+=(-auto-approve)
    fi

    if [[ -n "$plan_file" ]]; then
        terraform apply "${apply_args[@]}" "$plan_file"
    else
        terraform apply "${apply_args[@]}"
    fi

    # Clean up old plan files
    find . -name "tfplan-*.plan" -mtime +7 -delete

    log "Terraform apply completed for $environment"

    # Output important values
    log "Retrieving outputs..."
    terraform output -json > "outputs-$(date +%Y%m%d-%H%M%S).json"

    cd - > /dev/null
}

terraform_destroy() {
    local environment="$1"
    local workspace_dir="terraform/environments/$environment"

    if [[ ! -d "$workspace_dir" ]]; then
        error "Environment directory not found: $workspace_dir"
    fi

    if [[ "$environment" == "production" ]]; then
        error "Destruction of production environment requires manual confirmation"
    fi

    cd "$workspace_dir"

    warn "This will DESTROY all resources in $environment environment!"
    read -p "Are you absolutely sure? Type 'yes' to confirm: " confirmation

    if [[ "$confirmation" != "yes" ]]; then
        log "Destruction cancelled"
        cd - > /dev/null
        return
    fi

    log "Destroying Terraform resources for $environment..."

    terraform destroy \
        -var="project_name=$PROJECT_NAME" \
        -var-file="terraform.tfvars"

    log "Terraform destroy completed for $environment"

    cd - > /dev/null
}

state_management() {
    local action="$1"
    local environment="$2"
    local workspace_dir="terraform/environments/$environment"

    if [[ ! -d "$workspace_dir" ]]; then
        error "Environment directory not found: $workspace_dir"
    fi

    cd "$workspace_dir"

    case "$action" in
        "list")
            log "Listing Terraform state resources for $environment..."
            terraform state list
            ;;
        "show")
            local resource="$3"
            if [[ -z "$resource" ]]; then
                error "Resource name required for show command"
            fi
            terraform state show "$resource"
            ;;
        "pull")
            log "Pulling remote state for $environment..."
            terraform state pull > "state-backup-$(date +%Y%m%d-%H%M%S).json"
            log "State backed up locally"
            ;;
        "refresh")
            log "Refreshing Terraform state for $environment..."
            terraform refresh -var="project_name=$PROJECT_NAME" -var-file="terraform.tfvars"
            ;;
        *)
            error "Unknown state action: $action"
            ;;
    esac

    cd - > /dev/null
}

# Main command router
case "${1:-help}" in
    "init")
        check_prerequisites
        setup_terraform_backend "${2:-staging}"
        ;;
    "plan")
        check_prerequisites
        terraform_plan "${2:-staging}"
        ;;
    "apply")
        check_prerequisites
        terraform_apply "${2:-staging}" "${3:-false}"
        ;;
    "destroy")
        check_prerequisites
        terraform_destroy "${2:-staging}"
        ;;
    "state")
        check_prerequisites
        state_management "${2:-list}" "${3:-staging}" "${4:-}"
        ;;
    "help"|*)
        cat << EOF
Terraform Infrastructure Management

Usage: $0 <command> [options]

Commands:
    init <environment>              Initialize Terraform backend
    plan <environment>              Plan infrastructure changes
    apply <environment> [auto]      Apply infrastructure changes
    destroy <environment>           Destroy infrastructure (staging only)
    state <action> <environment>    Manage Terraform state

State Actions:
    list                           List all resources
    show <resource>                Show resource details
    pull                           Backup state locally
    refresh                        Refresh state from real infrastructure

Environments:
    staging                        Staging environment
    production                     Production environment

Examples:
    $0 init staging
    $0 plan production
    $0 apply staging auto
    $0 state list production
    $0 state show production aws_vpc.main
EOF
        ;;
esac

// Monitoring evolution: Reactive to Predictive Operations
const monitoringMaturity = {
  level1_Reactive: {
    approach: "Monitor basic uptime, react to customer complaints",
    characteristics: [
      "Ping monitoring for basic availability",
      "Manual log checking when problems occur",
      "Alerts come from angry customers on social media",
      "Debugging happens during outages",
    ],
    problems: [
      "Issues discovered hours after they occur",
      "No visibility into performance degradation",
      "Root cause analysis takes days",
      "Customer experience suffers",
    ],
    reality: "You're always one step behind problems",
  },

  level2_Proactive: {
    approach: "Monitor key metrics, alert before customer impact",
    characteristics: [
      "Application performance monitoring (APM)",
      "Infrastructure metrics and thresholds",
      "Automated alerts for critical issues",
      "Basic dashboards and visualization",
    ],
    benefits: [
      "Issues detected before customer complaints",
      "Faster mean time to resolution (MTTR)",
      "Better understanding of system behavior",
      "Reduced firefighting stress",
    ],
    limitations: "Still reactive to symptoms, not causes",
  },

  level3_Predictive: {
    approach: "Predict problems, prevent outages, optimize performance",
    characteristics: [
      "Machine learning-based anomaly detection",
      "Predictive alerting based on trends",
      "Automatic remediation for known issues",
      "Comprehensive observability platform",
    ],
    advantages: [
      "Problems prevented before they occur",
      "Automatic scaling and optimization",
      "Data-driven capacity planning",
      "Continuous performance improvement",
    ],
    outcome: "Operations become invisible to customers",
  },
};

# monitoring/prometheus/prometheus.yml - Production Prometheus configuration
global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
    cluster: "production"
    region: "us-west-2"
    environment: "production"

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093
      path_prefix: /alertmanager
      scheme: http

# Rules for alerts and recording rules
rule_files:
  - "/etc/prometheus/rules/*.yml"
  - "/etc/prometheus/alerts/*.yml"

# Scrape configurations
scrape_configs:
  # Prometheus self-monitoring
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]
    scrape_interval: 5s
    metrics_path: /metrics

  # Node Exporter for system metrics
  - job_name: "node-exporter"
    kubernetes_sd_configs:
      - role: endpoints
        namespaces:
          names:
            - monitoring
    relabel_configs:
      - source_labels: [__meta_kubernetes_endpoints_name]
        action: keep
        regex: node-exporter
      - source_labels: [__meta_kubernetes_endpoint_address_target_name]
        target_label: node
      - source_labels: [__meta_kubernetes_pod_node_name]
        target_label: instance

  # Kubernetes API Server
  - job_name: "kubernetes-apiservers"
    kubernetes_sd_configs:
      - role: endpoints
        namespaces:
          names:
            - default
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecure_skip_verify: true
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
      - source_labels:
          [
            __meta_kubernetes_namespace,
            __meta_kubernetes_service_name,
            __meta_kubernetes_endpoint_port_name,
          ]
        action: keep
        regex: default;kubernetes;https

  # Kubernetes nodes (kubelet)
  - job_name: "kubernetes-nodes"
    kubernetes_sd_configs:
      - role: node
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecure_skip_verify: true
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics

  # Application pods
  - job_name: "kubernetes-pods"
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels:
          [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name

  # Database monitoring (PostgreSQL)
  - job_name: "postgres-exporter"
    static_configs:
      - targets: ["postgres-exporter:9187"]
    scrape_interval: 30s

  # Redis monitoring
  - job_name: "redis-exporter"
    static_configs:
      - targets: ["redis-exporter:9121"]
    scrape_interval: 30s

  # AWS CloudWatch metrics
  - job_name: "cloudwatch-exporter"
    static_configs:
      - targets: ["cloudwatch-exporter:9106"]
    scrape_interval: 60s

  # Application-specific metrics
  - job_name: "myapp"
    kubernetes_sd_configs:
      - role: endpoints
        namespaces:
          names:
            - production
    relabel_configs:
      - source_labels:
          [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels:
          [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name

# Remote write configuration for long-term storage
remote_write:
  - url: "https://prometheus-prod.monitoring.myapp.com/api/v1/write"
    queue_config:
      max_samples_per_send: 1000
      max_shards: 200
      capacity: 2500

# monitoring/prometheus/alerts/application-alerts.yml - Comprehensive alerting rules
groups:
  - name: application.rules
    rules:
      # ========================================
      # Application Availability Alerts
      # ========================================
      - alert: ApplicationDown
        expr: up{job="myapp"} == 0
        for: 30s
        labels:
          severity: critical
          team: platform
          service: "{{ $labels.kubernetes_name }}"
        annotations:
          summary: "Application {{ $labels.instance }} is down"
          description: |
            Application {{ $labels.kubernetes_name }} in namespace {{ $labels.kubernetes_namespace }} 
            has been down for more than 30 seconds.

            Instance: {{ $labels.instance }}
            Job: {{ $labels.job }}
          runbook_url: "https://runbooks.myapp.com/application-down"
          dashboard_url: "https://grafana.myapp.com/d/app-overview"

      - alert: ApplicationHighErrorRate
        expr: |
          (
            rate(http_requests_total{job="myapp", status=~"5.."}[5m]) /
            rate(http_requests_total{job="myapp"}[5m])
          ) > 0.05
        for: 2m
        labels:
          severity: warning
          team: platform
          service: "{{ $labels.kubernetes_name }}"
        annotations:
          summary: "High error rate detected for {{ $labels.service }}"
          description: |
            Application {{ $labels.service }} is experiencing {{ $value | humanizePercentage }} error rate
            for more than 2 minutes.

            Current error rate: {{ $value | humanizePercentage }}
            Threshold: 5%
          runbook_url: "https://runbooks.myapp.com/high-error-rate"

      - alert: ApplicationHighLatency
        expr: |
          histogram_quantile(0.95, 
            rate(http_request_duration_seconds_bucket{job="myapp"}[5m])
          ) > 0.5
        for: 2m
        labels:
          severity: warning
          team: platform
          service: "{{ $labels.kubernetes_name }}"
        annotations:
          summary: "High latency detected for {{ $labels.service }}"
          description: |
            Application {{ $labels.service }} 95th percentile latency is {{ $value }}s
            for more than 2 minutes.

            Current P95 latency: {{ $value }}s
            Threshold: 0.5s
          runbook_url: "https://runbooks.myapp.com/high-latency"

      # ========================================
      # Infrastructure Alerts
      # ========================================
      - alert: HighCPUUsage
        expr: |
          (
            100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
          ) > 80
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: |
            CPU usage on {{ $labels.instance }} has been above 80% for more than 5 minutes.

            Current usage: {{ $value | humanizePercentage }}
            Threshold: 80%
          runbook_url: "https://runbooks.myapp.com/high-cpu"

      - alert: HighMemoryUsage
        expr: |
          (
            1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)
          ) > 0.85
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "High memory usage on {{ $labels.instance }}"
          description: |
            Memory usage on {{ $labels.instance }} has been above 85% for more than 5 minutes.

            Current usage: {{ $value | humanizePercentage }}
            Available: {{ with query "node_memory_MemAvailable_bytes{instance=\"" }}{{ . | first | value | humanize1024 }}B{{ end }}
            Total: {{ with query "node_memory_MemTotal_bytes{instance=\"" }}{{ . | first | value | humanize1024 }}B{{ end }}
          runbook_url: "https://runbooks.myapp.com/high-memory"

      - alert: DiskSpaceLow
        expr: |
          (
            100 - (node_filesystem_avail_bytes / node_filesystem_size_bytes) * 100
          ) > 85
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Disk space low on {{ $labels.instance }}"
          description: |
            Disk usage on {{ $labels.instance }}:{{ $labels.mountpoint }} is {{ $value | humanizePercentage }}.

            Available: {{ with query "node_filesystem_avail_bytes{instance=\"" }}{{ . | first | value | humanize1024 }}B{{ end }}
            Total: {{ with query "node_filesystem_size_bytes{instance=\"" }}{{ . | first | value | humanize1024 }}B{{ end }}
          runbook_url: "https://runbooks.myapp.com/disk-space-low"

      # ========================================
      # Database Alerts
      # ========================================
      - alert: DatabaseConnectionsHigh
        expr: |
          (pg_stat_activity_count / pg_settings_max_connections) > 0.8
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "High database connections on {{ $labels.instance }}"
          description: |
            Database {{ $labels.instance }} is using {{ $value | humanizePercentage }} of max connections.

            Current connections: {{ with query "pg_stat_activity_count{instance=\"" }}{{ . | first | value }}{{ end }}
            Max connections: {{ with query "pg_settings_max_connections{instance=\"" }}{{ . | first | value }}{{ end }}
          runbook_url: "https://runbooks.myapp.com/db-connections-high"

      - alert: DatabaseSlowQueries
        expr: |
          rate(pg_stat_statements_mean_time_ms[5m]) > 1000
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Slow database queries detected on {{ $labels.instance }}"
          description: |
            Database {{ $labels.instance }} has queries with average execution time of {{ $value }}ms.

            Query: {{ $labels.query }}
            Average time: {{ $value }}ms
          runbook_url: "https://runbooks.myapp.com/slow-queries"

      # ========================================
      # Kubernetes Alerts
      # ========================================
      - alert: KubernetesPodCrashLooping
        expr: |
          rate(kube_pod_container_status_restarts_total[10m]) > 0
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping"
          description: |
            Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container }} 
            is restarting {{ $value }} times per second.
          runbook_url: "https://runbooks.myapp.com/pod-crash-loop"

      - alert: KubernetesNodeNotReady
        expr: |
          kube_node_status_condition{condition="Ready",status="true"} == 0
        for: 5m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "Kubernetes node {{ $labels.node }} is not ready"
          description: |
            Node {{ $labels.node }} has been in NotReady state for more than 5 minutes.
          runbook_url: "https://runbooks.myapp.com/node-not-ready"

      - alert: KubernetesPodPending
        expr: |
          kube_pod_status_phase{phase="Pending"} == 1
        for: 10m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} stuck in Pending"
          description: |
            Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in Pending state for more than 10 minutes.
            This usually indicates resource constraints or scheduling issues.
          runbook_url: "https://runbooks.myapp.com/pod-pending"

  - name: sli.rules
    interval: 30s
    rules:
      # ========================================
      # SLI (Service Level Indicator) Recording Rules
      # ========================================
      - record: sli:http_requests:rate5m
        expr: |
          sum(rate(http_requests_total[5m])) by (service, method, status)

      - record: sli:http_request_duration:p95_5m
        expr: |
          histogram_quantile(0.95, 
            sum(rate(http_request_duration_seconds_bucket[5m])) by (service, le)
          )

      - record: sli:http_request_duration:p99_5m
        expr: |
          histogram_quantile(0.99, 
            sum(rate(http_request_duration_seconds_bucket[5m])) by (service, le)
          )

      - record: sli:availability:5m
        expr: |
          sum(rate(http_requests_total{status!~"5.."}[5m])) by (service) /
          sum(rate(http_requests_total[5m])) by (service)

      - record: sli:error_rate:5m
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service) /
          sum(rate(http_requests_total[5m])) by (service)