Design a GitOps workflow with Jenkins for automated, auditable deployments to multiple environments.

Step 1: Design GitOps Repository Structure

# Separate repositories for separation of concerns

app-repo/                    # Application source code
├── src/
├── Dockerfile
├── Jenkinsfile              # CI pipeline (build, test, push image)
└── package.json

infra-repo/                  # Infrastructure/deployment configuration
├── base/                    # Base Kubernetes manifests
│   ├── deployment.yaml
│   ├── service.yaml
│   ├── configmap.yaml
│   └── kustomization.yaml
├── environments/
│   ├── dev/
│   │   ├── kustomization.yaml
│   │   ├── replicas-patch.yaml
│   │   └── resources-patch.yaml
│   ├── staging/
│   │   ├── kustomization.yaml
│   │   ├── replicas-patch.yaml
│   │   └── resources-patch.yaml
│   └── production/
│       ├── kustomization.yaml
│       ├── replicas-patch.yaml
│       ├── hpa.yaml
│       └── pdb.yaml
├── Jenkinsfile              # CD pipeline (deploy to environments)
└── CODEOWNERS               # Require approvals for prod changes

Step 2: CI Pipeline - Build and Update Manifests

// app-repo/Jenkinsfile - CI Pipeline
pipeline {
    agent {
        kubernetes {
            yaml '''
apiVersion: v1
kind: Pod
spec:
  containers:
  - name: docker
    image: docker:24-cli
    command: ["sleep", "infinity"]
  - name: dind
    image: docker:24-dind
    securityContext:
      privileged: true
'''
        }
    }

    environment {
        REGISTRY = 'registry.company.com'
        APP_NAME = 'my-application'
        INFRA_REPO = 'github.com/company/infra-repo.git'
    }

    stages {
        stage('Build & Test') {
            steps {
                container('docker') {
                    sh '''
                        docker build -t ${REGISTRY}/${APP_NAME}:${GIT_COMMIT} .
                        docker run --rm ${REGISTRY}/${APP_NAME}:${GIT_COMMIT} npm test
                    '''
                }
            }
        }

        stage('Push Image') {
            steps {
                container('docker') {
                    withCredentials([usernamePassword(
                        credentialsId: 'registry-creds',
                        usernameVariable: 'REG_USER',
                        passwordVariable: 'REG_PASS'
                    )]) {
                        sh '''
                            echo $REG_PASS | docker login ${REGISTRY} -u $REG_USER --password-stdin
                            docker push ${REGISTRY}/${APP_NAME}:${GIT_COMMIT}

                            # Tag as latest for the branch
                            docker tag ${REGISTRY}/${APP_NAME}:${GIT_COMMIT} \
                                ${REGISTRY}/${APP_NAME}:${BRANCH_NAME}-latest
                            docker push ${REGISTRY}/${APP_NAME}:${BRANCH_NAME}-latest
                        '''
                    }
                }
            }
        }

        stage('Update Infra Repo') {
            when {
                anyOf {
                    branch 'main'
                    branch 'develop'
                }
            }
            steps {
                script {
                    def targetEnv = (env.BRANCH_NAME == 'main') ? 'production' : 'staging'

                    withCredentials([usernamePassword(
                        credentialsId: 'github-credentials',
                        usernameVariable: 'GIT_USER',
                        passwordVariable: 'GIT_TOKEN'
                    )]) {
                        sh """
                            # Clone infra repo
                            git clone https://${GIT_USER}:${GIT_TOKEN}@${INFRA_REPO} infra
                            cd infra

                            # Update image tag in kustomization
                            cd environments/${targetEnv}
                            kustomize edit set image ${REGISTRY}/${APP_NAME}:${GIT_COMMIT}

                            # Commit and push
                            git config user.email "jenkins@company.com"
                            git config user.name "Jenkins CI"
                            git add .
                            git commit -m "chore: Update ${APP_NAME} to ${GIT_COMMIT}

                            Source commit: ${GIT_COMMIT}
                            Source branch: ${BRANCH_NAME}
                            Jenkins build: ${BUILD_URL}
                            "

                            # For production, create PR instead of direct push
                            if [ "${targetEnv}" = "production" ]; then
                                git checkout -b deploy/${APP_NAME}-${GIT_COMMIT}
                                git push origin deploy/${APP_NAME}-${GIT_COMMIT}

                                # Create PR using GitHub CLI
                                gh pr create \
                                    --title "Deploy ${APP_NAME} to production (${GIT_COMMIT})" \
                                    --body "Automated deployment request\\n\\nSource: ${BUILD_URL}" \
                                    --reviewer platform-team
                            else
                                git push origin main
                            fi
                        """
                    }
                }
            }
        }
    }

    post {
        success {
            script {
                def targetEnv = (env.BRANCH_NAME == 'main') ? 'production' : 'staging'
                slackSend(
                    channel: '#deployments',
                    color: 'good',
                    message: """
                        *Image Ready for Deployment*
                        App: ${APP_NAME}
                        Tag: ${GIT_COMMIT}
                        Environment: ${targetEnv}
                        ${targetEnv == 'production' ? '⚠️ PR created - awaiting approval' : '✅ Auto-deploying to staging'}
                    """
                )
            }
        }
    }
}

Step 3: CD Pipeline - Deploy from Infra Repo

// infra-repo/Jenkinsfile - CD Pipeline
pipeline {
    agent {
        kubernetes {
            yaml '''
apiVersion: v1
kind: Pod
spec:
  containers:
  - name: kubectl
    image: bitnami/kubectl:latest
    command: ["sleep", "infinity"]
'''
        }
    }

    parameters {
        choice(
            name: 'ENVIRONMENT',
            choices: ['dev', 'staging', 'production'],
            description: 'Target environment'
        )
        booleanParam(
            name: 'DRY_RUN',
            defaultValue: false,
            description: 'Show what would be deployed without applying'
        )
    }

    stages {
        stage('Validate') {
            steps {
                container('kubectl') {
                    sh """
                        # Validate manifests
                        kustomize build environments/${params.ENVIRONMENT} | \
                            kubectl apply --dry-run=client -f -

                        echo "=== Resources to be deployed ==="
                        kustomize build environments/${params.ENVIRONMENT} | \
                            kubectl diff -f - || true
                    """
                }
            }
        }

        stage('Production Approval') {
            when {
                expression { params.ENVIRONMENT == 'production' && !params.DRY_RUN }
            }
            steps {
                script {
                    // Get list of changes
                    def changes = sh(
                        script: "git log --oneline HEAD~5..HEAD",
                        returnStdout: true
                    ).trim()

                    def approval = input(
                        message: "Deploy to Production?",
                        ok: 'Deploy',
                        submitter: 'release-managers,platform-team',
                        parameters: [
                            text(
                                name: 'APPROVAL_NOTE',
                                defaultValue: '',
                                description: 'Optional: Add deployment notes'
                            )
                        ]
                    )

                    env.APPROVER = currentBuild.getBuildCauses()[0]?.userId ?: 'unknown'
                    env.APPROVAL_NOTE = approval ?: ''
                }
            }
        }

        stage('Deploy') {
            when {
                expression { !params.DRY_RUN }
            }
            steps {
                container('kubectl') {
                    withCredentials([file(
                        credentialsId: "kubeconfig-${params.ENVIRONMENT}",
                        variable: 'KUBECONFIG'
                    )]) {
                        sh """
                            # Apply manifests
                            kustomize build environments/${params.ENVIRONMENT} | \
                                kubectl apply -f -

                            # Wait for rollout
                            kubectl rollout status deployment -n ${params.ENVIRONMENT} \
                                --timeout=300s
                        """
                    }
                }
            }
        }

        stage('Verify') {
            when {
                expression { !params.DRY_RUN }
            }
            steps {
                container('kubectl') {
                    withCredentials([file(
                        credentialsId: "kubeconfig-${params.ENVIRONMENT}",
                        variable: 'KUBECONFIG'
                    )]) {
                        sh """
                            # Health checks
                            kubectl get pods -n ${params.ENVIRONMENT}

                            # Run smoke tests
                            ENDPOINT=\$(kubectl get svc -n ${params.ENVIRONMENT} \
                                -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}')
                            curl -sf http://\$ENDPOINT/health || exit 1
                        """
                    }
                }
            }
        }

        stage('Record Deployment') {
            when {
                expression { !params.DRY_RUN }
            }
            steps {
                script {
                    // Create deployment record for audit
                    def deploymentRecord = [
                        timestamp: new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'"),
                        environment: params.ENVIRONMENT,
                        gitCommit: env.GIT_COMMIT,
                        gitBranch: env.GIT_BRANCH,
                        deployer: env.APPROVER ?: currentBuild.getBuildCauses()[0]?.userId,
                        buildUrl: env.BUILD_URL,
                        approvalNote: env.APPROVAL_NOTE ?: '',
                        status: 'success'
                    ]

                    writeJSON file: 'deployment-record.json', json: deploymentRecord

                    // Push to audit log
                    sh '''
                        # Send to audit system
                        curl -X POST https://audit.company.com/deployments \
                            -H "Content-Type: application/json" \
                            -d @deployment-record.json
                    '''

                    archiveArtifacts artifacts: 'deployment-record.json'
                }
            }
        }
    }

    post {
        success {
            slackSend(
                channel: '#deployments',
                color: 'good',
                message: """
                    *Deployment Successful*
                    Environment: ${params.ENVIRONMENT}
                    Commit: ${env.GIT_COMMIT}
                    Deployed by: ${env.APPROVER ?: 'automated'}
                    Build: ${env.BUILD_URL}
                """
            )
        }
        failure {
            slackSend(
                channel: '#deployments',
                color: 'danger',
                message: """
                    *Deployment Failed*
                    Environment: ${params.ENVIRONMENT}
                    Build: ${env.BUILD_URL}
                    Please check logs and consider rollback
                """
            )
        }
    }
}

Step 4: Implement Rollback Capability

// rollback-pipeline.groovy
pipeline {
    agent any

    parameters {
        choice(
            name: 'ENVIRONMENT',
            choices: ['staging', 'production'],
            description: 'Target environment'
        )
        string(
            name: 'TARGET_COMMIT',
            description: 'Git commit to rollback to (leave empty for previous)'
        )
    }

    stages {
        stage('Get Rollback Target') {
            steps {
                script {
                    if (params.TARGET_COMMIT?.trim()) {
                        env.ROLLBACK_COMMIT = params.TARGET_COMMIT
                    } else {
                        // Get previous deployment
                        env.ROLLBACK_COMMIT = sh(
                            script: "git log --skip=1 -1 --format=%H environments/${params.ENVIRONMENT}",
                            returnStdout: true
                        ).trim()
                    }

                    echo "Rolling back to commit: ${env.ROLLBACK_COMMIT}"
                }
            }
        }

        stage('Production Approval') {
            when {
                expression { params.ENVIRONMENT == 'production' }
            }
            steps {
                input(
                    message: "Confirm rollback to ${env.ROLLBACK_COMMIT}?",
                    ok: 'Rollback',
                    submitter: 'release-managers,platform-team'
                )
            }
        }

        stage('Execute Rollback') {
            steps {
                sh """
                    git checkout ${env.ROLLBACK_COMMIT} -- environments/${params.ENVIRONMENT}
                    git commit -m "Rollback ${params.ENVIRONMENT} to ${env.ROLLBACK_COMMIT}"
                    git push origin main
                """

                // Trigger deployment pipeline
                build(
                    job: 'infra-deploy',
                    parameters: [
                        string(name: 'ENVIRONMENT', value: params.ENVIRONMENT)
                    ],
                    wait: true
                )
            }
        }
    }
}

Step 5: Environment Promotion Flow

// promote-pipeline.groovy
pipeline {
    agent any

    parameters {
        choice(
            name: 'SOURCE_ENV',
            choices: ['dev', 'staging'],
            description: 'Source environment'
        )
        choice(
            name: 'TARGET_ENV',
            choices: ['staging', 'production'],
            description: 'Target environment'
        )
    }

    stages {
        stage('Validate Promotion Path') {
            steps {
                script {
                    def validPaths = [
                        'dev': ['staging'],
                        'staging': ['production']
                    ]

                    if (!validPaths[params.SOURCE_ENV].contains(params.TARGET_ENV)) {
                        error "Invalid promotion path: ${params.SOURCE_ENV} -> ${params.TARGET_ENV}"
                    }
                }
            }
        }

        stage('Get Current Version') {
            steps {
                script {
                    env.SOURCE_IMAGE = sh(
                        script: """
                            kustomize build environments/${params.SOURCE_ENV} | \
                            grep 'image:' | head -1 | awk '{print \$2}'
                        """,
                        returnStdout: true
                    ).trim()

                    echo "Promoting image: ${env.SOURCE_IMAGE}"
                }
            }
        }

        stage('Update Target Environment') {
            steps {
                sh """
                    cd environments/${params.TARGET_ENV}
                    kustomize edit set image ${env.SOURCE_IMAGE}

                    git add .
                    git commit -m "Promote ${params.SOURCE_ENV} -> ${params.TARGET_ENV}

                    Image: ${env.SOURCE_IMAGE}
                    "
                    git push origin main
                """
            }
        }
    }
}