Fredric News

쿠버네티스 파드 기반 Bitbucket Runner 배포 매니페스트

💡 핵심 요약 (Executive Summary)

본 가이드는 bitbucket runner.yaml 핵심 가이드 및 실전 아키텍처의 핵심 개념, 실전 구성 명령어 및 아키텍처 베스트 프랙티스를 다룹니다. 클라우드 네이티브 및 인프라 엔지니어링 실무에 즉시 활용 가능하도록 체계적으로 정리했습니다.

📑 Table of Contents
Reference & Guidelines (NOTE)

docker를 kubernetes CRI로 사용하는 경우를 가정함.

  • CRI버전에 따라 docker-in-docker 이미지 이슈가 존재합니다. 자세한 내용:

Job based (for Autoscaler)

Reference & Guidelines (NOTE)

autoscaler 설치 안해도 deployments 유사하게 동작 가능.

🔗 https://support.atlassian.com/bitbucket-cloud/kb/how-to-use-already-created-secrets-manager-in-pipelines-runners-autoscaler/
bash
generate-runner.sh
bash
#!/bin/bash
NS=bitbucket-runner
JOB_NAME=$1
DOCKER_EXEC="""
$2
"""
ACCOUNT_UUID=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e ACCOUNT_UUID=\{)[^}]+' )
REPOSITORY_UUID=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e REPOSITORY_UUID=\{)[^}]+' )
RUNNER_UUID=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e RUNNER_UUID=\{)[^}]+' )
OAUTH_CLIENT_ID=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e OAUTH_CLIENT_ID=)[^\s]+' )
OAUTH_CLIENT_SECRET=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e OAUTH_CLIENT_SECRET=)[^\s]+' )

CRI_VERSION=$(docker version | sed 's/ Docker Engine - Community//g' | yq '.Server.Engine.Version')

#kubectl apply -f - <<EOF
echo """
apiVersion: v1
kind: List
items:
  - apiVersion: v1
    kind: Secret
    metadata:
      name: ${JOB_NAME}-runner-oauth-credentials
      namespace: ${NS}
      labels:
        accountUuid: ${ACCOUNT_UUID} # Add your account uuid without curly braces to optionally allow finding the secret for an account
        repositoryUuid: ${REPOSITORY_UUID} # Add your repository uuid without curly braces to optionally allow finding the secret for a repository
        runnerUuid: ${RUNNER_UUID} # Add your runner uuid without curly braces to optionally allow finding the secret for a particular runner
    data:
      oauthClientId: $( echo -n ${OAUTH_CLIENT_ID} | base64 -w0 )
      oauthClientSecret: $( echo -n ${OAUTH_CLIENT_SECRET} | base64 -w0 )
  - apiVersion: batch/v1
    kind: Job
    metadata:
      name: ${JOB_NAME}-runner
      namespace: ${NS}
    spec:
      template:
        metadata:
          labels:
            accountUuid: ${ACCOUNT_UUID} # Add your account uuid without curly braces to optionally allow finding the secret for an account
            repositoryUuid: ${REPOSITORY_UUID} # Add your repository uuid without curly braces to optionally allow finding the secret for a repository
            runnerUuid: ${RUNNER_UUID} # Add your runner uuid without curly braces to optionally allow finding the secret for a particular runner
        spec:
          containers:
            - name: runner
              image: docker-public.packages.atlassian.com/sox/atlassian/bitbucket-pipelines-runner
              env:
                - name: ACCOUNT_UUID
                  value: '{${ACCOUNT_UUID}}' # Add your account uuid here
                - name: REPOSITORY_UUID
                  value: '{${REPOSITORY_UUID}}' # Add your repository uuid here
                - name: RUNNER_UUID
                  value: '{${RUNNER_UUID}}' # Add your runner uuid here
                - name: OAUTH_CLIENT_ID
                  valueFrom:
                    secretKeyRef:
                      name: ${JOB_NAME}-runner-oauth-credentials
                      key: oauthClientId
                - name: OAUTH_CLIENT_SECRET
                  valueFrom:
                    secretKeyRef:
                      name: ${JOB_NAME}-runner-oauth-credentials
                      key: oauthClientSecret
                - name: WORKING_DIRECTORY
                  value: "/tmp"
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                  readOnly: true # the runner only needs to read these files never write to them
                - name: var-run
                  mountPath: /var/run
            - name: docker-in-docker
              image: docker:${CRI_VERSION}-dind #docker:20.10.5-dind
              securityContext:
                privileged: true # required to allow docker in docker to run and assumes the namespace your applying this to has a pod security policy that allows privilege escalation
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                - name: var-run
                  mountPath: /var/run
          restartPolicy: OnFailure # this allows the runner to restart locally if it was to crash
          volumes:
            - name: tmp # required to share a working directory between docker in docker and the runner
            - name: docker-containers # required to share the containers directory between docker in docker and the runner
            - name: var-run # required to share the docker socket between docker in docker and the runner
        # backoffLimit: 6 # this is the default and means it will retry upto 6 times if it crashes before it considers itself a failure with an exponential backoff between
        # completions: 1 # this is the default the job should ideally never complete as the runner never shuts down successfully
        # parallelism: 1 # this is the default their should only be one instance of this particular runner
""" > ./${JOB_NAME}-runner.yaml
exec
bash
#example
./generate-runner.sh test \
'docker container run -it -v /tmp:/tmp -v /var/run/docker.sock:/var/ ... ' #bitbucket에서 복제
yaml
yaml
apiVersion: v1
kind: List
items:
  - apiVersion: v1
    kind: Secret
    metadata:
      name: ${JOB_NAME}-runner-oauth-credentials
      namespace: ${NS}
      labels:
        accountUuid: ${ACCOUNT_UUID} # Add your account uuid without curly braces to optionally allow finding the secret for an account
        repositoryUuid: ${REPOSITORY_UUID} # Add your repository uuid without curly braces to optionally allow finding the secret for a repository
        runnerUuid: ${RUNNER_UUID} # Add your runner uuid without curly braces to optionally allow finding the secret for a particular runner
    data:
      oauthClientId: $( echo -n ${OAUTH_CLIENT_ID} | base64 -w0 )
      oauthClientSecret: $( echo -n ${OAUTH_CLIENT_SECRET} | base64 -w0 )
  - apiVersion: batch/v1
    kind: Job
    metadata:
      name: ${JOB_NAME}-runner
      namespace: ${NS}
    spec:
      template:
        metadata:
          labels:
            accountUuid: ${ACCOUNT_UUID} # Add your account uuid without curly braces to optionally allow finding the secret for an account
            repositoryUuid: ${REPOSITORY_UUID} # Add your repository uuid without curly braces to optionally allow finding the secret for a repository
            runnerUuid: ${RUNNER_UUID} # Add your runner uuid without curly braces to optionally allow finding the secret for a particular runner
        spec:
          containers:
            - name: runner
              image: docker-public.packages.atlassian.com/sox/atlassian/bitbucket-pipelines-runner
              env:
                - name: ACCOUNT_UUID
                  value: '{${ACCOUNT_UUID}}' # Add your account uuid here
                - name: REPOSITORY_UUID
                  value: '{${REPOSITORY_UUID}}' # Add your repository uuid here
                - name: RUNNER_UUID
                  value: '{${RUNNER_UUID}}' # Add your runner uuid here
                - name: OAUTH_CLIENT_ID
                  valueFrom:
                    secretKeyRef:
                      name: ${JOB_NAME}-runner-oauth-credentials
                      key: oauthClientId
                - name: OAUTH_CLIENT_SECRET
                  valueFrom:
                    secretKeyRef:
                      name: ${JOB_NAME}-runner-oauth-credentials
                      key: oauthClientSecret
                - name: WORKING_DIRECTORY
                  value: "/tmp"
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                  readOnly: true # the runner only needs to read these files never write to them
                - name: var-run
                  mountPath: /var/run
            - name: docker-in-docker
              image: docker:${CRI_VERSION}-dind
              securityContext:
                privileged: true # required to allow docker in docker to run and assumes the namespace your applying this to has a pod security policy that allows privilege escalation
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                - name: var-run
                  mountPath: /var/run
          restartPolicy: OnFailure # this allows the runner to restart locally if it was to crash
          volumes:
            - name: tmp # required to share a working directory between docker in docker and the runner
            - name: docker-containers # required to share the containers directory between docker in docker and the runner
            - name: var-run # required to share the docker socket between docker in docker and the runner
        # backoffLimit: 6 # this is the default and means it will retry upto 6 times if it crashes before it considers itself a failure with an exponential backoff between
        # completions: 1 # this is the default the job should ideally never complete as the runner never shuts down successfully
        # parallelism: 1 # this is the default their should only be one instance of this particular runner

Deployment based

🔗 https://teyxt.medium.com/how-to-set-up-a-bitbucket-custom-runner-in-your-kubernetes-cluster-a-step-by-step-guide-e42f1d1f992d
yaml
apiVersion: v1
kind: List
items:
  - apiVersion: v1
    kind: Secret
    metadata:
      name: runner-oauth-credentials
      labels:
        accountUuid: 77eb8axx-xxx-xxx-xxx
        repositoryUuid: 23b695exxx-xxx-xxx
        runnerUuid: c64b921xxx-xxx-xxx
    data:
      oauthClientId: 9Glxxxxxx
      oauthClientSecret: AT0A6uexxxxx
#  - apiVersion: batch/v1
#    kind: Job
  - apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: runner
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: runner
      template:
        metadata:
          labels:
            accountUuid: 77eb8axx-xxx-xxx-xxx
            repositoryUuid: 23b695exxx-xxx-xxx
            runnerUuid: c64b921xxx-xxx-xxx
            app: runner
        spec:
          containers:
            - name: runner
              image: docker-public.packages.atlassian.com/sox/atlassian/bitbucket-pipelines-runner:1
              env:
                - name: ACCOUNT_UUID
                  value: "{77eb8axx-xxx-xxx-xxx}"
                - name: REPOSITORY_UUID
                  value: "{23b695exxx-xxx-xxx}"
                - name: RUNNER_UUID
                  value: "{c64b921xxx-xxx-xxx}"
                - name: OAUTH_CLIENT_ID
                  valueFrom:
                    secretKeyRef:
                      name: runner-oauth-credentials
                      key: oauthClientId
                - name: OAUTH_CLIENT_SECRET
                  valueFrom:
                    secretKeyRef:
                      name: runner-oauth-credentials
                      key: oauthClientSecret
                - name: WORKING_DIRECTORY
                  value: "/tmp"
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                  readOnly: true # the runner only needs to read these files never write to them
                - name: var-run
                  mountPath: /var/run
            - name: docker-in-docker
              image: docker:20.10.7-dind
              securityContext:
                privileged: true # required to allow docker in docker to run and assumes the namespace your applying this to has a pod security policy that allows privilege escalation
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                - name: var-run
                  mountPath: /var/run
          restartPolicy: Always # this allows the runner to restart locally if it was to crash
          volumes:
            - name: tmp # required to share a working directory between docker in docker and the runner
            - name: docker-containers # required to share the containers directory between docker in docker and the runner
            - name: var-run # required to share the docker socket between docker in docker and the runner
        # backoffLimit: 6 # this is the default and means it will retry upto 6 times if it crashes before it considers itself a failure with an exponential backoff between
        # completions: 1 # this is the default the job should ideally never complete as the runner never shuts down successfully
        # parallelism: 1
          nodeSelector:
            dedicated: runner-node

          tolerations:
            - key: "dedicated"
              operator: "Equal"
              value: "runner-node"
              effect: "NoSchedule"

Posted:

/

Category:

Author:

fredric

Tags:

, , , , , , , , , , , , , , ,

Comments

Leave a Reply

Your email address will not be published. Required fields are marked *