Fredric News

Bitbucket Pipelines Self-Hosted Runner on

๐Ÿ’ก Bitbucket Pipelines Self-Hosted Runner on โ€” Executive Summary

Technical architectural analysis, practical configuration manifests, and deployment instructions for Bitbucket Pipelines Self-Hosted Runner on.

References and Guidelines (NOTE)

Assume that docker is used as the Kubernetes CRI.

  • Depending on the CRI version, docker-in-docker image issues exist. More details:

Job based (for Autoscaler)

References and Guidelines (NOTE)

Can operate similarly to deployments even without installing autoscaler.

๐Ÿ”— https://support.atlassian.com/bitbucket-cloud/kb/how-to-use-already-created-secrets-manager-in-pipelines-runners-autoscaler/
bash
generate-runner.sh
bash
#!/bin/bash
NS=bitbucket-runner
JOB_NAME=$1
DOCKER_EXEC="""
$2
"""
ACCOUNT_UUID=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e ACCOUNT_UUID=\{)[^}]+' )
REPOSITORY_UUID=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e REPOSITORY_UUID=\{)[^}]+' )
RUNNER_UUID=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e RUNNER_UUID=\{)[^}]+' )
OAUTH_CLIENT_ID=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e OAUTH_CLIENT_ID=)[^\s]+' )
OAUTH_CLIENT_SECRET=$(echo "$DOCKER_EXEC" | grep -oP '(?<=-e OAUTH_CLIENT_SECRET=)[^\s]+' )

CRI_VERSION=$(docker version | sed 's/ Docker Engine - Community//g' | yq '.Server.Engine.Version')

#kubectl apply -f - <<EOF
echo """
apiVersion: v1
kind: List
items:
  - apiVersion: v1
    kind: Secret
    metadata:
      name: ${JOB_NAME}-runner-oauth-credentials
      namespace: ${NS}
      labels:
        accountUuid: ${ACCOUNT_UUID} # Add your account uuid without curly braces to optionally allow finding the secret for an account
        repositoryUuid: ${REPOSITORY_UUID} # Add your repository uuid without curly braces to optionally allow finding the secret for a repository
        runnerUuid: ${RUNNER_UUID} # Add your runner uuid without curly braces to optionally allow finding the secret for a particular runner
    data:
      oauthClientId: $( echo -n ${OAUTH_CLIENT_ID} | base64 -w0 )
      oauthClientSecret: $( echo -n ${OAUTH_CLIENT_SECRET} | base64 -w0 )
  - apiVersion: batch/v1
    kind: Job
    metadata:
      name: ${JOB_NAME}-runner
      namespace: ${NS}
    spec:
      template:
        metadata:
          labels:
            accountUuid: ${ACCOUNT_UUID} # Add your account uuid without curly braces to optionally allow finding the secret for an account
            repositoryUuid: ${REPOSITORY_UUID} # Add your repository uuid without curly braces to optionally allow finding the secret for a repository
            runnerUuid: ${RUNNER_UUID} # Add your runner uuid without curly braces to optionally allow finding the secret for a particular runner
        spec:
          containers:
            - name: runner
              image: docker-public.packages.atlassian.com/sox/atlassian/bitbucket-pipelines-runner
              env:
                - name: ACCOUNT_UUID
                  value: '{${ACCOUNT_UUID}}' # Add your account uuid here
                - name: REPOSITORY_UUID
                  value: '{${REPOSITORY_UUID}}' # Add your repository uuid here
                - name: RUNNER_UUID
                  value: '{${RUNNER_UUID}}' # Add your runner uuid here
                - name: OAUTH_CLIENT_ID
                  valueFrom:
                    secretKeyRef:
                      name: ${JOB_NAME}-runner-oauth-credentials
                      key: oauthClientId
                - name: OAUTH_CLIENT_SECRET
                  valueFrom:
                    secretKeyRef:
                      name: ${JOB_NAME}-runner-oauth-credentials
                      key: oauthClientSecret
                - name: WORKING_DIRECTORY
                  value: "/tmp"
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                  readOnly: true # the runner only needs to read these files never write to them
                - name: var-run
                  mountPath: /var/run
            - name: docker-in-docker
              image: docker:${CRI_VERSION}-dind #docker:20.10.5-dind
              securityContext:
                privileged: true # required to allow docker in docker to run and assumes the namespace your applying this to has a pod security policy that allows privilege escalation
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                - name: var-run
                  mountPath: /var/run
          restartPolicy: OnFailure # this allows the runner to restart locally if it was to crash
          volumes:
            - name: tmp # required to share a working directory between docker in docker and the runner
            - name: docker-containers # required to share the containers directory between docker in docker and the runner
            - name: var-run # required to share the docker socket between docker in docker and the runner
        # backoffLimit: 6 # this is the default and means it will retry upto 6 times if it crashes before it considers itself a failure with an exponential backoff between
        # completions: 1 # this is the default the job should ideally never complete as the runner never shuts down successfully
        # parallelism: 1 # this is the default their should only be one instance of this particular runner
""" > ./${JOB_NAME}-runner.yaml
exec
bash
#example
./generate-runner.sh test \
'docker container run -it -v /tmp:/tmp -v /var/run/docker.sock:/var/ ... ' #bitbucket์—์„œ ๋ณต์ œ
yaml
yaml
apiVersion: v1
kind: List
items:
  - apiVersion: v1
    kind: Secret
    metadata:
      name: ${JOB_NAME}-runner-oauth-credentials
      namespace: ${NS}
      labels:
        accountUuid: ${ACCOUNT_UUID} # Add your account uuid without curly braces to optionally allow finding the secret for an account
        repositoryUuid: ${REPOSITORY_UUID} # Add your repository uuid without curly braces to optionally allow finding the secret for a repository
        runnerUuid: ${RUNNER_UUID} # Add your runner uuid without curly braces to optionally allow finding the secret for a particular runner
    data:
      oauthClientId: $( echo -n ${OAUTH_CLIENT_ID} | base64 -w0 )
      oauthClientSecret: $( echo -n ${OAUTH_CLIENT_SECRET} | base64 -w0 )
  - apiVersion: batch/v1
    kind: Job
    metadata:
      name: ${JOB_NAME}-runner
      namespace: ${NS}
    spec:
      template:
        metadata:
          labels:
            accountUuid: ${ACCOUNT_UUID} # Add your account uuid without curly braces to optionally allow finding the secret for an account
            repositoryUuid: ${REPOSITORY_UUID} # Add your repository uuid without curly braces to optionally allow finding the secret for a repository
            runnerUuid: ${RUNNER_UUID} # Add your runner uuid without curly braces to optionally allow finding the secret for a particular runner
        spec:
          containers:
            - name: runner
              image: docker-public.packages.atlassian.com/sox/atlassian/bitbucket-pipelines-runner
              env:
                - name: ACCOUNT_UUID
                  value: '{${ACCOUNT_UUID}}' # Add your account uuid here
                - name: REPOSITORY_UUID
                  value: '{${REPOSITORY_UUID}}' # Add your repository uuid here
                - name: RUNNER_UUID
                  value: '{${RUNNER_UUID}}' # Add your runner uuid here
                - name: OAUTH_CLIENT_ID
                  valueFrom:
                    secretKeyRef:
                      name: ${JOB_NAME}-runner-oauth-credentials
                      key: oauthClientId
                - name: OAUTH_CLIENT_SECRET
                  valueFrom:
                    secretKeyRef:
                      name: ${JOB_NAME}-runner-oauth-credentials
                      key: oauthClientSecret
                - name: WORKING_DIRECTORY
                  value: "/tmp"
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                  readOnly: true # the runner only needs to read these files never write to them
                - name: var-run
                  mountPath: /var/run
            - name: docker-in-docker
              image: docker:${CRI_VERSION}-dind
              securityContext:
                privileged: true # required to allow docker in docker to run and assumes the namespace your applying this to has a pod security policy that allows privilege escalation
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                - name: var-run
                  mountPath: /var/run
          restartPolicy: OnFailure # this allows the runner to restart locally if it was to crash
          volumes:
            - name: tmp # required to share a working directory between docker in docker and the runner
            - name: docker-containers # required to share the containers directory between docker in docker and the runner
            - name: var-run # required to share the docker socket between docker in docker and the runner
        # backoffLimit: 6 # this is the default and means it will retry upto 6 times if it crashes before it considers itself a failure with an exponential backoff between
        # completions: 1 # this is the default the job should ideally never complete as the runner never shuts down successfully
        # parallelism: 1 # this is the default their should only be one instance of this particular runner

Deployment based

๐Ÿ”— https://teyxt.medium.com/how-to-set-up-a-bitbucket-custom-runner-in-your-kubernetes-cluster-a-step-by-step-guide-e42f1d1f992d
yaml
apiVersion: v1
kind: List
items:
  - apiVersion: v1
    kind: Secret
    metadata:
      name: runner-oauth-credentials
      labels:
        accountUuid: 77eb8axx-xxx-xxx-xxx
        repositoryUuid: 23b695exxx-xxx-xxx
        runnerUuid: c64b921xxx-xxx-xxx
    data:
      oauthClientId: 9Glxxxxxx
      oauthClientSecret: AT0A6uexxxxx
#  - apiVersion: batch/v1
#    kind: Job
  - apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: runner
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: runner
      template:
        metadata:
          labels:
            accountUuid: 77eb8axx-xxx-xxx-xxx
            repositoryUuid: 23b695exxx-xxx-xxx
            runnerUuid: c64b921xxx-xxx-xxx
            app: runner
        spec:
          containers:
            - name: runner
              image: docker-public.packages.atlassian.com/sox/atlassian/bitbucket-pipelines-runner:1
              env:
                - name: ACCOUNT_UUID
                  value: "{77eb8axx-xxx-xxx-xxx}"
                - name: REPOSITORY_UUID
                  value: "{23b695exxx-xxx-xxx}"
                - name: RUNNER_UUID
                  value: "{c64b921xxx-xxx-xxx}"
                - name: OAUTH_CLIENT_ID
                  valueFrom:
                    secretKeyRef:
                      name: runner-oauth-credentials
                      key: oauthClientId
                - name: OAUTH_CLIENT_SECRET
                  valueFrom:
                    secretKeyRef:
                      name: runner-oauth-credentials
                      key: oauthClientSecret
                - name: WORKING_DIRECTORY
                  value: "/tmp"
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                  readOnly: true # the runner only needs to read these files never write to them
                - name: var-run
                  mountPath: /var/run
            - name: docker-in-docker
              image: docker:20.10.7-dind
              securityContext:
                privileged: true # required to allow docker in docker to run and assumes the namespace your applying this to has a pod security policy that allows privilege escalation
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
                - name: docker-containers
                  mountPath: /var/lib/docker/containers
                - name: var-run
                  mountPath: /var/run
          restartPolicy: Always # this allows the runner to restart locally if it was to crash
          volumes:
            - name: tmp # required to share a working directory between docker in docker and the runner
            - name: docker-containers # required to share the containers directory between docker in docker and the runner
            - name: var-run # required to share the docker socket between docker in docker and the runner
        # backoffLimit: 6 # this is the default and means it will retry upto 6 times if it crashes before it considers itself a failure with an exponential backoff between
        # completions: 1 # this is the default the job should ideally never complete as the runner never shuts down successfully
        # parallelism: 1
          nodeSelector:
            dedicated: runner-node

          tolerations:
            - key: "dedicated"
              operator: "Equal"
              value: "runner-node"
              effect: "NoSchedule"

Posted:

/

Category:

Author:

fredric

Tags:

, , , , , , , , , , , , , , , , ,

Comments

Leave a Reply

Your email address will not be published. Required fields are marked *