Merge pull request #3704 from lxm/feature/multiarch

[feature] use buildx to produce multi arch image #3683
2021-04-27 16:15:04 +08:00
parent c3522d940a ae641d957f
commit cad6296f4e
12 changed files with 243 additions and 2 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,3 +1,5 @@
 # exclude all files and folders except bin folder
-**
 !bin
+.idea/
+.vscode/
+.git/
--- a/.github/workflows/build-multiarch.yaml
+++ b/.github/workflows/build-multiarch.yaml
@@ -0,0 +1,48 @@
+name: BuildMultiArch
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'release*'
+      - '*'
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - 'master'
+      - 'release*'
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    env:
+      GO111MODULE: on
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: "Set up QEMU"
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+
+      - name: "Set up Docker buildx"
+        uses: "docker/setup-buildx-action@v1"
+
+      - name: Get branch name
+        id: extract_branch
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+
+      - name: Build and push docker images
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        if: github.event_name == 'push'
+        run: |
+          echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+          bash hack/docker_build_multiarch.sh ${{ steps.extract_branch.outputs.branch }}
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -0,0 +1,44 @@
+FROM ubuntu:18.04 as binarydownloader
+ARG HELM_VERSION=v3.5.2
+ARG KUSTOMIZE_VERSION=v4.0.5
+ARG KUBE_VERSION=v1.16.4
+ARG ETCD_VERSION=v3.3.11
+RUN apt update && apt -y install wget curl
+ADD ./hack /opt/hack/
+
+RUN bash /opt/hack/build-scripts/get-helm.sh
+RUN bash /opt/hack/build-scripts/get-kustomize.sh
+RUN bash /opt/hack/build-scripts/get-etcd.sh
+RUN bash /opt/hack/build-scripts/get-kubernetes.sh
+
+
+FROM golang:1.13 as builder
+ADD . /opt/kubesphere/
+WORKDIR /opt/kubesphere/
+RUN apt update && apt -y install sudo
+RUN bash /opt/kubesphere/hack/install_kubebuilder.sh
+COPY --from=binarydownloader /usr/local/kubernetes/server/bin/kube-apiserver /usr/local/kubebuilder/bin/kube-apiserver
+COPY --from=binarydownloader /usr/local/kubernetes/server/bin/kubectl /usr/local/kubebuilder/bin/kubectl
+COPY --from=binarydownloader /usr/local/bin/etcd /usr/local/kubebuilder/bin/etcd
+RUN bash -c "source /opt/kubesphere/hack/init_env.sh && make ks-apiserver"
+RUN bash -c "source /opt/kubesphere/hack/init_env.sh && make ks-controller-manager"
+
+
+FROM alpine:3.11 as ks-apiserver
+RUN apk add --no-cache ca-certificates
+COPY --from=binarydownloader /usr/bin/helm /usr/bin/helm
+COPY --from=builder /opt/kubesphere/bin/cmd/ks-apiserver /usr/local/bin/
+
+EXPOSE 9090
+
+CMD ["sh"]
+
+FROM alpine:3.11 as ks-controller-manager
+RUN apk add --no-cache ca-certificates
+COPY --from=binarydownloader /usr/bin/helm /usr/bin/helm
+COPY --from=binarydownloader /usr/bin/kustomize /usr/bin/kustomize
+COPY --from=builder /opt/kubesphere/bin/cmd/controller-manager /usr/local/bin/
+
+EXPOSE 8443 8080
+
+CMD ["sh"]
--- a/docs/build-multiarch-images.md
+++ b/docs/build-multiarch-images.md
@@ -0,0 +1,29 @@
+# build mulit-arch image with docker buildx
+
+## set up env for docker buildx
+
+run follow commands
+
+```
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+docker buildx rm arm64builder
+docker buildx create --name arm64builder
+docker buildx use arm64builder
+docker buildx inspect --bootstrap
+```
+
+## build images
+
+If build multi platform one time, can not use `--load` param, You must have privileges to push images to `$REPO` 
+
+```
+docker buildx build --platform="linux/amd64,linux/arm64" . -t $REPO/ks-apiserver:$TAG -f build-multiarch/Dockerfile --target=ks-apiserver --push
+docker buildx build --platform="linux/amd64,linux/arm64" . -t $REPO/ks-controller-manager:$TAG -f build-multiarch/Dockerfile --target=ks-controller-manager --push
+```
+
+Also you can build a image for local build with only one arch
+
+```
+docker buildx build --platform="linux/arm64" . -t "ks-apiserver:arm64" -f build-multiarch/Dockerfile --target=ks-apiserver --load
+docker buildx build --platform="linux/amd64" . -t "ks-apiserver:amd64" -f build-multiarch/Dockerfile --target=ks-apiserver --load
+```
--- a/hack/build-scripts/get-etcd.sh
+++ b/hack/build-scripts/get-etcd.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+ARCH=`uname -m`
+if [ "$ARCH" == "x86_64" ]; then
+   echo "x86_64"
+   wget https://storage.googleapis.com/etcd/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz
+   tar xvf etcd-${ETCD_VERSION}-linux-amd64.tar.gz && \
+   mv etcd-${ETCD_VERSION}-linux-amd64/etcd /usr/local/bin/etcd
+elif [ "$ARCH" == "aarch64" ]; then
+   echo "arm arch"
+   wget https://storage.googleapis.com/etcd/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-arm64.tar.gz
+   tar xvf etcd-${ETCD_VERSION}-linux-arm64.tar.gz && \
+   mv etcd-${ETCD_VERSION}-linux-arm64/etcd /usr/local/bin/etcd
+fi
--- a/hack/build-scripts/get-helm.sh
+++ b/hack/build-scripts/get-helm.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+ARCH=`uname -m`
+if [ "$ARCH" == "x86_64" ]; then
+   echo "x86_64"
+   wget https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz && \
+   tar xvf helm-${HELM_VERSION}-linux-amd64.tar.gz && \
+   rm helm-${HELM_VERSION}-linux-amd64.tar.gz && \
+   mv linux-amd64/helm /usr/bin/ && \
+   rm -rf linux-amd64
+elif [ "$ARCH" == "aarch64" ]; then
+   echo "arm arch"
+   wget https://get.helm.sh/helm-${HELM_VERSION}-linux-arm64.tar.gz && \
+   tar xvf helm-${HELM_VERSION}-linux-arm64.tar.gz && \
+   rm helm-${HELM_VERSION}-linux-arm64.tar.gz && \
+   mv linux-arm64/helm /usr/bin/ && \
+   rm -rf linux-arm64
+fi
--- a/hack/build-scripts/get-kubernetes.sh
+++ b/hack/build-scripts/get-kubernetes.sh
@@ -0,0 +1,14 @@
+
+#!/bin/bash
+ARCH=`uname -m`
+if [ "$ARCH" == "x86_64" ]; then
+   echo "x86_64"
+   wget https://dl.k8s.io/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz && \
+   tar xvf kubernetes-server-linux-amd64.tar.gz &&
+   mv kubernetes /usr/local/
+elif [ "$ARCH" == "aarch64" ]; then
+   echo "arm arch"
+   wget https://dl.k8s.io/${KUBE_VERSION}/kubernetes-server-linux-arm64.tar.gz && \
+   tar xvf kubernetes-server-linux-arm64.tar.gz &&
+   mv kubernetes /usr/local/
+fi
--- a/hack/build-scripts/get-kustomize.sh
+++ b/hack/build-scripts/get-kustomize.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+ARCH=`uname -m`
+if [ "$ARCH" == "x86_64" ]; then
+   echo "x86_64"
+   wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \
+   tar xvf kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \
+   rm kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \
+   mv kustomize /usr/bin
+elif [ "$ARCH" == "aarch64" ]; then
+   echo "arm arch"
+   wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_arm64.tar.gz && \
+   tar xvf kustomize_${KUSTOMIZE_VERSION}_linux_arm64.tar.gz && \
+   rm kustomize_${KUSTOMIZE_VERSION}_linux_arm64.tar.gz && \
+   mv kustomize /usr/bin
+fi
--- a/hack/docker_build.sh
+++ b/hack/docker_build.sh
@@ -40,6 +40,7 @@ if [[ $? != 0 ]]; then
  echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
 fi

+TAG=$TAG-multiarch
 docker build -f build/ks-apiserver/Dockerfile -t $REPO/ks-apiserver:$TAG .
 docker push $REPO/ks-apiserver:$TAG
 # print the full docker image path for your convience
--- a/hack/docker_build_multiarch.sh
+++ b/hack/docker_build_multiarch.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -ex
+set -o pipefail
+BUILDPLATFORM="linux/amd64,linux/arm64"
+tag_for_branch() {
+    local tag=$1
+    if [[ "${tag}" == "" ]]; then
+        tag=$(git branch --show-current)
+        tag=${tag/\//-}
+    fi
+
+    if [[ "${tag}" == "master" ]]; then
+        tag="latest"
+    fi
+    echo ${tag}
+}
+
+get_repo() {
+    local repo=${REPO} # read from env
+    repo=${repo:-kubespheredev}
+    if [[ "$1" != "" ]]; then
+      repo="$1"
+    fi
+
+    # set the default value if there's no user defined
+    if [[ "${repo}" == "" ]]; then
+      repo="kubespheredev"
+    fi
+    echo "$repo"
+}
+
+# push to kubespheredev with default latest tag
+TAG=$(tag_for_branch "$1")
+REPO=$(get_repo "$2")
+
+# Push image to dockerhub, need to support multiple push
+cat ~/.docker/config.json | grep index.docker.io
+if [[ $? != 0 ]]; then
+  echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+fi
+
+docker buildx build --platform=${BUILDPLATFORM} \
+                    -f build/Dockerfile \
+                    -t $REPO/ks-apiserver-multiarch:$TAG . \
+                    --target=ks-apiserver --push
+
+docker buildx build --platform=${BUILDPLATFORM} \
+                    -f build/Dockerfile \
+                    -t $REPO/ks-controller-manager-multiarch:$TAG . \
+                    --target=ks-controller-manager --push
--- a/hack/init_env.sh
+++ b/hack/init_env.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+ARCH=`uname -m`
+if [ "$ARCH" == "aarch64" ]; then
+  export ETCD_UNSUPPORTED_ARCH=arm64
+fi
--- a/hack/install_kubebuilder.sh
+++ b/hack/install_kubebuilder.sh
@@ -35,8 +35,10 @@ HW=$(uname -m)
 case $HW in
    x86_64)
      ARCH=amd64 ;;
+    aarch64)
+      ARCH=arm64 ;;
    *)
-      echo "Only x86_64 machines are supported !"
+      echo "Only x86_64/arm64 machines are supported !"
      exit 1
      ;;
 esac