From af536c0c9676c328b0bbcf18e96ed910e650f33b Mon Sep 17 00:00:00 2001 From: lxm Date: Sat, 10 Apr 2021 00:06:07 +0800 Subject: [PATCH 1/3] [feature] use buildx to produce multi arch image kubesphere#3683 Signed-off-by: lxm --- .dockerignore | 4 +- .github/workflows/build-multiarch.yaml | 48 ++++++++++++++++++++++++ build/Dockerfile | 44 ++++++++++++++++++++++ docs/build-multiarch-images.md | 29 +++++++++++++++ hack/build-scripts/get-etcd.sh | 13 +++++++ hack/build-scripts/get-helm.sh | 17 +++++++++ hack/build-scripts/get-kubernetes.sh | 14 +++++++ hack/build-scripts/get-kustomize.sh | 15 ++++++++ hack/docker_build_multiarch.sh | 51 ++++++++++++++++++++++++++ hack/init_env.sh | 5 +++ hack/install_kubebuilder.sh | 4 +- 11 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-multiarch.yaml create mode 100644 build/Dockerfile create mode 100644 docs/build-multiarch-images.md create mode 100644 hack/build-scripts/get-etcd.sh create mode 100644 hack/build-scripts/get-helm.sh create mode 100644 hack/build-scripts/get-kubernetes.sh create mode 100644 hack/build-scripts/get-kustomize.sh create mode 100644 hack/docker_build_multiarch.sh create mode 100644 hack/init_env.sh diff --git a/.dockerignore b/.dockerignore index 8effd60d7..41d36906c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ # exclude all files and folders except bin folder -** !bin +.idea/ +.vscode/ +.git/ \ No newline at end of file diff --git a/.github/workflows/build-multiarch.yaml b/.github/workflows/build-multiarch.yaml new file mode 100644 index 000000000..bb7d0da64 --- /dev/null +++ b/.github/workflows/build-multiarch.yaml @@ -0,0 +1,48 @@ +name: BuildMultiArch + +on: + push: + branches: + - 'master' + - 'release*' + - '*' + tags: + - 'v*' + pull_request: + branches: + - 'master' + - 'release*' + +jobs: + build: + name: Build + runs-on: ubuntu-latest + env: + GO111MODULE: on + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: "Set up QEMU" + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + - name: "Set up Docker buildx" + uses: "docker/setup-buildx-action@v1" + + - name: Get branch name + id: extract_branch + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + + - name: Build and push docker images + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + # if: github.event_name == 'push' + run: | + echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + bash hack/docker_build_multiarch.sh ${{ steps.extract_branch.outputs.branch }} diff --git a/build/Dockerfile b/build/Dockerfile new file mode 100644 index 000000000..def8870c0 --- /dev/null +++ b/build/Dockerfile @@ -0,0 +1,44 @@ +FROM ubuntu:18.04 as binarydownloader +ARG HELM_VERSION=v3.5.2 +ARG KUSTOMIZE_VERSION=v4.0.5 +ARG KUBE_VERSION=v1.16.4 +ARG ETCD_VERSION=v3.3.11 +RUN apt update && apt -y install wget curl +ADD ./hack /opt/hack/ + +RUN bash /opt/hack/build-scripts/get-helm.sh +RUN bash /opt/hack/build-scripts/get-kustomize.sh +RUN bash /opt/hack/build-scripts/get-etcd.sh +RUN bash /opt/hack/build-scripts/get-kubernetes.sh + + +FROM golang:1.13 as builder +ADD . /opt/kubesphere/ +WORKDIR /opt/kubesphere/ +RUN apt update && apt -y install sudo +RUN bash /opt/kubesphere/hack/install_kubebuilder.sh +COPY --from=binarydownloader /usr/local/kubernetes/server/bin/kube-apiserver /usr/local/kubebuilder/bin/kube-apiserver +COPY --from=binarydownloader /usr/local/kubernetes/server/bin/kubectl /usr/local/kubebuilder/bin/kubectl +COPY --from=binarydownloader /usr/local/bin/etcd /usr/local/kubebuilder/bin/etcd +RUN bash -c "source /opt/kubesphere/hack/init_env.sh && make ks-apiserver" +RUN bash -c "source /opt/kubesphere/hack/init_env.sh && make ks-controller-manager" + + +FROM alpine:3.11 as ks-apiserver +RUN apk add --no-cache ca-certificates +COPY --from=binarydownloader /usr/bin/helm /usr/bin/helm +COPY --from=builder /opt/kubesphere/bin/cmd/ks-apiserver /usr/local/bin/ + +EXPOSE 9090 + +CMD ["sh"] + +FROM alpine:3.11 as ks-controller-manager +RUN apk add --no-cache ca-certificates +COPY --from=binarydownloader /usr/bin/helm /usr/bin/helm +COPY --from=binarydownloader /usr/bin/kustomize /usr/bin/kustomize +COPY --from=builder /opt/kubesphere/bin/cmd/controller-manager /usr/local/bin/ + +EXPOSE 8443 8080 + +CMD ["sh"] \ No newline at end of file diff --git a/docs/build-multiarch-images.md b/docs/build-multiarch-images.md new file mode 100644 index 000000000..d22d87c8e --- /dev/null +++ b/docs/build-multiarch-images.md @@ -0,0 +1,29 @@ +# build mulit-arch image with docker buildx + +## set up env for docker buildx + +run follow commands + +``` +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +docker buildx rm arm64builder +docker buildx create --name arm64builder +docker buildx use arm64builder +docker buildx inspect --bootstrap +``` + +## build images + +If build multi platform one time, can not use `--load` param, You must have privileges to push images to `$REPO` + +``` +docker buildx build --platform="linux/amd64,linux/arm64" . -t $REPO/ks-apiserver:$TAG -f build-multiarch/Dockerfile --target=ks-apiserver --push +docker buildx build --platform="linux/amd64,linux/arm64" . -t $REPO/ks-controller-manager:$TAG -f build-multiarch/Dockerfile --target=ks-controller-manager --push +``` + +Also you can build a image for local build with only one arch + +``` +docker buildx build --platform="linux/arm64" . -t "ks-apiserver:arm64" -f build-multiarch/Dockerfile --target=ks-apiserver --load +docker buildx build --platform="linux/amd64" . -t "ks-apiserver:amd64" -f build-multiarch/Dockerfile --target=ks-apiserver --load +``` \ No newline at end of file diff --git a/hack/build-scripts/get-etcd.sh b/hack/build-scripts/get-etcd.sh new file mode 100644 index 000000000..01806f1dd --- /dev/null +++ b/hack/build-scripts/get-etcd.sh @@ -0,0 +1,13 @@ +#!/bin/bash +ARCH=`uname -m` +if [ "$ARCH" == "x86_64" ]; then + echo "x86_64" + wget https://storage.googleapis.com/etcd/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz + tar xvf etcd-${ETCD_VERSION}-linux-amd64.tar.gz && \ + mv etcd-${ETCD_VERSION}-linux-amd64/etcd /usr/local/bin/etcd +elif [ "$ARCH" == "aarch64" ]; then + echo "arm arch" + wget https://storage.googleapis.com/etcd/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-arm64.tar.gz + tar xvf etcd-${ETCD_VERSION}-linux-arm64.tar.gz && \ + mv etcd-${ETCD_VERSION}-linux-arm64/etcd /usr/local/bin/etcd +fi \ No newline at end of file diff --git a/hack/build-scripts/get-helm.sh b/hack/build-scripts/get-helm.sh new file mode 100644 index 000000000..ae954e7f6 --- /dev/null +++ b/hack/build-scripts/get-helm.sh @@ -0,0 +1,17 @@ +#!/bin/bash +ARCH=`uname -m` +if [ "$ARCH" == "x86_64" ]; then + echo "x86_64" + wget https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz && \ + tar xvf helm-${HELM_VERSION}-linux-amd64.tar.gz && \ + rm helm-${HELM_VERSION}-linux-amd64.tar.gz && \ + mv linux-amd64/helm /usr/bin/ && \ + rm -rf linux-amd64 +elif [ "$ARCH" == "aarch64" ]; then + echo "arm arch" + wget https://get.helm.sh/helm-${HELM_VERSION}-linux-arm64.tar.gz && \ + tar xvf helm-${HELM_VERSION}-linux-arm64.tar.gz && \ + rm helm-${HELM_VERSION}-linux-arm64.tar.gz && \ + mv linux-arm64/helm /usr/bin/ && \ + rm -rf linux-arm64 +fi \ No newline at end of file diff --git a/hack/build-scripts/get-kubernetes.sh b/hack/build-scripts/get-kubernetes.sh new file mode 100644 index 000000000..ca925b3bb --- /dev/null +++ b/hack/build-scripts/get-kubernetes.sh @@ -0,0 +1,14 @@ + +#!/bin/bash +ARCH=`uname -m` +if [ "$ARCH" == "x86_64" ]; then + echo "x86_64" + wget https://dl.k8s.io/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz && \ + tar xvf kubernetes-server-linux-amd64.tar.gz && + mv kubernetes /usr/local/ +elif [ "$ARCH" == "aarch64" ]; then + echo "arm arch" + wget https://dl.k8s.io/${KUBE_VERSION}/kubernetes-server-linux-arm64.tar.gz && \ + tar xvf kubernetes-server-linux-arm64.tar.gz && + mv kubernetes /usr/local/ +fi \ No newline at end of file diff --git a/hack/build-scripts/get-kustomize.sh b/hack/build-scripts/get-kustomize.sh new file mode 100644 index 000000000..b0413e84b --- /dev/null +++ b/hack/build-scripts/get-kustomize.sh @@ -0,0 +1,15 @@ +#!/bin/bash +ARCH=`uname -m` +if [ "$ARCH" == "x86_64" ]; then + echo "x86_64" + wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \ + tar xvf kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \ + rm kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \ + mv kustomize /usr/bin +elif [ "$ARCH" == "aarch64" ]; then + echo "arm arch" + wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_arm64.tar.gz && \ + tar xvf kustomize_${KUSTOMIZE_VERSION}_linux_arm64.tar.gz && \ + rm kustomize_${KUSTOMIZE_VERSION}_linux_arm64.tar.gz && \ + mv kustomize /usr/bin +fi \ No newline at end of file diff --git a/hack/docker_build_multiarch.sh b/hack/docker_build_multiarch.sh new file mode 100644 index 000000000..d617922b9 --- /dev/null +++ b/hack/docker_build_multiarch.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -ex +set -o pipefail +BUILDPLATFORM="linux/amd64,linux/arm64" +tag_for_branch() { + local tag=$1 + if [[ "${tag}" == "" ]]; then + tag=$(git branch --show-current) + tag=${tag/\//-} + fi + + if [[ "${tag}" == "master" ]]; then + tag="latest" + fi + echo ${tag} +} + +get_repo() { + local repo=${REPO} # read from env + repo=${repo:-kubespheredev} + if [[ "$1" != "" ]]; then + repo="$1" + fi + + # set the default value if there's no user defined + if [[ "${repo}" == "" ]]; then + repo="kubespheredev" + fi + echo "$repo" +} + +# push to kubespheredev with default latest tag +TAG=$(tag_for_branch "$1") +REPO=$(get_repo "$2") + +# Push image to dockerhub, need to support multiple push +cat ~/.docker/config.json | grep index.docker.io +if [[ $? != 0 ]]; then + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin +fi + +docker buildx build --platform=${BUILDPLATFORM} \ + -f build/Dockerfile \ + -t $REPO/ks-apiserver-multiarch:$TAG . \ + --target=ks-apiserver --push + +docker buildx build --platform=${BUILDPLATFORM} \ + -f build/Dockerfile \ + -t $REPO/ks-controller-manager-multiarch:$TAG . \ + --target=ks-controller-manager --push \ No newline at end of file diff --git a/hack/init_env.sh b/hack/init_env.sh new file mode 100644 index 000000000..c1cebb6cd --- /dev/null +++ b/hack/init_env.sh @@ -0,0 +1,5 @@ +#!/bin/bash +ARCH=`uname -m` +if [ "$ARCH" == "aarch64" ]; then + export ETCD_UNSUPPORTED_ARCH=arm64 +fi \ No newline at end of file diff --git a/hack/install_kubebuilder.sh b/hack/install_kubebuilder.sh index b3ec57554..5dbc89b50 100755 --- a/hack/install_kubebuilder.sh +++ b/hack/install_kubebuilder.sh @@ -35,8 +35,10 @@ HW=$(uname -m) case $HW in x86_64) ARCH=amd64 ;; + aarch64) + ARCH=arm64 ;; *) - echo "Only x86_64 machines are supported !" + echo "Only x86_64/arm64 machines are supported !" exit 1 ;; esac From 31fbe51327252a967b78410b0b507baf73733a95 Mon Sep 17 00:00:00 2001 From: lxm Date: Thu, 22 Apr 2021 11:34:13 +0800 Subject: [PATCH 2/3] add suffix to image tag Signed-off-by: lxm --- hack/docker_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/docker_build.sh b/hack/docker_build.sh index 404d5120a..d75aec639 100755 --- a/hack/docker_build.sh +++ b/hack/docker_build.sh @@ -40,6 +40,7 @@ if [[ $? != 0 ]]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin fi +TAG=$TAG-multiarch docker build -f build/ks-apiserver/Dockerfile -t $REPO/ks-apiserver:$TAG . docker push $REPO/ks-apiserver:$TAG # print the full docker image path for your convience From ae641d957fec6587d3ea4461f5c7f0db21140218 Mon Sep 17 00:00:00 2001 From: lxm Date: Mon, 26 Apr 2021 18:53:01 +0800 Subject: [PATCH 3/3] only trigger github action on push Signed-off-by: lxm --- .github/workflows/build-multiarch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-multiarch.yaml b/.github/workflows/build-multiarch.yaml index bb7d0da64..0f12c1ac3 100644 --- a/.github/workflows/build-multiarch.yaml +++ b/.github/workflows/build-multiarch.yaml @@ -42,7 +42,7 @@ jobs: env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - # if: github.event_name == 'push' + if: github.event_name == 'push' run: | echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin bash hack/docker_build_multiarch.sh ${{ steps.extract_branch.outputs.branch }}