admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: update build and dependency management scripts (#6513)

Browse Source 
Signed-off-by: hongming <coder.scala@gmail.com>
This commit is contained in:
hongming
2025-05-27 16:03:39 +08:00
committed by GitHub
parent 0b5bce2757
commit 281b2091a5
2140 changed files with 113942 additions and 106023 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hack/boilerplate.go.txt
View File

@@ -1,5 +1,2 @@
/*
* Copyright 2024 the KubeSphere Authors.
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
// Copyright 2025 The KubeSphere Authors.
// SPDX-License-Identifier: LicenseRef-KubeSphere-Open-Source

935
hack/lib/golang.sh
View File

@@ -1,10 +1,475 @@
#!/usr/bin/env bash
# This is a modified version of Kubernetes
KUBE_GO_PACKAGE=kubesphere.io/kubesphere
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# shellcheck disable=SC2034 # Variables sourced in other scripts.
readonly KUBE_GOPATH="${KUBE_GOPATH:-"${KUBE_OUTPUT}/go"}"
export KUBE_GOPATH
# The server platform we are building on.
readonly KUBE_SUPPORTED_SERVER_PLATFORMS=(
linux/amd64
linux/arm64
)
# The node platforms we build for
readonly KUBE_SUPPORTED_NODE_PLATFORMS=(
linux/amd64
linux/arm64
)
# If we update this we should also update the set of platforms whose standard
# library is precompiled for in build/build-image/cross/Dockerfile
readonly KUBE_SUPPORTED_CLIENT_PLATFORMS=(
linux/amd64
linux/arm64
)
# Which platforms we should compile test targets for.
# Not all client platforms need these tests
readonly KUBE_SUPPORTED_TEST_PLATFORMS=(
linux/amd64
linux/arm64
)
# The set of server targets that we are only building for Linux
kube::golang::server_targets() {
local targets=()
echo "${targets[@]}"
}
IFS=" " read -ra KUBE_SERVER_TARGETS <<< "$(kube::golang::server_targets)"
readonly KUBE_SERVER_TARGETS
readonly KUBE_SERVER_BINARIES=("${KUBE_SERVER_TARGETS[@]##*/}")
# The set of server targets we build docker images for
kube::golang::server_image_targets() {
# NOTE: this contains cmd targets for kube::build::get_docker_wrapped_binaries
local targets=()
echo "${targets[@]}"
}
IFS=" " read -ra KUBE_SERVER_IMAGE_TARGETS <<< "$(kube::golang::server_image_targets)"
readonly KUBE_SERVER_IMAGE_TARGETS
readonly KUBE_SERVER_IMAGE_BINARIES=("${KUBE_SERVER_IMAGE_TARGETS[@]##*/}")
# The set of conformance targets we build docker image for
kube::golang::conformance_image_targets() {
# NOTE: this contains cmd targets for kube::release::build_conformance_image
local targets=()
echo "${targets[@]}"
}
IFS=" " read -ra KUBE_CONFORMANCE_IMAGE_TARGETS <<< "$(kube::golang::conformance_image_targets)"
readonly KUBE_CONFORMANCE_IMAGE_TARGETS
# The set of server targets that we are only building for Kubernetes nodes
kube::golang::node_targets() {
local targets=()
echo "${targets[@]}"
}
IFS=" " read -ra KUBE_NODE_TARGETS <<< "$(kube::golang::node_targets)"
readonly KUBE_NODE_TARGETS
readonly KUBE_NODE_BINARIES=("${KUBE_NODE_TARGETS[@]##*/}")
readonly KUBE_NODE_BINARIES_WIN=("${KUBE_NODE_BINARIES[@]/%/.exe}")
# ------------
# NOTE: All functions that return lists should use newlines.
# bash functions can't return arrays, and spaces are tricky, so newline
# separators are the preferred pattern.
# To transform a string of newline-separated items to an array, use kube::util::read-array:
# kube::util::read-array FOO < <(kube::golang::dups a b c a)
#
# ALWAYS remember to quote your subshells. Not doing so will break in
# bash 4.3, and potentially cause other issues.
# ------------
# Returns a sorted newline-separated list containing only duplicated items.
kube::golang::dups() {
# We use printf to insert newlines, which are required by sort.
printf "%s\n" "$@" | sort | uniq -d
}
# Returns a sorted newline-separated list with duplicated items removed.
kube::golang::dedup() {
# We use printf to insert newlines, which are required by sort.
printf "%s\n" "$@" | sort -u
}
# Depends on values of user-facing KUBE_BUILD_PLATFORMS, KUBE_FASTBUILD,
# and KUBE_BUILDER_OS.
# Configures KUBE_SERVER_PLATFORMS, KUBE_NODE_PLATFOMRS,
# KUBE_TEST_PLATFORMS, and KUBE_CLIENT_PLATFORMS, then sets them
# to readonly.
# The configured vars will only contain platforms allowed by the
# KUBE_SUPPORTED* vars at the top of this file.
declare -a KUBE_SERVER_PLATFORMS
declare -a KUBE_CLIENT_PLATFORMS
declare -a KUBE_NODE_PLATFORMS
declare -a KUBE_TEST_PLATFORMS
kube::golang::setup_platforms() {
if [[ -n "${KUBE_BUILD_PLATFORMS:-}" ]]; then
# KUBE_BUILD_PLATFORMS needs to be read into an array before the next
# step, or quoting treats it all as one element.
local -a platforms
IFS=" " read -ra platforms <<< "${KUBE_BUILD_PLATFORMS}"
# Deduplicate to ensure the intersection trick with kube::golang::dups
# is not defeated by duplicates in user input.
kube::util::read-array platforms < <(kube::golang::dedup "${platforms[@]}")
# Use kube::golang::dups to restrict the builds to the platforms in
# KUBE_SUPPORTED_*_PLATFORMS. Items should only appear at most once in each
# set, so if they appear twice after the merge they are in the intersection.
kube::util::read-array KUBE_SERVER_PLATFORMS < <(kube::golang::dups \
"${platforms[@]}" \
"${KUBE_SUPPORTED_SERVER_PLATFORMS[@]}" \
)
readonly KUBE_SERVER_PLATFORMS
kube::util::read-array KUBE_NODE_PLATFORMS < <(kube::golang::dups \
"${platforms[@]}" \
"${KUBE_SUPPORTED_NODE_PLATFORMS[@]}" \
)
readonly KUBE_NODE_PLATFORMS
kube::util::read-array KUBE_TEST_PLATFORMS < <(kube::golang::dups \
"${platforms[@]}" \
"${KUBE_SUPPORTED_TEST_PLATFORMS[@]}" \
)
readonly KUBE_TEST_PLATFORMS
kube::util::read-array KUBE_CLIENT_PLATFORMS < <(kube::golang::dups \
"${platforms[@]}" \
"${KUBE_SUPPORTED_CLIENT_PLATFORMS[@]}" \
)
readonly KUBE_CLIENT_PLATFORMS
elif [[ "${KUBE_FASTBUILD:-}" == "true" ]]; then
host_arch=$(kube::util::host_arch)
if [[ "${host_arch}" != "amd64" && "${host_arch}" != "arm64" && "${host_arch}" != "ppc64le" && "${host_arch}" != "s390x" ]]; then
# on any platform other than amd64, arm64, ppc64le and s390x, we just default to amd64
host_arch="amd64"
fi
KUBE_SERVER_PLATFORMS=("linux/${host_arch}")
readonly KUBE_SERVER_PLATFORMS
KUBE_NODE_PLATFORMS=("linux/${host_arch}")
readonly KUBE_NODE_PLATFORMS
if [[ "${KUBE_BUILDER_OS:-}" == "darwin"* ]]; then
KUBE_TEST_PLATFORMS=(
"darwin/${host_arch}"
"linux/${host_arch}"
)
readonly KUBE_TEST_PLATFORMS
KUBE_CLIENT_PLATFORMS=(
"darwin/${host_arch}"
"linux/${host_arch}"
)
readonly KUBE_CLIENT_PLATFORMS
else
KUBE_TEST_PLATFORMS=("linux/${host_arch}")
readonly KUBE_TEST_PLATFORMS
KUBE_CLIENT_PLATFORMS=("linux/${host_arch}")
readonly KUBE_CLIENT_PLATFORMS
fi
else
KUBE_SERVER_PLATFORMS=("${KUBE_SUPPORTED_SERVER_PLATFORMS[@]}")
readonly KUBE_SERVER_PLATFORMS
KUBE_NODE_PLATFORMS=("${KUBE_SUPPORTED_NODE_PLATFORMS[@]}")
readonly KUBE_NODE_PLATFORMS
KUBE_CLIENT_PLATFORMS=("${KUBE_SUPPORTED_CLIENT_PLATFORMS[@]}")
readonly KUBE_CLIENT_PLATFORMS
KUBE_TEST_PLATFORMS=("${KUBE_SUPPORTED_TEST_PLATFORMS[@]}")
readonly KUBE_TEST_PLATFORMS
fi
}
kube::golang::setup_platforms
# The set of client targets that we are building for all platforms
readonly KUBE_CLIENT_TARGETS=(
cmd/kubectl
cmd/kubectl-convert
)
readonly KUBE_CLIENT_BINARIES=("${KUBE_CLIENT_TARGETS[@]##*/}")
readonly KUBE_CLIENT_BINARIES_WIN=("${KUBE_CLIENT_BINARIES[@]/%/.exe}")
# The set of test targets that we are building for all platforms
kube::golang::test_targets() {
local targets=()
echo "${targets[@]}"
}
IFS=" " read -ra KUBE_TEST_TARGETS <<< "$(kube::golang::test_targets)"
readonly KUBE_TEST_TARGETS
readonly KUBE_TEST_BINARIES=("${KUBE_TEST_TARGETS[@]##*/}")
readonly KUBE_TEST_BINARIES_WIN=("${KUBE_TEST_BINARIES[@]/%/.exe}")
readonly KUBE_TEST_PORTABLE=(
test/e2e/testing-manifests
test/kubemark
hack/e2e-internal
hack/get-build.sh
hack/ginkgo-e2e.sh
hack/lib
)
# Test targets which run on the Kubernetes clusters directly, so we only
# need to target server platforms.
# These binaries will be distributed in the kubernetes-test tarball.
kube::golang::server_test_targets() {
local targets=(
cmd/kubemark
ginkgo
)
if [[ "${OSTYPE:-}" == "linux"* ]]; then
targets+=( test/e2e_node/e2e_node.test )
fi
echo "${targets[@]}"
}
IFS=" " read -ra KUBE_TEST_SERVER_TARGETS <<< "$(kube::golang::server_test_targets)"
readonly KUBE_TEST_SERVER_TARGETS
readonly KUBE_TEST_SERVER_BINARIES=("${KUBE_TEST_SERVER_TARGETS[@]##*/}")
readonly KUBE_TEST_SERVER_PLATFORMS=("${KUBE_SERVER_PLATFORMS[@]:+"${KUBE_SERVER_PLATFORMS[@]}"}")
# Gigabytes necessary for parallel platform builds.
# As of March 2021 (go 1.16/amd64), the RSS usage is 2GiB by using cached
# memory of 15GiB.
# This variable can be overwritten at your own risk.
# It's defaulting to 20G to provide some headroom.
readonly KUBE_PARALLEL_BUILD_MEMORY=${KUBE_PARALLEL_BUILD_MEMORY:-20}
readonly KUBE_ALL_TARGETS=(
"${KUBE_SERVER_TARGETS[@]}"
"${KUBE_CLIENT_TARGETS[@]}"
"${KUBE_TEST_TARGETS[@]}"
"${KUBE_TEST_SERVER_TARGETS[@]}"
)
readonly KUBE_ALL_BINARIES=("${KUBE_ALL_TARGETS[@]##*/}")
readonly KUBE_STATIC_BINARIES=(
ks-apiserver
ks-controller-manager
)
# Fully-qualified package names that we want to instrument for coverage information.
readonly KUBE_COVERAGE_INSTRUMENTED_PACKAGES=()
# KUBE_CGO_OVERRIDES is a space-separated list of binaries which should be built
# with CGO enabled, assuming CGO is supported on the target platform.
# This overrides any entry in KUBE_STATIC_BINARIES.
IFS=" " read -ra KUBE_CGO_OVERRIDES_LIST <<< "${KUBE_CGO_OVERRIDES:-}"
readonly KUBE_CGO_OVERRIDES_LIST
# KUBE_STATIC_OVERRIDES is a space-separated list of binaries which should be
# built with CGO disabled. This is in addition to the list in
# KUBE_STATIC_BINARIES.
IFS=" " read -ra KUBE_STATIC_OVERRIDES_LIST <<< "${KUBE_STATIC_OVERRIDES:-}"
readonly KUBE_STATIC_OVERRIDES_LIST
kube::golang::is_statically_linked() {
local e
if [[ -n "${KUBE_CGO_OVERRIDES_LIST:+x}" ]]; then
for e in "${KUBE_CGO_OVERRIDES_LIST[@]}"; do [[ "${1}" == *"/${e}" ]] && return 1; done;
fi
for e in "${KUBE_STATIC_BINARIES[@]}"; do [[ "${1}" == *"/${e}" ]] && return 0; done;
if [[ -n "${KUBE_STATIC_OVERRIDES_LIST:+x}" ]]; then
for e in "${KUBE_STATIC_OVERRIDES_LIST[@]}"; do [[ "${1}" == *"/${e}" ]] && return 0; done;
fi
return 1;
}
# kube::golang::best_guess_go_targets takes a list of build targets, which might
# be Go-style names (e.g. example.com/foo/bar or ./foo/bar) or just local paths
# (e.g. foo/bar) and produces a respective list (on stdout) of our best guess at
# Go target names.
kube::golang::best_guess_go_targets() {
local target
for target; do
if [ "${target}" = "ginkgo" ] ||
[ "${target}" = "github.com/onsi/ginkgo/ginkgo" ] ||
[ "${target}" = "vendor/github.com/onsi/ginkgo/ginkgo" ]; then
# Aliases that build the ginkgo CLI for hack/ginkgo-e2e.sh.
# "ginkgo" is the one that is documented in the Makefile. The others
# are for backwards compatibility.
echo "github.com/onsi/ginkgo/v2/ginkgo"
continue
fi
if [[ "${target}" =~ ^([[:alnum:]]+".")+[[:alnum:]]+"/".+ ]]; then
# If the target starts with what looks like a domain name, assume it has a
# fully-qualified Go package name.
echo "${target}"
continue
fi
if [[ "${target}" =~ ^vendor/ ]]; then
# Strip vendor/ prefix, since we're building in gomodule mode. This is
# for backwards compatibility.
echo "${target#"vendor/"}"
continue
fi
# If the target starts with "./", assume it is a local path which qualifies
# as a Go target name.
if [[ "${target}" =~ ^\./ ]]; then
echo "${target}"
continue
fi
# Otherwise assume it's a relative path (e.g. foo/bar or foo/bar/bar.test).
# We probably SHOULDN'T accept this, but we did in the past and it would be
# rude to break things if we don't NEED to. We can't really test if it
# exists or not, because the last element might be an output file (e.g.
# bar.test) or even "...".
echo "./${target}"
done
}
kube::golang::internal::lazy_normalize() {
target="$1"
if [[ "${target}" =~ ^([[:alnum:]]+".")+[[:alnum:]]+"/".+ ]]; then
# If the target starts with what looks like a domain name, assume it has a
# fully-qualified Go package name.
echo "${target}"
return
fi
go list -find -e "${target}"
}
# kube::golang::normalize_go_targets takes a list of build targets, which might
# be Go-style names (e.g. example.com/foo/bar or ./foo/bar) or just local paths
# (e.g. foo/bar) and produces a respective list (on stdout) of Go package
# names.
#
# If this cannot find (go list -find -e) one or more inputs, it will emit the
# them on stdout, so callers can at least get a useful error.
kube::golang::normalize_go_targets() {
local targets=()
kube::util::read-array targets < <(kube::golang::best_guess_go_targets "$@")
kube::util::read-array targets < <(kube::golang::dedup "${targets[@]}")
set -- "${targets[@]}"
for target; do
if [[ "${target}" =~ ".test"$ ]]; then
local dir
dir="$(dirname "${target}")"
local tst
tst="$(basename "${target}")"
local pkg
pkg="$(kube::golang::internal::lazy_normalize "${dir}")"
echo "${pkg}/${tst}"
continue
fi
if [[ "${target}" =~ "/..."$ ]]; then
local dir
dir="$(dirname "${target}")"
local pkg
pkg="$(kube::golang::internal::lazy_normalize "${dir}")"
echo "${pkg}/..."
continue
fi
kube::golang::internal::lazy_normalize "${target}"
done
}
# Asks golang what it thinks the host platform is. The go tool chain does some
# slightly different things when the target platform matches the host platform.
kube::golang::host_platform() {
echo "$(go env GOHOSTOS)/$(go env GOHOSTARCH)"
}
# Takes the platform name ($1) and sets the appropriate golang env variables
# for that platform.
kube::golang::set_platform_envs() {
[[ -n ${1-} ]] || {
kube::log::error_exit "!!! Internal error. No platform set in kube::golang::set_platform_envs"
}
export GOOS=${platform%/*}
export GOARCH=${platform##*/}
# Do not set CC when building natively on a platform, only if cross-compiling
if [[ $(kube::golang::host_platform) != "$platform" ]]; then
# Dynamic CGO linking for other server architectures than host architecture goes here
# If you want to include support for more server platforms than these, add arch-specific gcc names here
case "${platform}" in
"linux/amd64")
export CGO_ENABLED=1
export CC=${KUBE_LINUX_AMD64_CC:-x86_64-linux-gnu-gcc}
;;
"linux/arm")
export CGO_ENABLED=1
export CC=${KUBE_LINUX_ARM_CC:-arm-linux-gnueabihf-gcc}
;;
"linux/arm64")
export CGO_ENABLED=1
export CC=${KUBE_LINUX_ARM64_CC:-aarch64-linux-gnu-gcc}
;;
"linux/ppc64le")
export CGO_ENABLED=1
export CC=${KUBE_LINUX_PPC64LE_CC:-powerpc64le-linux-gnu-gcc}
;;
"linux/s390x")
export CGO_ENABLED=1
export CC=${KUBE_LINUX_S390X_CC:-s390x-linux-gnu-gcc}
;;
esac
fi
# if CC is defined for platform then always enable it
ccenv=$(echo "$platform" | awk -F/ '{print "KUBE_" toupper($1) "_" toupper($2) "_CC"}')
if [ -n "${!ccenv-}" ]; then
export CGO_ENABLED=1
export CC="${!ccenv}"
fi
}
# Ensure the go tool exists and is a viable version.
kube::golang::verify_go_version() {
# Inputs:
# env-var GO_VERSION is the desired go version to use, downloading it if needed (defaults to content of .go-version)
# env-var FORCE_HOST_GO set to a non-empty value uses the go version in the $PATH and skips ensuring $GO_VERSION is used
kube::golang::internal::verify_go_version() {
# default GO_VERSION to content of .go-version
GO_VERSION="${GO_VERSION:-"$(cat "${KUBE_ROOT}/.go-version")"}"
if [ "${GOTOOLCHAIN:-auto}" != 'auto' ]; then
# no-op, just respect GOTOOLCHAIN
:
elif [ -n "${FORCE_HOST_GO:-}" ]; then
# ensure existing host version is used, like before GOTOOLCHAIN existed
export GOTOOLCHAIN='local'
else
# otherwise, we want to ensure the go version matches GO_VERSION
GOTOOLCHAIN="go${GO_VERSION}"
export GOTOOLCHAIN
# if go is either not installed or too old to respect GOTOOLCHAIN then use gimme
if ! (command -v go >/dev/null && [ "$(go version | cut -d' ' -f3)" = "${GOTOOLCHAIN}" ]); then
export GIMME_ENV_PREFIX=${GIMME_ENV_PREFIX:-"${KUBE_OUTPUT}/.gimme/envs"}
export GIMME_VERSION_PREFIX=${GIMME_VERSION_PREFIX:-"${KUBE_OUTPUT}/.gimme/versions"}
# eval because the output of this is shell to set PATH etc.
eval "$("${KUBE_ROOT}/third_party/gimme/gimme" "${GO_VERSION}")"
fi
fi
if [[ -z "$(command -v go)" ]]; then
kube::log::usage_from_stdin <<EOF
Can't find 'go' in PATH, please fix and retry.
@@ -14,9 +479,9 @@ EOF
fi
local go_version
IFS=" " read -ra go_version <<< "$(go version)"
IFS=" " read -ra go_version <<< "$(GOFLAGS='' go version)"
local minimum_go_version
minimum_go_version=go1.20
minimum_go_version=go1.23
if [[ "${minimum_go_version}" != $(echo -e "${minimum_go_version}\n${go_version[2]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "${go_version[2]}" != "devel" ]]; then
kube::log::usage_from_stdin <<EOF
Detected go version: ${go_version[*]}.
@@ -27,38 +492,440 @@ EOF
fi
}
# Prints the value that needs to be passed to the -ldflags parameter of go build
# in order to set the Kubernetes based on the git tree status.
# IMPORTANT: if you update any of these, also update the lists in
# pkg/version/def.bzl and hack/print-workspace-status.sh.
kube::version::ldflags() {
kube::version::get_version_vars
# kube::golang::setup_env will check that the `go` commands is available in
# ${PATH}. It will also check that the Go version is good enough for the
# Kubernetes build.
#
# Outputs:
# env-var GOPATH points to our local output dir
# env-var GOBIN is unset (we want binaries in a predictable place)
# env-var PATH includes the local GOPATH
kube::golang::setup_env() {
# Even in module mode, we need to set GOPATH for `go build` and `go install`
# to work. We build various tools (usually via `go install`) from a lot of
# scripts.
# * We can't just set GOBIN because that does not work on cross-compiles.
# * We could always use `go build -o <something>`, but it's subtle wrt
# cross-compiles and whether the <something> is a file or a directory,
# and EVERY caller has to get it *just* right.
# * We could leave GOPATH alone and let `go install` write binaries
# wherever the user's GOPATH says (or doesn't say).
#
# Instead we set it to a phony local path and process the results ourselves.
# In particular, GOPATH[0]/bin will be used for `go install`, with
# cross-compiles adding an extra directory under that.
export GOPATH="${KUBE_GOPATH}"
local -a ldflags
function add_ldflag() {
local key=${1}
local val=${2}
# If you update these, also update the list component-base/version/def.bzl.
ldflags+=(
"-X '${KUBE_GO_PACKAGE}/pkg/version.${key}=${val}'"
)
}
# If these are not set, set them now. This ensures that any subsequent
# scripts we run (which may call this function again) use the same values.
export GOCACHE="${GOCACHE:-"${KUBE_GOPATH}/cache/build"}"
export GOMODCACHE="${GOMODCACHE:-"${KUBE_GOPATH}/cache/mod"}"
add_ldflag "buildDate" "$(date ${SOURCE_DATE_EPOCH:+"--date=@${SOURCE_DATE_EPOCH}"} -u +'%Y-%m-%dT%H:%M:%SZ')"
if [[ -n ${KUBE_GIT_COMMIT-} ]]; then
add_ldflag "gitCommit" "${KUBE_GIT_COMMIT}"
add_ldflag "gitTreeState" "${KUBE_GIT_TREE_STATE}"
fi
# Make sure our own Go binaries are in PATH.
export PATH="${KUBE_GOPATH}/bin:${PATH}"
if [[ -n ${KUBE_GIT_VERSION-} ]]; then
add_ldflag "gitVersion" "${KUBE_GIT_VERSION}"
fi
# Unset GOBIN in case it already exists in the current session.
# Cross-compiles will not work with it set.
unset GOBIN
if [[ -n ${KUBE_GIT_MAJOR-} && -n ${KUBE_GIT_MINOR-} ]]; then
add_ldflag "gitMajor" "${KUBE_GIT_MAJOR}"
add_ldflag "gitMinor" "${KUBE_GIT_MINOR}"
fi
# Turn on modules and workspaces (both are default-on).
unset GO111MODULE
unset GOWORK
# The -ldflags parameter takes a single string, so join the output.
echo "${ldflags[*]-}"
# This may try to download our specific Go version. Do it last so it uses
# the above-configured environment.
kube::golang::internal::verify_go_version
}
# kube::golang::hack_tools_gotoolchain outputs the value to use for $GOTOOLCHAIN,
# using $KUBE_HACK_TOOLS_GOTOOLCHAIN if set, falling back to $GOTOOLCHAIN if set,
# or outputting the empty string.
#
# Use this when installing / building tools specified in the hack/tools module:
# GOTOOLCHAIN="$(kube::golang::hack_tools_gotoolchain)" go install ...
kube::golang::hack_tools_gotoolchain() {
local hack_tools_gotoolchain="${GOTOOLCHAIN:-}"
if [ -n "${KUBE_HACK_TOOLS_GOTOOLCHAIN:-}" ]; then
hack_tools_gotoolchain="${KUBE_HACK_TOOLS_GOTOOLCHAIN}";
fi
echo -n "${hack_tools_gotoolchain}"
}
kube::golang::setup_gomaxprocs() {
# GOMAXPROCS by default does not reflect the number of cpu(s) available
# when running in a container, please see https://github.com/golang/go/issues/33803
if [[ -z "${GOMAXPROCS:-}" ]]; then
if ! command -v ncpu >/dev/null 2>&1; then
GOTOOLCHAIN="$(kube::golang::hack_tools_gotoolchain)" go -C "${KUBE_ROOT}/hack/tools" install ./ncpu || echo "Will not automatically set GOMAXPROCS"
fi
if command -v ncpu >/dev/null 2>&1; then
GOMAXPROCS=$(ncpu)
export GOMAXPROCS
kube::log::status "Set GOMAXPROCS automatically to ${GOMAXPROCS}"
fi
fi
}
# This will take binaries from $GOPATH/bin and copy them to the appropriate
# place in ${KUBE_OUTPUT_BIN}
#
# Ideally this wouldn't be necessary and we could just set GOBIN to
# KUBE_OUTPUT_BIN but that won't work in the face of cross compilation. 'go
# install' will place binaries that match the host platform directly in $GOBIN
# while placing cross compiled binaries into `platform_arch` subdirs. This
# complicates pretty much everything else we do around packaging and such.
kube::golang::place_bins() {
local host_platform
host_platform=$(kube::golang::host_platform)
V=2 kube::log::status "Placing binaries"
local platform
for platform in "${KUBE_CLIENT_PLATFORMS[@]}"; do
# The substitution on platform_src below will replace all slashes with
# underscores. It'll transform darwin/amd64 -> darwin_amd64.
local platform_src="/${platform//\//_}"
if [[ "${platform}" == "${host_platform}" ]]; then
platform_src=""
rm -f "${THIS_PLATFORM_BIN}"
mkdir -p "$(dirname "${THIS_PLATFORM_BIN}")"
ln -s "${KUBE_OUTPUT_BIN}/${platform}" "${THIS_PLATFORM_BIN}"
fi
V=3 kube::log::status "Placing binaries for ${platform} in ${KUBE_OUTPUT_BIN}/${platform}"
local full_binpath_src="${KUBE_GOPATH}/bin${platform_src}"
if [[ -d "${full_binpath_src}" ]]; then
mkdir -p "${KUBE_OUTPUT_BIN}/${platform}"
find "${full_binpath_src}" -maxdepth 1 -type f -exec \
rsync -pc {} "${KUBE_OUTPUT_BIN}/${platform}" \;
fi
done
}
# Try and replicate the native binary placement of go install without
# calling go install.
kube::golang::outfile_for_binary() {
local binary=$1
local platform=$2
local output_path="${KUBE_GOPATH}/bin"
local bin
bin=$(basename "${binary}")
if [[ "${platform}" != "${host_platform}" ]]; then
output_path="${output_path}/${platform//\//_}"
fi
if [[ ${GOOS} == "windows" ]]; then
bin="${bin}.exe"
fi
echo "${output_path}/${bin}"
}
# Argument: the name of a Kubernetes package.
# Returns 0 if the binary can be built with coverage, 1 otherwise.
# NB: this ignores whether coverage is globally enabled or not.
kube::golang::is_instrumented_package() {
if kube::util::array_contains "$1" "${KUBE_COVERAGE_INSTRUMENTED_PACKAGES[@]}"; then
return 0
fi
# Some cases, like `make kubectl`, pass $1 as "./cmd/kubectl" rather than
# "k8s.io/kubernetes/kubectl". Try to normalize and handle that. We don't
# do this always because it is a bit slow.
pkg=$(go list -find "$1")
if kube::util::array_contains "${pkg}" "${KUBE_COVERAGE_INSTRUMENTED_PACKAGES[@]}"; then
return 0
fi
return 1
}
# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler)
# Echos the path to a dummy test used for coverage information.
kube::golang::path_for_coverage_dummy_test() {
local package="$1"
local path
path=$(go list -find -f '{{.Dir}}' "${package}")
local name
name=$(basename "${package}")
echo "${path}/zz_generated_${name}_test.go"
}
# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler).
# Creates a dummy unit test on disk in the source directory for the given package.
# This unit test will invoke the package's standard entry point when run.
kube::golang::create_coverage_dummy_test() {
local package="$1"
local name
name="$(basename "${package}")"
cat <<EOF > "$(kube::golang::path_for_coverage_dummy_test "${package}")"
package main
import (
"testing"
"k8s.io/kubernetes/pkg/util/coverage"
)
func TestMain(m *testing.M) {
// Get coverage running
coverage.InitCoverage("${name}")
// Go!
main()
// Make sure we actually write the profiling information to disk, if we make it here.
// On long-running services, or anything that calls os.Exit(), this is insufficient,
// so we also flush periodically with a default period of five seconds (configurable by
// the KUBE_COVERAGE_FLUSH_INTERVAL environment variable).
coverage.FlushCoverage()
}
EOF
}
# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler).
# Deletes a test generated by kube::golang::create_coverage_dummy_test.
# It is not an error to call this for a nonexistent test.
kube::golang::delete_coverage_dummy_test() {
local package="$1"
rm -f "$(kube::golang::path_for_coverage_dummy_test "${package}")"
}
# Arguments: a list of kubernetes packages to build.
# Expected variables: ${build_args} should be set to an array of Go build arguments.
# In addition, ${package} and ${platform} should have been set earlier, and if
# ${KUBE_BUILD_WITH_COVERAGE} is set, coverage instrumentation will be enabled.
#
# Invokes Go to actually build some packages. If coverage is disabled, simply invokes
# go install. If coverage is enabled, builds covered binaries using go test, temporarily
# producing the required unit test files and then cleaning up after itself.
# Non-covered binaries are then built using go install as usual.
#
# See comments in kube::golang::setup_env regarding where built binaries go.
kube::golang::build_some_binaries() {
if [[ -n "${KUBE_BUILD_WITH_COVERAGE:-}" ]]; then
local -a uncovered=()
for package in "$@"; do
if kube::golang::is_instrumented_package "${package}"; then
V=2 kube::log::info "Building ${package} with coverage..."
kube::golang::create_coverage_dummy_test "${package}"
kube::util::trap_add "kube::golang::delete_coverage_dummy_test \"${package}\"" EXIT
go test -c -o "$(kube::golang::outfile_for_binary "${package}" "${platform}")" \
-covermode count \
-coverpkg k8s.io/... \
"${build_args[@]}" \
-tags coverage \
"${package}"
else
uncovered+=("${package}")
fi
done
if [[ "${#uncovered[@]}" != 0 ]]; then
V=2 kube::log::info "Building ${uncovered[*]} without coverage..."
GOPROXY=off go install "${build_args[@]}" "${uncovered[@]}"
else
V=2 kube::log::info "Nothing to build without coverage."
fi
else
V=2 kube::log::info "Coverage is disabled."
GOPROXY=off go install "${build_args[@]}" "$@"
fi
}
# Args:
# $1: platform (e.g. darwin/amd64)
kube::golang::build_binaries_for_platform() {
# This is for sanity. Without it, user umasks can leak through.
umask 0022
local platform=$1
local -a statics=()
local -a nonstatics=()
local -a tests=()
for binary in "${binaries[@]}"; do
if [[ "${binary}" =~ ".test"$ ]]; then
tests+=("${binary}")
kube::log::info " ${binary} (test)"
elif kube::golang::is_statically_linked "${binary}"; then
statics+=("${binary}")
kube::log::info " ${binary} (static)"
else
nonstatics+=("${binary}")
kube::log::info " ${binary} (non-static)"
fi
done
V=2 kube::log::info "Env for ${platform}: GOPATH=${GOPATH-} GOOS=${GOOS-} GOARCH=${GOARCH-} GOROOT=${GOROOT-} CGO_ENABLED=${CGO_ENABLED-} CC=${CC-}"
V=3 kube::log::info "Building binaries with GCFLAGS=${gogcflags} LDFLAGS=${goldflags}"
local -a build_args
if [[ "${#statics[@]}" != 0 ]]; then
build_args=(
-installsuffix=static
${goflags:+"${goflags[@]}"}
-gcflags="${gogcflags}"
-ldflags="${goldflags}"
-tags="${gotags:-}"
)
CGO_ENABLED=0 kube::golang::build_some_binaries "${statics[@]}"
fi
if [[ "${#nonstatics[@]}" != 0 ]]; then
build_args=(
${goflags:+"${goflags[@]}"}
-gcflags="${gogcflags}"
-ldflags="${goldflags}"
-tags="${gotags:-}"
)
kube::golang::build_some_binaries "${nonstatics[@]}"
fi
for test in "${tests[@]:+${tests[@]}}"; do
local outfile testpkg
outfile=$(kube::golang::outfile_for_binary "${test}" "${platform}")
testpkg=$(dirname "${test}")
mkdir -p "$(dirname "${outfile}")"
go test -c \
${goflags:+"${goflags[@]}"} \
-gcflags="${gogcflags}" \
-ldflags="${goldflags}" \
-tags="${gotags:-}" \
-o "${outfile}" \
"${testpkg}"
done
}
# Return approximate physical memory available in gigabytes.
kube::golang::get_physmem() {
local mem
# Linux kernel version >=3.14, in kb
if mem=$(grep MemAvailable /proc/meminfo | awk '{ print $2 }'); then
echo $(( mem / 1048576 ))
return
fi
# Linux, in kb
if mem=$(grep MemTotal /proc/meminfo | awk '{ print $2 }'); then
echo $(( mem / 1048576 ))
return
fi
# OS X, in bytes. Note that get_physmem, as used, should only ever
# run in a Linux container (because it's only used in the multiple
# platform case, which is a Dockerized build), but this is provided
# for completeness.
if mem=$(sysctl -n hw.memsize 2>/dev/null); then
echo $(( mem / 1073741824 ))
return
fi
# If we can't infer it, just give up and assume a low memory system
echo 1
}
# Build binaries targets specified
#
# Input:
# $@ - targets and go flags. If no targets are set then all binaries targets
# are built.
# KUBE_BUILD_PLATFORMS - Incoming variable of targets to build for. If unset
# then just the host architecture is built.
kube::golang::build_binaries() {
V=2 kube::log::info "Go version: $(GOFLAGS='' go version)"
local host_platform
host_platform=$(kube::golang::host_platform)
# These are "local" but are visible to and relied on by functions this
# function calls. They are effectively part of the calling API to
# build_binaries_for_platform.
local goflags goldflags gogcflags gotags
goflags=()
gogcflags="${GOGCFLAGS:-}"
goldflags="all=$(kube::version::ldflags) ${GOLDFLAGS:-}"
if [[ "${DBG:-}" == 1 ]]; then
# Debugging - disable optimizations and inlining and trimPath
gogcflags="${gogcflags} all=-N -l"
else
# Not debugging - disable symbols and DWARF, trim embedded paths
goldflags="${goldflags} -s -w"
goflags+=("-trimpath")
fi
# Extract tags if any specified in GOFLAGS
gotags="selinux,notest,$(echo "${GOFLAGS:-}" | sed -ne 's|.*-tags=\([^-]*\).*|\1|p')"
local -a targets=()
local arg
for arg; do
if [[ "${arg}" == -* ]]; then
# Assume arguments starting with a dash are flags to pass to go.
goflags+=("${arg}")
else
targets+=("${arg}")
fi
done
local -a platforms
IFS=" " read -ra platforms <<< "${KUBE_BUILD_PLATFORMS:-}"
if [[ ${#platforms[@]} -eq 0 ]]; then
platforms=("${host_platform}")
fi
if [[ ${#targets[@]} -eq 0 ]]; then
targets=("${KUBE_ALL_TARGETS[@]}")
fi
kube::util::read-array targets < <(kube::golang::dedup "${targets[@]}")
local -a binaries
kube::util::read-array binaries < <(kube::golang::normalize_go_targets "${targets[@]}")
kube::util::read-array binaries < <(kube::golang::dedup "${binaries[@]}")
local parallel=false
if [[ ${#platforms[@]} -gt 1 ]]; then
local gigs
gigs=$(kube::golang::get_physmem)
if [[ ${gigs} -ge ${KUBE_PARALLEL_BUILD_MEMORY} ]]; then
kube::log::status "Multiple platforms requested and available ${gigs}G >= threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in parallel"
parallel=true
else
kube::log::status "Multiple platforms requested, but available ${gigs}G < threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in serial"
parallel=false
fi
fi
if [[ "${parallel}" == "true" ]]; then
kube::log::status "Building go targets for {${platforms[*]}} in parallel (output will appear in a burst when complete):" "${targets[@]}"
local platform
for platform in "${platforms[@]}"; do (
kube::golang::set_platform_envs "${platform}"
kube::log::status "${platform}: build started"
kube::golang::build_binaries_for_platform "${platform}"
kube::log::status "${platform}: build finished"
) &> "/tmp//${platform//\//_}.build" &
done
local fails=0
for job in $(jobs -p); do
wait "${job}" || (( fails+=1 ))
done
for platform in "${platforms[@]}"; do
cat "/tmp//${platform//\//_}.build"
done
return "${fails}"
else
for platform in "${platforms[@]}"; do
kube::log::status "Building go targets for ${platform}"
(
kube::golang::set_platform_envs "${platform}"
kube::golang::build_binaries_for_platform "${platform}"
)
done
fi
}

62
hack/lib/init.sh
View File

@@ -1,35 +1,66 @@
#!/usr/bin/env bash
# This script is modified version of Kubernetes script
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
export GO111MODULE=auto
# Short-circuit if init.sh has already been sourced
[[ $(type -t kube::init::loaded) == function ]] && return 0
# Unset CDPATH so that path interpolation can work correctly
# https://github.com/kubernetes/kubernetes/issues/52255
unset CDPATH
# The root of the build/dist directory
KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
KUBE_OUTPUT_SUBPATH="${KUBE_OUTPUT_SUBPATH:-_output/local}"
KUBE_OUTPUT="${KUBE_ROOT}/${KUBE_OUTPUT_SUBPATH}"
KUBE_OUTPUT_BINPATH="${KUBE_OUTPUT}/bin"
# Where output goes. We should avoid redefining these anywhere else.
#
# KUBE_OUTPUT: the root directory (absolute) where this build should drop any
# files (subdirs are encouraged).
# KUBE_OUTPUT_BIN: the directory in which compiled binaries will be placed,
# under OS/ARCH specific subdirs
# THIS_PLATFORM_BIN: a symlink to the output directory for binaries built for
# the current host platform (e.g. build/test tools).
#
# Compat: The KUBE_OUTPUT_SUBPATH variable is sometimes passed in by callers.
# If it is specified, we'll use it in KUBE_OUTPUT.
_KUBE_OUTPUT_SUBPATH="${KUBE_OUTPUT_SUBPATH:-_output/local}"
export KUBE_OUTPUT="${KUBE_ROOT}/${_KUBE_OUTPUT_SUBPATH}"
export KUBE_OUTPUT_BIN="${KUBE_OUTPUT}/bin"
export THIS_PLATFORM_BIN="${KUBE_ROOT}/_output/bin"
# This controls rsync compression. Set to a value > 0 to enable rsync
# compression for build container
KUBE_RSYNC_COMPRESS="${KUBE_RSYNC_COMPRESS:-0}"
# Set no_proxy for localhost if behind a proxy, otherwise,
# the connections to localhost in scripts will time out
export no_proxy="127.0.0.1,localhost${no_proxy:+,${no_proxy}}"
source "${KUBE_ROOT}/hack/lib/util.sh"
source "${KUBE_ROOT}/hack/lib/logging.sh"
source "${KUBE_ROOT}/hack/lib/version.sh"
kube::log::install_errexit
kube::util::ensure-bash-version
source "${KUBE_ROOT}/hack/lib/version.sh"
source "${KUBE_ROOT}/hack/lib/golang.sh"
KUBE_OUTPUT_HOSTBIN="${KUBE_OUTPUT_BINPATH}/$(kube::util::host_platform)"
export KUBE_OUTPUT_HOSTBIN
# This emulates "readlink -f" which is not available on MacOS X.
# Test:
# T=/tmp/$$.$RANDOM
@@ -109,4 +140,9 @@ kube::realpath() {
return 1
fi
kube::readlinkdashf "${1}"
}
}
# Marker function to indicate init.sh has been fully sourced
kube::init::loaded() {
return 0
}

312
hack/lib/util.sh
View File

@@ -54,7 +54,7 @@ kube::util::wait_for_url() {
local i
for i in $(seq 1 "${times}"); do
local out
if out=$(curl --max-time "${maxtime}" -gkfs "${url}" 2>/dev/null); then
if out=$(curl --max-time "${maxtime}" -gkfs "${@:6}" "${url}" 2>/dev/null); then
kube::log::status "On try ${i}, ${prefix}: ${out}"
return 0
fi
@@ -64,6 +64,35 @@ kube::util::wait_for_url() {
return 1
}
kube::util::wait_for_url_with_bearer_token() {
local url=$1
local token=$2
local prefix=${3:-}
local wait=${4:-1}
local times=${5:-30}
local maxtime=${6:-1}
kube::util::wait_for_url "${url}" "${prefix}" "${wait}" "${times}" "${maxtime}" -H "Authorization: Bearer ${token}"
}
# Example: kube::util::wait_for_success 120 5 "kubectl get nodes|grep localhost"
# arguments: wait time, sleep time, shell command
# returns 0 if the shell command get output, 1 otherwise.
kube::util::wait_for_success(){
local wait_time="$1"
local sleep_time="$2"
local cmd="$3"
while [ "$wait_time" -gt 0 ]; do
if eval "$cmd"; then
return 0
else
sleep "$sleep_time"
wait_time=$((wait_time-sleep_time))
fi
done
return 1
}
# Example: kube::util::trap_add 'echo "in trap DEBUG"' DEBUG
# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal
kube::util::trap_add() {
@@ -182,17 +211,29 @@ kube::util::find-binary-for-platform() {
"${KUBE_ROOT}/_output/local/bin/${platform}/${lookfor}"
"${KUBE_ROOT}/platforms/${platform}/${lookfor}"
)
# Also search for binary in bazel build tree.
# The bazel go rules place some binaries in subtrees like
# "bazel-bin/source/path/linux_amd64_pure_stripped/binaryname", so make sure
# the platform name is matched in the path.
while IFS=$'\n' read -r location; do
locations+=("$location");
done < <(find "${KUBE_ROOT}/bazel-bin/" -type f -executable \
\( -path "*/${platform/\//_}*/${lookfor}" -o -path "*/${lookfor}" \) 2>/dev/null || true)
# if we're looking for the host platform, add local non-platform-qualified search paths
if [[ "${platform}" = "$(kube::util::host_platform)" ]]; then
locations+=(
"${KUBE_ROOT}/_output/local/go/bin/${lookfor}"
"${KUBE_ROOT}/_output/dockerized/go/bin/${lookfor}"
);
fi
# looks for $1 in the $PATH
if which "${lookfor}" >/dev/null; then
local -r local_bin="$(which "${lookfor}")"
locations+=( "${local_bin}" );
fi
# List most recently-updated location.
local -r bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )
if [[ -z "${bin}" ]]; then
kube::log::error "Failed to find binary ${lookfor} for platform ${platform}"
return 1
fi
echo -n "${bin}"
}
@@ -202,86 +243,40 @@ kube::util::find-binary() {
kube::util::find-binary-for-platform "$1" "$(kube::util::host_platform)"
}
# Run all known doc generators (today gendocs and genman for kubectl)
# $1 is the directory to put those generated documents
kube::util::gen-docs() {
local dest="$1"
# Find binary
gendocs=$(kube::util::find-binary "gendocs")
genkubedocs=$(kube::util::find-binary "genkubedocs")
genman=$(kube::util::find-binary "genman")
genyaml=$(kube::util::find-binary "genyaml")
genfeddocs=$(kube::util::find-binary "genfeddocs")
# TODO: If ${genfeddocs} is not used from anywhere (it isn't used at
# least from k/k tree), remove it completely.
kube::util::sourced_variable "${genfeddocs}"
mkdir -p "${dest}/docs/user-guide/kubectl/"
"${gendocs}" "${dest}/docs/user-guide/kubectl/"
mkdir -p "${dest}/docs/admin/"
"${genkubedocs}" "${dest}/docs/admin/" "kube-apiserver"
"${genkubedocs}" "${dest}/docs/admin/" "kube-controller-manager"
"${genkubedocs}" "${dest}/docs/admin/" "kube-proxy"
"${genkubedocs}" "${dest}/docs/admin/" "kube-scheduler"
"${genkubedocs}" "${dest}/docs/admin/" "kubelet"
"${genkubedocs}" "${dest}/docs/admin/" "kubeadm"
mkdir -p "${dest}/docs/man/man1/"
"${genman}" "${dest}/docs/man/man1/" "kube-apiserver"
"${genman}" "${dest}/docs/man/man1/" "kube-controller-manager"
"${genman}" "${dest}/docs/man/man1/" "kube-proxy"
"${genman}" "${dest}/docs/man/man1/" "kube-scheduler"
"${genman}" "${dest}/docs/man/man1/" "kubelet"
"${genman}" "${dest}/docs/man/man1/" "kubectl"
"${genman}" "${dest}/docs/man/man1/" "kubeadm"
mkdir -p "${dest}/docs/yaml/kubectl/"
"${genyaml}" "${dest}/docs/yaml/kubectl/"
# create the list of generated files
pushd "${dest}" > /dev/null || return 1
touch docs/.generated_docs
find . -type f | cut -sd / -f 2- | LC_ALL=C sort > docs/.generated_docs
popd > /dev/null || return 1
}
# Removes previously generated docs-- we don't want to check them in. $KUBE_ROOT
# must be set.
kube::util::remove-gen-docs() {
if [ -e "${KUBE_ROOT}/docs/.generated_docs" ]; then
# remove all of the old docs; we don't want to check them in.
while read -r file; do
rm "${KUBE_ROOT}/${file}" 2>/dev/null || true
done <"${KUBE_ROOT}/docs/.generated_docs"
# The docs/.generated_docs file lists itself, so we don't need to explicitly
# delete it.
fi
}
# Takes a group/version and returns the path to its location on disk, sans
# "pkg". E.g.:
# * default behavior: extensions/v1beta1 -> apis/extensions/v1beta1
# * default behavior for only a group: experimental -> apis/experimental
# * Special handling for empty group: v1 -> api/v1, unversioned -> api/unversioned
# * Special handling for groups suffixed with ".k8s.io": foo.k8s.io/v1 -> apis/foo/v1
# * Special handling for groups suffixed with ".kubesphere.io": foo.kubesphere.io/v1 -> apis/foo/v1
# * Very special handling for when both group and version are "": / -> api
#
# $KUBE_ROOT must be set.
kube::util::group-version-to-pkg-path() {
local group_version="$1"
while IFS=$'\n' read -r api; do
if [[ "${api}" = "${group_version/.*k8s.io/}" ]]; then
echo "vendor/k8s.io/api/${group_version/.*k8s.io/}"
# Make a list of all know APIs by listing their dirs.
local apidirs=()
kube::util::read-array apidirs < <(
cd "${KUBE_ROOT}/staging/src/kubesphere.io/api" || return 1 # make shellcheck happy
find . -name types.go -exec dirname {} \; \
| sed "s|\./||g" \
| LC_ALL=C sort -u)
# Compare each API dir against the requested GV, and if we find it, no
# special handling needed.
for api in "${apidirs[@]}"; do
# Change "foo.bar.kubesphere.io/v1" -> "foo/v1" notation.
local simple_gv="${group_version/.*kubesphere.io/}"
if [[ "${api}" = "${simple_gv}" ]]; then
echo "staging/src/kubesphere.io/api/${simple_gv}"
return
fi
done < <(cd "${KUBE_ROOT}/staging/src/k8s.io/api" && find . -name types.go -exec dirname {} \; | sed "s|\./||g" | sort)
done
# "v1" is the API GroupVersion
if [[ "${group_version}" == "v1" ]]; then
echo "vendor/k8s.io/api/core/v1"
echo "staging/src/kubesphere.io/api/core/v1"
return
fi
@@ -293,17 +288,11 @@ kube::util::group-version-to-pkg-path() {
__internal)
echo "pkg/apis/core"
;;
meta/v1)
echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1"
*.kubesphere.io)
echo "pkg/apis/${group_version%.*kubesphere.io}"
;;
meta/v1beta1)
echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1beta1"
;;
*.k8s.io)
echo "pkg/apis/${group_version%.*k8s.io}"
;;
*.k8s.io/*)
echo "pkg/apis/${group_version/.*k8s.io/}"
*.kubesphere.io/*)
echo "pkg/apis/${group_version/.*kubesphere.io/}"
;;
*)
echo "pkg/apis/${group_version%__internal}"
@@ -330,7 +319,7 @@ kube::util::gv-to-swagger-name() {
# repo, e.g. "upstream" or "origin".
kube::util::git_upstream_remote_name() {
git remote -v | grep fetch |\
grep -E 'github.com[/:]kubernetes/kubernetes|k8s.io/kubernetes' |\
grep -E 'github.com[/:]kubesphere/kubesphere|kubesphere.io/kubesphere' |\
head -n 1 | awk '{print $1}'
}
@@ -429,6 +418,22 @@ function kube::util::test_openssl_installed {
OPENSSL_BIN=$(command -v openssl)
}
# Query the API server for client certificate authentication capabilities
function kube::util::test_client_certificate_authentication_enabled {
local output
kube::util::test_openssl_installed
output=$(echo \
| "${OPENSSL_BIN}" s_client -connect "127.0.0.1:${SECURE_API_PORT}" 2> /dev/null \
| grep -A3 'Acceptable client certificate CA names')
if [[ "${output}" != *"/CN=127.0.0.1"* ]] && [[ "${output}" != *"CN = 127.0.0.1"* ]]; then
echo "API server not configured for client certificate authentication"
echo "Output of from acceptable client certificate check: ${output}"
exit 1
fi
}
# creates a client CA, args are sudo, dest-dir, ca-id, purpose
# purpose is dropped in after "key encipherment", you usually want
# '"client auth"'
@@ -535,8 +540,8 @@ EOF
EOF
}
# list_staging_repos outputs a sorted list of repos in staging/src/k8s.io
# each entry will just be the $repo portion of staging/src/k8s.io/$repo/...
# list_staging_repos outputs a sorted list of repos in staging/src/kubesphere.io
# each entry will just be the $repo portion of staging/src/kubesphere.io/$repo/...
# $KUBE_ROOT must be set.
function kube::util::list_staging_repos() {
(
@@ -548,22 +553,18 @@ function kube::util::list_staging_repos() {
# Determines if docker can be run, failures may simply require that the user be added to the docker group.
function kube::util::ensure_docker_daemon_connectivity {
IFS=" " read -ra DOCKER <<< "${DOCKER_OPTS}"
# Expand ${DOCKER[@]} only if it's not unset. This is to work around
# Bash 3 issue with unbound variable.
DOCKER=(docker ${DOCKER[@]:+"${DOCKER[@]}"})
if ! "${DOCKER[@]}" info > /dev/null 2>&1 ; then
DOCKER_OPTS=${DOCKER_OPTS:-""}
IFS=" " read -ra docker_opts <<< "${DOCKER_OPTS}"
if ! docker "${docker_opts[@]:+"${docker_opts[@]}"}" info > /dev/null 2>&1 ; then
cat <<'EOF' >&2
Can't connect to 'docker' daemon. please fix and retry.
Possible causes:
- Docker Daemon not started
- Linux: confirm via your init system
- macOS w/ docker-machine: run `docker-machine ls` and `docker-machine start <name>`
- macOS w/ Docker for Mac: Check the menu bar and start the Docker application
- DOCKER_HOST hasn't been set or is set incorrectly
- Linux: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}`
- macOS w/ docker-machine: run `eval "$(docker-machine env <name>)"`
- macOS w/ Docker for Mac: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}`
- Other things to check:
- Linux: User isn't in 'docker' group. Add and relogin.
@@ -605,6 +606,7 @@ function kube::util::join {
# CFSSL_BIN: The path of the installed cfssl binary
# CFSSLJSON_BIN: The path of the installed cfssljson binary
#
# shellcheck disable=SC2120 # optional parameters
function kube::util::ensure-cfssl {
if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null; then
CFSSL_BIN=$(command -v cfssl)
@@ -617,7 +619,7 @@ function kube::util::ensure-cfssl {
if [[ "${host_arch}" != "amd64" ]]; then
echo "Cannot download cfssl on non-amd64 hosts and cfssl does not appear to be installed."
echo "Please install cfssl and cfssljson and verify they are in \$PATH."
echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..."
echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go install github.com/cloudflare/cfssl/cmd/...@latest"
exit 1
fi
@@ -635,12 +637,12 @@ function kube::util::ensure-cfssl {
kernel=$(uname -s)
case "${kernel}" in
Linux)
curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
curl --retry 10 -L -o cfssl https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_linux_amd64
curl --retry 10 -L -o cfssljson https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_linux_amd64
;;
Darwin)
curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64
curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64
curl --retry 10 -L -o cfssl https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_darwin_amd64
curl --retry 10 -L -o cfssljson https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_darwin_amd64
;;
*)
echo "Unknown, unsupported platform: ${kernel}." >&2
@@ -655,20 +657,35 @@ function kube::util::ensure-cfssl {
CFSSLJSON_BIN="${cfssldir}/cfssljson"
if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} ]]; then
echo "Failed to download 'cfssl'. Please install cfssl and cfssljson and verify they are in \$PATH."
echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..."
echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go install github.com/cloudflare/cfssl/cmd/...@latest"
exit 1
fi
popd > /dev/null || return 1
}
# kube::util::ensure_dockerized
# Confirms that the script is being run inside a kube-build image
# kube::util::ensure-docker-buildx
# Check if we have "docker buildx" commands available
#
function kube::util::ensure_dockerized {
if [[ -f /kube-build-image ]]; then
function kube::util::ensure-docker-buildx {
# podman returns 0 on `docker buildx version`, docker on `docker buildx`. One of them must succeed.
if docker buildx version >/dev/null 2>&1 || docker buildx >/dev/null 2>&1; then
return 0
else
echo "ERROR: This script is designed to be run inside a kube-build container"
echo "ERROR: docker buildx not available. Docker 19.03 or higher is required with experimental features enabled"
exit 1
fi
}
# kube::util::ensure-bash-version
# Check if we are using a supported bash version
#
function kube::util::ensure-bash-version {
# shellcheck disable=SC2004
if ((${BASH_VERSINFO[0]}<4)) || ( ((${BASH_VERSINFO[0]}==4)) && ((${BASH_VERSINFO[1]}<2)) ); then
echo "ERROR: This script requires a minimum bash version of 4.2, but got version of ${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}"
if [ "$(uname)" = 'Darwin' ]; then
echo "On macOS with homebrew 'brew install bash' is sufficient."
fi
exit 1
fi
}
@@ -680,7 +697,10 @@ function kube::util::ensure_dockerized {
# SED: The name of the gnu-sed binary
#
function kube::util::ensure-gnu-sed {
if LANG=C sed --help 2>&1 | grep -q GNU; then
# NOTE: the echo below is a workaround to ensure sed is executed before the grep.
# see: https://github.com/kubernetes/kubernetes/issues/87251
sed_help="$(LANG=C sed --help 2>&1 || true)"
if echo "${sed_help}" | grep -q "GNU\|BusyBox"; then
SED="sed"
elif command -v gsed &>/dev/null; then
SED="gsed"
@@ -691,6 +711,27 @@ function kube::util::ensure-gnu-sed {
kube::util::sourced_variable "${SED}"
}
# kube::util::ensure-gnu-date
# Determines which date binary is gnu-date on linux/darwin
#
# Sets:
# DATE: The name of the gnu-date binary
#
function kube::util::ensure-gnu-date {
# NOTE: the echo below is a workaround to ensure date is executed before the grep.
# see: https://github.com/kubernetes/kubernetes/issues/87251
date_help="$(LANG=C date --help 2>&1 || true)"
if echo "${date_help}" | grep -q "GNU\|BusyBox"; then
DATE="date"
elif command -v gdate &>/dev/null; then
DATE="gdate"
else
kube::log::error "Failed to find GNU date as date or gdate. If you are on Mac: brew install coreutils." >&2
return 1
fi
kube::util::sourced_variable "${DATE}"
}
# kube::util::check-file-in-alphabetical-order <file>
# Check that the file is in alphabetical order
#
@@ -712,7 +753,7 @@ function kube::util::check-file-in-alphabetical-order {
# Checks whether jq is installed.
function kube::util::require-jq {
if ! command -v jq &>/dev/null; then
echo "jq not found. Please install." 1>&2
kube::log::error "jq not found. Please install."
return 1
fi
}
@@ -728,19 +769,66 @@ function kube::util::md5() {
# kube::util::read-array
# Reads in stdin and adds it line by line to the array provided. This can be
# used instead of "mapfile -t", and is bash 3 compatible.
# used instead of "mapfile -t", and is bash 3 compatible. If the named array
# exists and is an array, it will be overwritten. Otherwise it will be unset
# and recreated.
#
# Assumed vars:
# $1 (name of array to create/modify)
#
# Example usage:
# kube::util::read-array files < <(ls -1)
# kube::util::read-array files < <(ls -1)
#
# When in doubt:
# $ W=abc # a string
# $ X=(a b c) # an array
# $ declare -A Y # an associative array
# $ unset Z # not set at all
# $ declare -p W X Y Z
# declare -- W="abc"
# declare -a X=([0]="a" [1]="b" [2]="c")
# declare -A Y
# bash: line 26: declare: Z: not found
# $ kube::util::read-array W < <(echo -ne "1 1\n2 2\n3 3\n")
# bash: W is defined but isn't an array
# $ kube::util::read-array X < <(echo -ne "1 1\n2 2\n3 3\n")
# $ kube::util::read-array Y < <(echo -ne "1 1\n2 2\n3 3\n")
# bash: Y is defined but isn't an array
# $ kube::util::read-array Z < <(echo -ne "1 1\n2 2\n3 3\n")
# $ declare -p W X Y Z
# declare -- W="abc"
# declare -a X=([0]="1 1" [1]="2 2" [2]="3 3")
# declare -A Y
# declare -a Z=([0]="1 1" [1]="2 2" [2]="3 3")
function kube::util::read-array {
local i=0
unset -v "$1"
while IFS= read -r "$1[i++]"; do :; done
eval "[[ \${$1[--i]} ]]" || unset "$1[i]" # ensures last element isn't empty
if [[ -z "$1" ]]; then
echo "usage: ${FUNCNAME[0]} <varname>" >&2
return 1
fi
if [[ -n $(declare -p "$1" 2>/dev/null) ]]; then
if ! declare -p "$1" 2>/dev/null | grep -q '^declare -a'; then
echo "${FUNCNAME[0]}: $1 is defined but isn't an array" >&2
return 2
fi
fi
# shellcheck disable=SC2034 # this variable _is_ used
local __read_array_i=0
while IFS= read -r "$1[__read_array_i++]"; do :; done
if ! eval "[[ \${$1[--__read_array_i]} ]]"; then
unset "$1[__read_array_i]" # ensures last element isn't empty
fi
}
# kube::util::run-in
# Changes directory to "$1", runs the rest of the arguments, and restores the initial directory
# Returns 1 if a directory change fails, the result of running the arguments otherwise
function kube::util::run-in {
pushd "$1" > /dev/null || return 1
shift
"$@"
local result=$?
popd > /dev/null || return 1
return $result
}
# Some useful colors.

68
hack/lib/verify-generated.sh Executable file
View File

@@ -0,0 +1,68 @@
#!/usr/bin/env bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Short-circuit if verify-generated.sh has already been sourced.
[[ $(type -t kube::verify::generated::loaded) == function ]] && return 0
source "${KUBE_ROOT}/hack/lib/init.sh"
# This function verifies whether generated files are up-to-date. The first two
# parameters are messages that get printed to stderr when changes are found,
# the rest are the function or command and its parameters for generating files
# in the work tree.
#
# Example: kube::verify::generated "Mock files are out of date" "Please run 'hack/update-mocks.sh'" hack/update-mocks.sh
kube::verify::generated() {
( # a subshell prevents environment changes from leaking out of this function
local failure_header=$1
shift
local failure_tail=$1
shift
kube::util::ensure_clean_working_dir
# This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
kube::golang::setup_env
_tmpdir="$(kube::realpath "$(mktemp -d -t "verify-generated-$(basename "$1").XXXXXX")")"
git worktree add -f -q "${_tmpdir}" HEAD
kube::util::trap_add "git worktree remove -f ${_tmpdir}" EXIT
cd "${_tmpdir}"
# Update generated files.
"$@"
# Test for diffs
diffs=$(git status --porcelain | wc -l)
if [[ ${diffs} -gt 0 ]]; then
if [[ -n "${failure_header}" ]]; then
echo "${failure_header}" >&2
fi
git status >&2
git diff >&2
if [[ -n "${failure_tail}" ]]; then
echo "" >&2
echo "${failure_tail}" >&2
fi
return 1
fi
)
}
# Marker function to indicate verify-generated.sh has been fully sourced.
kube::verify::generated::loaded() {
return 0
}

91
hack/lib/version.sh
View File

@@ -1,5 +1,19 @@
#!/usr/bin/env bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -----------------------------------------------------------------------------
# Version management helpers. These functions help to set, save and load the
# following variables:
@@ -18,6 +32,11 @@
# If KUBE_GIT_VERSION_FILE, this function will load from that file instead of
# querying git.
kube::version::get_version_vars() {
if [[ -n ${KUBE_GIT_VERSION_FILE-} ]]; then
kube::version::load_version_vars "${KUBE_GIT_VERSION_FILE}"
return
fi
# If the kubernetes source was exported through git archive, then
# we likely don't have a git tree, but these magic values may be filled in.
# shellcheck disable=SC2016,SC2050
@@ -80,7 +99,6 @@ kube::version::get_version_vars() {
# the "major" and "minor" versions and whether this is the exact tagged
# version or whether the tree is between two tagged versions.
if [[ "${KUBE_GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?([-].*)?([+].*)?$ ]]; then
# shellcheck disable=SC2034
KUBE_GIT_MAJOR=${BASH_REMATCH[1]}
KUBE_GIT_MINOR=${BASH_REMATCH[2]}
if [[ -n "${BASH_REMATCH[4]}" ]]; then
@@ -90,10 +108,77 @@ kube::version::get_version_vars() {
# If KUBE_GIT_VERSION is not a valid Semantic Version, then refuse to build.
if ! [[ "${KUBE_GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then
echo "KUBE_GIT_VERSION should be a valid Semantic Version. Current value: ${KUBE_GIT_VERSION}"
echo "Please see more details here: https://semver.org"
kube::log::error "KUBE_GIT_VERSION should be a valid Semantic Version. Current value: ${KUBE_GIT_VERSION}"
kube::log::error "Please see more details here: https://semver.org"
exit 1
fi
fi
fi
}
# Saves the environment flags to $1
kube::version::save_version_vars() {
local version_file=${1-}
[[ -n ${version_file} ]] || {
echo "!!! Internal error. No file specified in kube::version::save_version_vars"
return 1
}
cat <<EOF >"${version_file}"
KUBE_GIT_COMMIT='${KUBE_GIT_COMMIT-}'
KUBE_GIT_TREE_STATE='${KUBE_GIT_TREE_STATE-}'
KUBE_GIT_VERSION='${KUBE_GIT_VERSION-}'
KUBE_GIT_MAJOR='${KUBE_GIT_MAJOR-}'
KUBE_GIT_MINOR='${KUBE_GIT_MINOR-}'
EOF
}
# Loads up the version variables from file $1
kube::version::load_version_vars() {
local version_file=${1-}
[[ -n ${version_file} ]] || {
echo "!!! Internal error. No file specified in kube::version::load_version_vars"
return 1
}
source "${version_file}"
}
KUBE_GO_PACKAGE=kubesphere.io/kubesphere
# Prints the value that needs to be passed to the -ldflags parameter of go build
# in order to set the Kubernetes based on the git tree status.
# IMPORTANT: if you update any of these, also update the lists in
# hack/print-workspace-status.sh.
kube::version::ldflags() {
kube::version::get_version_vars
local -a ldflags
function add_ldflag() {
local key=${1}
local val=${2}
ldflags+=(
"-X '${KUBE_GO_PACKAGE}/pkg/version.${key}=${val}'"
)
}
kube::util::ensure-gnu-date
add_ldflag "buildDate" "$(${DATE} ${SOURCE_DATE_EPOCH:+"--date=@${SOURCE_DATE_EPOCH}"} -u +'%Y-%m-%dT%H:%M:%SZ')"
if [[ -n ${KUBE_GIT_COMMIT-} ]]; then
add_ldflag "gitCommit" "${KUBE_GIT_COMMIT}"
add_ldflag "gitTreeState" "${KUBE_GIT_TREE_STATE}"
fi
if [[ -n ${KUBE_GIT_VERSION-} ]]; then
add_ldflag "gitVersion" "${KUBE_GIT_VERSION}"
fi
if [[ -n ${KUBE_GIT_MAJOR-} && -n ${KUBE_GIT_MINOR-} ]]; then
add_ldflag "gitMajor" "${KUBE_GIT_MAJOR}"
add_ldflag "gitMinor" "${KUBE_GIT_MINOR}"
fi
# The -ldflags parameter takes a single string, so join the output.
echo "${ldflags[*]-}"
}

80
hack/lint-dependencies.sh
View File

@@ -14,6 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# This script checks version dependencies of modules. It checks whether all
# pinned versions of checked dependencies match their preferred version or not.
# Usage: `hack/lint-dependencies.sh`.
set -o errexit
set -o nounset
set -o pipefail
@@ -21,43 +25,44 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on
# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
export GOFLAGS=-mod=mod
# Detect problematic GOPROXY settings that prevent lookup of dependencies
if [[ "${GOPROXY:-}" == "off" ]]; then
kube::log::error "Cannot run with \$GOPROXY=off"
exit 1
fi
kube::golang::verify_go_version
kube::golang::setup_env
kube::util::require-jq
case "${1:-}" in
"--all")
echo "Checking all dependencies"
filter=''
;;
"-a")
echo "Checking all dependencies"
filter=''
;;
"")
# by default, skip checking golang.org/x/... dependencies... we pin to levels that match our go version for those
echo "Skipping golang.org/x/... dependencies, pass --all to include"
filter='select(.Path | startswith("golang.org/x/") | not) |'
;;
*)
kube::log::error "Unrecognized arg: ${1}"
exit 1
;;
esac
# Set the Go environment, otherwise we get "can't compute 'all' using the
# vendor directory".
export GOWORK=off
export GOFLAGS=-mod=mod
# let us log all errors before we exit
rc=0
# List of dependencies we need to avoid dragging back into kubesphere/kubesphere
# Check if unwanted dependencies are removed
# The array and map in `unwanted-dependencies.json` are in alphabetical order.
go run ./tools/cmd/dependencyverifier "${KUBE_ROOT}/hack/unwanted-dependencies.json"
ks_module_regex="kubesphere[.]io/(kubesphere"
for repo in $(kube::util::list_staging_repos); do
ks_module_regex="${ks_module_regex}|${repo}"
done
ks_module_regex="${ks_module_regex})"
recursive_dependencies=$(go mod graph | grep -E " ${ks_module_regex}" | grep -E -v "^${ks_module_regex}" || true)
if [[ -n "${recursive_dependencies}" ]]; then
echo "These external modules depend on kubesphere.io/kubesphere or staging modules, which is not allowed:"
echo ""
echo "${recursive_dependencies}"
fi
outdated=$(go list -m -json all | jq -r "
select(.Replace.Version != null) |
select(.Version != .Replace.Version) |
${filter}
select(.Path) |
\"\(.Path)
pinned: \(.Replace.Version)
@@ -75,18 +80,33 @@ if [[ -n "${outdated}" ]]; then
echo "${outdated}"
fi
noncanonical=$(go list -m -json all | jq -r "
select(.Replace.Version != null) |
select(.Path != .Replace.Path) |
select(.Path) |
\" \(.Path) is replaced with \(.Replace.Path)\"
")
if [[ -n "${noncanonical}" ]]; then
echo ""
echo "These modules are pinned to non-canonical repos."
echo "Revert to using the canonical repo for these modules before merge"
echo ""
echo "${noncanonical}"
fi
unused=$(comm -23 \
<(go mod edit -json | jq -r '.Replace[] | select(.New.Version != null) | .Old.Path' | sort) \
<(go list -m -json all | jq -r .Path | sort))
if [[ -n "${unused}" ]]; then
echo ""
echo "Use the given commands to remove pinned module versions that aren't actually used:"
echo "${unused}" | xargs -L 1 echo 'GO111MODULE=on go mod edit -dropreplace'
echo "${unused}" | xargs -L 1 echo 'go mod edit -dropreplace'
fi
if [[ -n "${unused}${outdated}" ]]; then
exit 1
if [[ -n "${unused}${outdated}${noncanonical}${recursive_dependencies}" ]]; then
rc=1
else
echo "All pinned versions of checked dependencies match their preferred version."
fi
echo "All pinned versions of checked dependencies match their preferred version."
exit 0
exit $rc

29
hack/make-rules/build.sh Executable file
View File

@@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script sets up a go workspace locally and builds all go components.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
KUBE_VERBOSE="${KUBE_VERBOSE:-1}"
source "${KUBE_ROOT}/hack/lib/init.sh"
kube::golang::setup_env
kube::golang::build_binaries "$@"
kube::golang::place_bins

37
hack/make-rules/clean.sh Executable file
View File

@@ -0,0 +1,37 @@
#!/usr/bin/env bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
source "${KUBE_ROOT}/hack/lib/util.sh"
CLEAN_PATTERNS=(
"_tmp"
"doc_tmp"
"test/e2e/generated/bindata.go"
)
for item in "${CLEAN_PATTERNS[@]}"; do
# Shellcheck wants the ":?" because of paranoia about 'rm -rf /'. It will
# cause an error if unset, which is already true because of "nounset", but
# belts AND suspenders is fine for this.
rm -rf "${KUBE_ROOT:?}/${item:?}"
done
# ex: ts=2 sw=2 et filetype=sh

38
hack/make-rules/cross.sh Executable file
View File

@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script sets up a go workspace locally and builds all for all appropriate
# platforms.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
source "${KUBE_ROOT}/hack/lib/init.sh"
# NOTE: Using "${array[*]}" here is correct. [@] becomes distinct words (in
# bash parlance).
make all WHAT="${KUBE_SERVER_TARGETS[*]}" KUBE_BUILD_PLATFORMS="${KUBE_SERVER_PLATFORMS[*]}"
make all WHAT="${KUBE_NODE_TARGETS[*]}" KUBE_BUILD_PLATFORMS="${KUBE_NODE_PLATFORMS[*]}"
make all WHAT="${KUBE_CLIENT_TARGETS[*]}" KUBE_BUILD_PLATFORMS="${KUBE_CLIENT_PLATFORMS[*]}"
make all WHAT="${KUBE_TEST_TARGETS[*]}" KUBE_BUILD_PLATFORMS="${KUBE_TEST_PLATFORMS[*]}"
make all WHAT="${KUBE_TEST_SERVER_TARGETS[*]}" KUBE_BUILD_PLATFORMS="${KUBE_TEST_SERVER_PLATFORMS[*]}"

50
hack/make-rules/make-help.sh Executable file
View File

@@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
red=$(tput setaf 1)
reset=$(tput sgr0)
readonly red reset
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
ALL_TARGETS=$(make -C "${KUBE_ROOT}" PRINT_HELP=y -rpn | sed -n -e '/^$/ { n ; /^[^ .#][^ ]*:/ { s/:.*$// ; p ; } ; }' | sort)
CMD_TARGETS=$(cd "${KUBE_ROOT}/cmd"; find . -mindepth 1 -maxdepth 1 -type d | cut -c 3-)
CMD_FLAG=false
echo "--------------------------------------------------------------------------------"
for tar in ${ALL_TARGETS}; do
for cmdtar in ${CMD_TARGETS}; do
if [ "${tar}" = "${cmdtar}" ]; then
if [ ${CMD_FLAG} = true ]; then
continue 2;
fi
echo -e "${red}${CMD_TARGETS}${reset}"
make -C "${KUBE_ROOT}" "${tar}" PRINT_HELP=y
echo "---------------------------------------------------------------------------------"
CMD_FLAG=true
continue 2
fi
done
echo -e "${red}${tar}${reset}"
make -C "${KUBE_ROOT}" "${tar}" PRINT_HELP=y
echo "---------------------------------------------------------------------------------"
done

67
hack/make-rules/update.sh Executable file
View File

@@ -0,0 +1,67 @@
#!/usr/bin/env bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A single script that runs a predefined set of update-* scripts, as they often go together.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
source "${KUBE_ROOT}/hack/lib/init.sh"
SILENT=${SILENT:-true}
ALL=${FORCE_ALL:-false}
trap 'exit 1' SIGINT
if ${SILENT} ; then
echo "Running in silent mode, run with SILENT=false if you want to see script logs."
fi
if ! ${ALL} ; then
echo "Running in short-circuit mode; run with FORCE_ALL=true to force all scripts to run."
fi
BASH_TARGETS=(
update-codegen
update-featuregates
update-generated-api-compatibility-data
update-generated-docs
update-openapi-spec
update-gofmt
update-golangci-lint-config
)
for t in "${BASH_TARGETS[@]}"; do
echo -e "${color_yellow:?}Running ${t}${color_norm:?}"
if ${SILENT} ; then
if ! bash "${KUBE_ROOT}/hack/${t}.sh" 1> /dev/null; then
echo -e "${color_red:?}Running ${t} FAILED${color_norm}"
if ! ${ALL}; then
exit 1
fi
fi
else
if ! bash "${KUBE_ROOT}/hack/${t}.sh"; then
echo -e "${color_red}Running ${t} FAILED${color_norm}"
if ! ${ALL}; then
exit 1
fi
fi
fi
done
echo -e "${color_green:?}Update scripts completed successfully${color_norm}"

253
hack/make-rules/verify.sh Executable file
View File

@@ -0,0 +1,253 @@
#!/usr/bin/env bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Indirect calls through kube::util::run-in aren't interpreted
# shellcheck disable=SC2317
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
source "${KUBE_ROOT}/hack/lib/init.sh"
kube::golang::setup_env
# If KUBE_JUNIT_REPORT_DIR is unset, and ARTIFACTS is set, then have them match.
if [[ -z "${KUBE_JUNIT_REPORT_DIR:-}" && -n "${ARTIFACTS:-}" ]]; then
export KUBE_JUNIT_REPORT_DIR="${ARTIFACTS}"
fi
# include shell2junit library
source "${KUBE_ROOT}/third_party/forked/shell2junit/sh2ju.sh"
# Excluded check patterns are always skipped.
EXCLUDED_PATTERNS=(
"verify-all.sh" # this script calls the make rule and would cause a loop
"verify-*-dockerized.sh" # Don't run any scripts that intended to be run dockerized
"verify-golangci-lint-pr.sh" # Runs in a separate job for PRs.
"verify-golangci-lint-pr-hints.sh" # Runs in a separate job for PRs.
"verify-licenses.sh" # runs in a separate job to monitor availability of the dependencies periodically
"verify-openapi-docs-urls.sh" # Spams docs URLs, don't run in CI.
)
# Exclude typecheck in certain cases, if they're running in a separate job.
if [[ ${EXCLUDE_TYPECHECK:-} =~ ^[yY]$ ]]; then
EXCLUDED_PATTERNS+=(
"verify-typecheck.sh" # runs in separate typecheck job
)
fi
# Exclude dependency checks in certain cases, if they're running in a separate job.
# From @cblecker: We can't change the variable name here, unless we update it throughout
# test-infra (and we would need to pick it backwards).
if [[ ${EXCLUDE_GODEP:-} =~ ^[yY]$ ]]; then
EXCLUDED_PATTERNS+=(
"verify-external-dependencies-version.sh" # runs in separate dependencies job
"verify-vendor.sh" # runs in separate dependencies job
"verify-vendor-licenses.sh" # runs in separate dependencies job
)
fi
# Exclude golangci-lint if requested, for example in pull-kubernetes-verify.
if [[ ${EXCLUDE_GOLANGCI_LINT:-} =~ ^[yY]$ ]]; then
EXCLUDED_PATTERNS+=(
"verify-golangci-lint.sh" # runs in separate pull-kubernetes-verify-lint
)
fi
# Exclude readonly package check in certain cases, aka, in periodic jobs we don't care and a readonly package won't be touched
if [[ ${EXCLUDE_READONLY_PACKAGE:-} =~ ^[yY]$ ]]; then
EXCLUDED_PATTERNS+=(
"verify-readonly-packages.sh" # skip in CI, if env is set
)
fi
# Only run known fast checks in quick mode.
# These ideally run in less than 10s.
QUICK_PATTERNS+=(
"verify-api-groups.sh"
"verify-boilerplate.sh"
"verify-external-dependencies-version.sh"
"verify-featuregates.sh"
"verify-fieldname-docs.sh"
"verify-gofmt.sh"
"verify-imports.sh"
"verify-non-mutating-validation.sh"
"verify-pkg-names.sh"
"verify-readonly-packages.sh"
"verify-spelling.sh"
"verify-staging-client-go.sh"
"verify-staging-meta-files.sh"
"verify-test-featuregates.sh"
"verify-test-images.sh"
"verify-vendor-licenses.sh"
)
while IFS='' read -r line; do EXCLUDED_CHECKS+=("$line"); done < <(ls "${EXCLUDED_PATTERNS[@]/#/${KUBE_ROOT}/hack/}" 2>/dev/null || true)
while IFS='' read -r line; do QUICK_CHECKS+=("$line"); done < <(ls "${QUICK_PATTERNS[@]/#/${KUBE_ROOT}/hack/}" 2>/dev/null || true)
TARGET_LIST=()
IFS=" " read -r -a TARGET_LIST <<< "${WHAT:-}"
function is-excluded {
for e in "${EXCLUDED_CHECKS[@]}"; do
if [[ $1 -ef "${e}" ]]; then
return
fi
done
return 1
}
function is-quick {
for e in "${QUICK_CHECKS[@]}"; do
if [[ $1 -ef "${e}" ]]; then
return
fi
done
return 1
}
function is-explicitly-chosen {
local name="${1#verify-}"
name="${name%.*}"
index=0
for e in "${TARGET_LIST[@]}"; do
if [[ "${e}" == "${name}" ]]; then
TARGET_LIST[index]=""
return
fi
index=$((index + 1))
done
return 1
}
function run-cmd {
local filename="${2##*/verify-}"
local testname="${filename%%.*}"
local output="${KUBE_JUNIT_REPORT_DIR:-/tmp/junit-results}"
local tr
if ${SILENT}; then
juLog -output="${output}" -class="verify" -name="${testname}" -fail="^ERROR: " "$@" &> /dev/null
tr=$?
else
juLog -output="${output}" -class="verify" -name="${testname}" -fail="^ERROR: " "$@"
tr=$?
fi
return "${tr}"
}
# Collect Failed tests in this Array , initialize it to nil
FAILED_TESTS=()
function print-failed-tests {
echo -e "========================"
echo -e "${color_red:?}FAILED TESTS${color_norm:?}"
echo -e "========================"
for t in "${FAILED_TESTS[@]}"; do
echo -e "${color_red}${t}${color_norm}"
done
}
function run-checks {
local -r pattern=$1
local -r runner=$2
local t
for t in ${pattern}
do
if [ "$t" = "$pattern" ]; then
# The pattern didn't match any files
continue
fi
local check_name
check_name="$(basename "${t}")"
if [[ -n ${WHAT:-} ]]; then
if ! is-explicitly-chosen "${check_name}"; then
continue
fi
else
if is-excluded "${t}" ; then
echo "Skipping ${check_name}"
continue
fi
if ${QUICK} && ! is-quick "${t}" ; then
echo "Skipping ${check_name} in quick mode"
continue
fi
fi
echo -e "Verifying ${check_name}"
local start
start=$(date +%s)
run-cmd "${runner}" "${t}" && tr=$? || tr=$?
local elapsed=$(($(date +%s) - start))
if [[ ${tr} -eq 0 ]]; then
echo -e "${color_green:?}SUCCESS${color_norm} ${check_name}\t${elapsed}s"
else
echo -e "${color_red}FAILED${color_norm} ${check_name}\t${elapsed}s"
ret=1
FAILED_TESTS+=("${PWD}/${t}")
fi
done
}
# Check invalid targets specified in "WHAT" and mark them as failure cases
function missing-target-checks {
# In case WHAT is not specified
[[ ${#TARGET_LIST[@]} -eq 0 ]] && return
for v in "${TARGET_LIST[@]}"
do
[[ -z "${v}" ]] && continue
FAILED_TESTS+=("${v}")
ret=1
done
}
SILENT=${SILENT:-false}
QUICK=${QUICK:-false}
if ${SILENT} ; then
echo "Running in silent mode, run with SILENT=false if you want to see script logs."
fi
if ${QUICK} ; then
echo "Running in quick mode (QUICK=true). Only fast checks will run."
fi
shopt -s globstar
export API_KNOWN_VIOLATIONS_DIR="${KUBE_ROOT}"/api/api-rules
ret=0
# Modules are discovered by looking for go.mod rather than asking go
# to ensure that modules that aren't part of the workspace and/or are
# not dependencies are checked too.
# . and staging are listed explicitly here to avoid _output
for module in ./go.mod ./staging/**/go.mod; do
module="${module%/go.mod}"
if [ -d "$module/hack" ]; then
kube::util::run-in "$module" run-checks "hack/verify-*.sh" bash
kube::util::run-in "$module" run-checks "hack/verify-*.py" python3
fi
done
missing-target-checks
if [[ ${ret} -eq 1 ]]; then
print-failed-tests
fi
exit ${ret}
# ex: ts=2 sw=2 et filetype=sh

25
hack/make-rules/vet.sh Executable file
View File

@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
# Ignore the usual golangci.yaml config because it would
# enable additional linters, then enable just "go vet".
"${KUBE_ROOT}/hack/verify-golangci-lint.sh" -c none -- --disable-all --enable=govet "$@"

126
hack/pin-dependency.sh
View File

@@ -1,5 +1,23 @@
#!/usr/bin/env bash
# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script switches to the preferred version for specified module.
# Usage: `hack/pin-dependency.sh $MODULE $SHA-OR-TAG`.
# Example: `hack/pin-dependency.sh github.com/docker/docker 501cb131a7b7`.
set -o errexit
set -o nounset
set -o pipefail
@@ -7,73 +25,89 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
# Usage:
# hack/pin-dependency.sh $MODULE $SHA-OR-TAG
#
# Example:
# hack/pin-dependency.sh github.com/docker/docker 501cb131a7b7
# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on
# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
export GOFLAGS=
# Detect problematic GOPROXY settings that prevent lookup of dependencies
if [[ "${GOPROXY:-}" == "off" ]]; then
kube::log::error "Cannot run with \$GOPROXY=off"
exit 1
fi
kube::golang::verify_go_version
kube::golang::setup_env
kube::util::require-jq
# Explicitly set GOFLAGS to ignore vendor, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
export GOWORK=off
export GOFLAGS=-mod=mod
dep="${1:-}"
sha="${2:-}"
if [[ -z "${dep}" || -z "${sha}" ]]; then
# Specifying a different repo is optional.
replacement=
case ${dep} in
*=*)
# shellcheck disable=SC2001
replacement=$(echo "${dep}" | sed -e 's/.*=//')
# shellcheck disable=SC2001
dep=$(echo "${dep}" | sed -e 's/=.*//')
;;
*)
replacement="${dep}"
;;
esac
if [[ -z "${dep}" || -z "${replacement}" || -z "${sha}" ]]; then
echo "Usage:"
echo " hack/pin-dependency.sh \$MODULE \$SHA-OR-TAG"
echo " hack/pin-dependency.sh \$MODULE[=\$REPLACEMENT] \$SHA-OR-TAG"
echo ""
echo "Example:"
echo "Examples:"
echo " hack/pin-dependency.sh github.com/docker/docker 501cb131a7b7"
echo " hack/pin-dependency.sh github.com/docker/docker=github.com/johndoe/docker my-experimental-branch"
echo ""
echo "Replacing with a different repository is useful for testing but"
echo "the result should never be merged into Kubernetes!"
echo ""
exit 1
fi
_tmp="${KUBE_ROOT}/_tmp"
cleanup() {
rm -rf "${_tmp}"
}
trap "cleanup" EXIT SIGINT
cleanup
mkdir -p "${_tmp}"
# Find the resolved version before trying to use it.
echo "Running: go mod download ${replacement}@${sha}"
if meta=$(go mod download -json "${replacement}@${sha}"); then
rev=$(echo "${meta}" | jq -r ".Version")
else
error=$(echo "${meta}" | jq -r ".Error")
echo "Download failed: ${error}" >&2
exit 1
fi
echo "Resolved to ${replacement}@${rev}"
# Add the require directive
echo "Running: go get ${dep}@${sha}"
go get -d "${dep}@${sha}"
# Find the resolved version
rev=$(go mod edit -json | jq -r ".Require[] | select(.Path == \"${dep}\") | .Version")
# No entry in go.mod, we must be using the natural version indirectly
if [[ -z "${rev}" ]]; then
# backup the go.mod file, since go list modifies it
cp go.mod "${_tmp}/go.mod.bak"
# find the revision
rev=$(go list -m -json "${dep}" | jq -r .Version)
# restore the go.mod file
mv "${_tmp}/go.mod.bak" go.mod
fi
# No entry found
if [[ -z "${rev}" ]]; then
echo "Could not resolve ${sha}"
exit 1
fi
echo "Resolved to ${dep}@${rev}"
echo "Running: go mod edit -require ${dep}@${rev}"
go mod edit -require "${dep}@${rev}"
# Add the replace directive
echo "Running: go mod edit -replace ${dep}=${dep}@${rev}"
go mod edit -replace "${dep}=${dep}@${rev}"
if [ "${replacement}" != "${dep}" ]; then
echo "Running: go mod edit -replace ${dep}=${replacement}@${rev}"
go mod edit -replace "${dep}=${replacement}@${rev}"
fi
# Propagate pinned version to staging repos
for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/kubesphere.io/${repo}" >/dev/null 2>&1
go mod edit -require "${dep}@${rev}"
# When replacing with a fork, always add a replace statement in all go.mod
# files (not just the root of the staging repos!) because there might be
# indirect dependencies on the fork.
#
# This is excessive, but the resulting commit should never be merged, so it
# isn't that important to get this exactly right.
if [ "${replacement}" != "${dep}" ]; then
find . -name go.mod -print | while read -r modfile; do
(cd "$(dirname "${modfile}")" && go mod edit -replace "${dep}=${replacement}@${rev}")
done
fi
popd >/dev/null 2>&1
done
echo ""
echo "Run hack/update-vendor.sh to rebuild the vendor directory"

14
hack/setup-envtest.sh
View File

@@ -5,22 +5,12 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
kube::golang::verify_go_version
# Ensure that we find the binaries we build before anything else.
export GOBIN="${KUBE_OUTPUT_BINPATH}"
PATH="${GOBIN}:${PATH}"
# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on
kube::golang::setup_env
if ! command -v setup-envtest ; then
echo 'installing setup-envtest'
# While it's preferable not to use @latest here, we have no choice at the moment. Details at
# https://github.com/kubernetes-sigs/kubebuilder/issues/2480
GO111MODULE=auto go install -mod=mod sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20240521074430-fbb7d370bebc
go install -mod=mod sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20240521074430-fbb7d370bebc
fi
setup-envtest use 1.23.x --bin-dir="${KUBE_OUTPUT_BINPATH}"

38
hack/test-go.sh Executable file
View File

@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script runs all *_test.go files. It is equivalent to `make test`.
# Usage: `hack/test-go.sh` or `make test`.
# Note: This script is a vestigial redirection. Please do not add "real" logic.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
kube::golang::setup_env
cd "${KUBE_ROOT}" || exit 1
ENVTEST_K8S_VERSION=1.28.x
KUBEBUILDER_ASSETS=$(setup-envtest use $ENVTEST_K8S_VERSION -p path --bin-dir="${KUBE_OUTPUT_BIN}")
export KUBEBUILDER_ASSETS
go test ./pkg/... ./cmd/... -covermode=atomic -coverprofile=coverage.txt
go -C staging/src/kubesphere.io/api test ./...
go -C staging/src/kubesphere.io/client-go test ./...

545
hack/unwanted-dependencies.json Normal file
View File

@@ -0,0 +1,545 @@
{
"spec": {
"unwantedModules": {
"cloud.google.com/go": "cloud dependency",
"cloud.google.com/go/bigquery": "cloud dependency",
"cloud.google.com/go/compute": "cloud dependency",
"cloud.google.com/go/firestore": "db/datastore clients should not be required",
"cloud.google.com/go/storage": "cloud dependency",
"github.com/GoogleCloudPlatform/k8s-cloud-provider": "cloud dependency",
"github.com/PuerkitoBio/urlesc": "unmaintained, archive mode",
"github.com/armon/consul-api": "MPL license not in CNCF allowlist",
"github.com/bketelsen/crypt": "unused, crypto",
"github.com/containerd/cgroups": "standardize on single cgroups library from runc, refer #128157",
"github.com/davecgh/go-spew": "refer to #103942",
"github.com/form3tech-oss/jwt-go": "unmaintained, archive mode",
"github.com/getsentry/raven-go": "unmaintained, archive mode",
"github.com/go-bindata/go-bindata": "refer to #99829",
"github.com/go-kit/kit": "lots of transitive dependencies, see https://github.com/prometheus/common/issues/255",
"github.com/gogo/googleapis": "depends on unmaintained github.com/gogo/protobuf",
"github.com/gogo/protobuf": "unmaintained",
"github.com/golang/mock": "unmaintained, archive mode",
"github.com/golang/protobuf": "replace with google.golang.org/protobuf",
"github.com/golang/groupcache": "unmaintained",
"github.com/google/gofuzz": "unmaintained, use sigs.k8s.io/randfill",
"github.com/google/s2a-go": "cloud dependency, unstable",
"github.com/google/shlex": "unmaintained, archive mode",
"github.com/googleapis/enterprise-certificate-proxy": "references cloud dependencies",
"github.com/googleapis/gax-go/v2": "references cloud dependencies",
"github.com/gorilla/handlers": "unmaintained, archive mode",
"github.com/gorilla/mux": "unmaintained, archive mode",
"github.com/gorilla/rpc": "unmaintained, archive mode",
"github.com/gorilla/schema": "unmaintained, archive mode",
"github.com/gregjones/httpcache": "unmaintained, archive mode",
"github.com/grpc-ecosystem/go-grpc-prometheus": "unmaintained, archive mode",
"github.com/grpc-ecosystem/grpc-gateway": "use github.com/grpc-ecosystem/grpc-gateway/v2",
"github.com/hashicorp/consul": "MPL license not in CNCF allowlist",
"github.com/hashicorp/errwrap": "MPL license not in CNCF allowlist",
"github.com/hashicorp/go-immutable-radix": "MPL license not in CNCF allowlist",
"github.com/hashicorp/go-multierror": "MPL license not in CNCF allowlist",
"github.com/hashicorp/go-retryablehttp": "MPL license not in CNCF allowlist",
"github.com/hashicorp/go-rootcerts": "MPL license not in CNCF allowlist",
"github.com/hashicorp/go-sockaddr": "MPL license not in CNCF allowlist",
"github.com/hashicorp/go-uuid": "MPL license not in CNCF allowlist",
"github.com/hashicorp/golang-lru": "MPL license not in CNCF allowlist",
"github.com/hashicorp/hcl": "MPL license not in CNCF allowlist",
"github.com/hashicorp/logutils": "MPL license not in CNCF allowlist",
"github.com/hashicorp/memberlist": "MPL license not in CNCF allowlist",
"github.com/hashicorp/serf": "MPL license not in CNCF allowlist",
"github.com/imdario/mergo": "see https://github.com/kubernetes/kubernetes/issues/107499",
"github.com/influxdata/influxdb1-client": "db/datastore clients should not be required",
"github.com/json-iterator/go": "refer to #105030",
"github.com/klauspost/compress": "unreviewable assembly code, `prometheus/client_golang` should use stdlib instead",
"github.com/mailru/easyjson": "unmaintained",
"github.com/miekg/dns": "no dns client/server should be required",
"github.com/mindprince/gonvml": "depends on nvml.h that does not appear to permit modification, redistribution",
"github.com/mitchellh/cli": "MPL license not in CNCF allowlist",
"github.com/mitchellh/gox": "MPL license not in CNCF allowlist",
"github.com/mndrix/tap-go": "unmaintained",
"github.com/modern-go/concurrent": "problematic reliance on golang internals, e.g. https://github.com/modern-go/reflect2/issues/24",
"github.com/modern-go/reflect2": "problematic reliance on golang internals, e.g. https://github.com/modern-go/reflect2/issues/24",
"github.com/onsi/ginkgo": "Ginkgo has been migrated to V2, refer to #109111",
"github.com/pkg/errors": "unmaintained, archive mode",
"github.com/planetscale/vtprotobuf": "avoid using additional proto implementations",
"github.com/smartystreets/goconvey": "MPL license not in CNCF allowlist",
"github.com/xeipuuv/gojsonschema": "unmaintained",
"go.mongodb.org/mongo-driver": "",
"go.opencensus.io": "unmaintained, https://github.com/census-instrumentation/opencensus-go archive mode",
"golang.org/x/exp": "This subrepository holds experimental and deprecated packages",
"golang.org/x/lint": "unmaintained, archive mode",
"google.golang.org/api": "cloud dependency",
"google.golang.org/appengine": "cloud dependency",
"google.golang.org/genproto": "refer to #113366",
"gopkg.in/square/go-jose.v2": "obsolete, use gopkg.in/go-jose/go-jose.v2",
"gopkg.in/fsnotify.v1": "obsolete, use github.com/fsnotify/fsnotify",
"gopkg.in/yaml.v2": "prefer sigs.k8s.io/yaml",
"gopkg.in/yaml.v3": "prefer sigs.k8s.io/yaml/goyaml.v3",
"k8s.io/klog": "we have switched to klog v2, so avoid klog v1",
"rsc.io/quote": "refer to #102833",
"rsc.io/sampler": "refer to #102833"
}
},
"status": {
"unwantedReferences": {
"cloud.google.com/go": [
"github.com/spf13/afero"
],
"cloud.google.com/go/compute": [
"github.com/google/go-containerregistry"
],
"cloud.google.com/go/storage": [
"github.com/spf13/afero"
],
"github.com/containerd/cgroups": [
"github.com/containerd/containerd"
],
"github.com/davecgh/go-spew": [
"github.com/Masterminds/sprig/v3",
"github.com/Masterminds/squirrel",
"github.com/containerd/containerd",
"github.com/containerd/platforms",
"github.com/cyphar/filepath-securejoin",
"github.com/go-git/go-billy/v5",
"github.com/go-git/go-git/v5",
"github.com/go-gorp/gorp/v3",
"github.com/go-jose/go-jose/v4",
"github.com/go-openapi/errors",
"github.com/go-openapi/jsonpointer",
"github.com/go-openapi/swag",
"github.com/go-task/slim-sprig/v3",
"github.com/jmespath/go-jmespath/internal/testify",
"github.com/json-iterator/go",
"github.com/pelletier/go-toml/v2",
"github.com/prometheus/common",
"github.com/sagikazarmark/locafero",
"github.com/sergi/go-diff",
"github.com/sirupsen/logrus",
"github.com/sourcegraph/conc",
"github.com/spf13/viper",
"github.com/stretchr/testify",
"github.com/subosito/gotenv",
"go.etcd.io/etcd/client/pkg/v3",
"go.mongodb.org/mongo-driver",
"go.opentelemetry.io/auto/sdk",
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc",
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp",
"go.opentelemetry.io/otel",
"go.opentelemetry.io/otel/exporters/otlp/otlptrace",
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc",
"go.opentelemetry.io/otel/metric",
"go.opentelemetry.io/otel/sdk",
"go.opentelemetry.io/otel/trace",
"go.uber.org/multierr",
"go.uber.org/zap",
"gomodules.xyz/jsonpatch/v2",
"helm.sh/helm/v3",
"k8s.io/api",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/code-generator",
"k8s.io/component-base",
"k8s.io/kube-openapi",
"k8s.io/kubectl",
"k8s.io/utils",
"oras.land/oras-go",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/kustomize/api",
"sigs.k8s.io/kustomize/kyaml"
],
"github.com/gogo/protobuf": [
"github.com/containerd/containerd",
"github.com/google/go-containerregistry",
"go.etcd.io/etcd/api/v3",
"go.etcd.io/etcd/client/v3",
"helm.sh/helm/v3",
"k8s.io/api",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/code-generator",
"k8s.io/component-base",
"k8s.io/kms",
"k8s.io/kubectl",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/controller-tools"
],
"github.com/golang/groupcache": [
"github.com/containerd/containerd",
"github.com/go-git/go-git/v5",
"github.com/spf13/afero",
"sigs.k8s.io/application",
"sigs.k8s.io/controller-runtime"
],
"github.com/golang/protobuf": [
"github.com/containerd/containerd",
"github.com/google/gnostic-models",
"github.com/google/go-containerregistry",
"github.com/open-policy-agent/opa",
"go.etcd.io/etcd/api/v3",
"go.etcd.io/etcd/client/v3",
"google.golang.org/grpc",
"google.golang.org/protobuf",
"helm.sh/helm/v3",
"k8s.io/api",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/code-generator",
"k8s.io/component-base",
"k8s.io/kube-openapi",
"k8s.io/kubectl",
"oras.land/oras-go",
"sigs.k8s.io/apiserver-network-proxy/konnectivity-client",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/kustomize/api",
"sigs.k8s.io/kustomize/kyaml"
],
"github.com/google/gofuzz": [
"github.com/containerd/containerd",
"github.com/json-iterator/go",
"helm.sh/helm/v3",
"k8s.io/api",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/code-generator",
"k8s.io/component-base",
"k8s.io/kube-openapi",
"k8s.io/kubectl",
"kubesphere.io/client-go",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/controller-tools",
"sigs.k8s.io/kustomize/kyaml",
"sigs.k8s.io/structured-merge-diff/v4"
],
"github.com/google/s2a-go": [
"github.com/spf13/afero"
],
"github.com/google/shlex": [
"helm.sh/helm/v3",
"k8s.io/cli-runtime",
"k8s.io/kubectl",
"sigs.k8s.io/kustomize/api"
],
"github.com/googleapis/enterprise-certificate-proxy": [
"github.com/spf13/afero"
],
"github.com/googleapis/gax-go/v2": [
"github.com/spf13/afero"
],
"github.com/gorilla/handlers": [
"helm.sh/helm/v3",
"oras.land/oras-go"
],
"github.com/gorilla/mux": [
"github.com/open-policy-agent/opa",
"helm.sh/helm/v3",
"oras.land/oras-go"
],
"github.com/gregjones/httpcache": [
"helm.sh/helm/v3",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/kubectl"
],
"github.com/grpc-ecosystem/go-grpc-prometheus": [
"github.com/containerd/containerd",
"go.etcd.io/etcd/client/v3",
"k8s.io/apiextensions-apiserver",
"k8s.io/apiserver"
],
"github.com/grpc-ecosystem/grpc-gateway": [
"go.etcd.io/etcd/api/v3",
"k8s.io/apiextensions-apiserver",
"k8s.io/apiserver"
],
"github.com/hashicorp/errwrap": [
"github.com/containerd/containerd",
"github.com/hashicorp/go-multierror",
"github.com/rubenv/sql-migrate",
"helm.sh/helm/v3"
],
"github.com/hashicorp/go-multierror": [
"github.com/rubenv/sql-migrate",
"helm.sh/helm/v3"
],
"github.com/hashicorp/golang-lru": [
"helm.sh/helm/v3"
],
"github.com/imdario/mergo": [
"github.com/rubenv/sql-migrate",
"sigs.k8s.io/controller-runtime"
],
"github.com/json-iterator/go": [
"github.com/containerd/containerd",
"github.com/emicklei/go-restful-openapi/v2",
"github.com/prometheus/client_golang",
"helm.sh/helm/v3",
"k8s.io/api",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/code-generator",
"k8s.io/component-base",
"k8s.io/kube-openapi",
"k8s.io/kubectl",
"kubesphere.io/kubesphere",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/controller-tools",
"sigs.k8s.io/structured-merge-diff/v4"
],
"github.com/klauspost/compress": [
"github.com/containerd/containerd",
"github.com/google/go-containerregistry",
"github.com/open-policy-agent/opa",
"github.com/prometheus/client_golang",
"go.mongodb.org/mongo-driver",
"helm.sh/helm/v3",
"oras.land/oras-go"
],
"github.com/mailru/easyjson": [
"github.com/go-openapi/jsonpointer",
"github.com/go-openapi/swag",
"helm.sh/helm/v3",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/code-generator",
"k8s.io/component-base",
"k8s.io/kube-openapi",
"k8s.io/kubectl",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/kustomize/api",
"sigs.k8s.io/kustomize/kyaml"
],
"github.com/miekg/dns": [
"github.com/open-policy-agent/opa",
"helm.sh/helm/v3"
],
"github.com/mitchellh/cli": [
"github.com/rubenv/sql-migrate"
],
"github.com/modern-go/concurrent": [
"github.com/containerd/containerd",
"github.com/json-iterator/go",
"github.com/prometheus/client_golang",
"helm.sh/helm/v3",
"k8s.io/api",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/code-generator",
"k8s.io/component-base",
"k8s.io/kube-openapi",
"k8s.io/kubectl",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/controller-tools",
"sigs.k8s.io/structured-merge-diff/v4"
],
"github.com/modern-go/reflect2": [
"github.com/containerd/containerd",
"github.com/json-iterator/go",
"github.com/prometheus/client_golang",
"helm.sh/helm/v3",
"k8s.io/api",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/code-generator",
"k8s.io/component-base",
"k8s.io/kube-openapi",
"k8s.io/kubectl",
"kubesphere.io/kubesphere",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/controller-tools"
],
"github.com/onsi/ginkgo": [
"sigs.k8s.io/application",
"sigs.k8s.io/controller-tools"
],
"github.com/pkg/errors": [
"github.com/containerd/containerd",
"github.com/evanphx/json-patch/v5",
"github.com/go-git/gcfg",
"github.com/google/go-containerregistry",
"go.mongodb.org/mongo-driver",
"gomodules.xyz/jsonpatch/v2",
"helm.sh/helm/v3",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/component-base",
"k8s.io/kubectl",
"kubesphere.io/kubesphere",
"kubesphere.io/utils",
"oras.land/oras-go",
"sigs.k8s.io/application",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/kustomize/api"
],
"github.com/planetscale/vtprotobuf": [
"github.com/spf13/afero",
"google.golang.org/grpc"
],
"github.com/xeipuuv/gojsonschema": [
"helm.sh/helm/v3"
],
"go.mongodb.org/mongo-driver": [
"github.com/go-openapi/strfmt"
],
"go.opencensus.io": [
"github.com/containerd/containerd",
"github.com/spf13/afero"
],
"golang.org/x/exp": [
"github.com/antlr4-go/antlr/v4",
"github.com/go-git/go-billy/v5",
"github.com/google/cel-go",
"github.com/rubenv/sql-migrate",
"k8s.io/apiextensions-apiserver",
"k8s.io/apiserver",
"kubesphere.io/kubesphere",
"sigs.k8s.io/controller-runtime"
],
"google.golang.org/api": [
"github.com/spf13/afero"
],
"google.golang.org/appengine": [
"github.com/containerd/containerd",
"github.com/google/go-containerregistry"
],
"google.golang.org/genproto": [
"github.com/containerd/containerd",
"github.com/spf13/afero",
"go.etcd.io/etcd/api/v3",
"go.etcd.io/etcd/client/v3",
"k8s.io/apiextensions-apiserver",
"k8s.io/apiserver"
],
"gopkg.in/square/go-jose.v2": [
"k8s.io/apiserver"
],
"gopkg.in/yaml.v2": [
"github.com/Masterminds/sprig/v3",
"github.com/containerd/containerd",
"github.com/go-openapi/loads",
"github.com/go-openapi/spec",
"github.com/go-openapi/validate",
"github.com/jmespath/go-jmespath/internal/testify",
"github.com/prometheus/client_golang",
"github.com/prometheus/common",
"github.com/rubenv/sql-migrate",
"github.com/sergi/go-diff",
"go.etcd.io/etcd/api/v3",
"go.etcd.io/etcd/client/v3",
"gopkg.in/cas.v2",
"helm.sh/helm/v3",
"oras.land/oras-go",
"sigs.k8s.io/application",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/controller-tools"
],
"gopkg.in/yaml.v3": [
"dario.cat/mergo",
"github.com/containerd/containerd",
"github.com/containerd/platforms",
"github.com/coreos/go-semver",
"github.com/cyphar/filepath-securejoin",
"github.com/go-git/go-billy/v5",
"github.com/go-git/go-git/v5",
"github.com/go-gorp/gorp/v3",
"github.com/go-jose/go-jose/v4",
"github.com/go-openapi/errors",
"github.com/go-openapi/jsonpointer",
"github.com/go-openapi/swag",
"github.com/go-task/slim-sprig/v3",
"github.com/google/gnostic-models",
"github.com/google/go-containerregistry",
"github.com/grpc-ecosystem/grpc-gateway/v2",
"github.com/onsi/ginkgo/v2",
"github.com/onsi/gomega",
"github.com/open-policy-agent/opa",
"github.com/pelletier/go-toml/v2",
"github.com/prometheus/common",
"github.com/sagikazarmark/locafero",
"github.com/sourcegraph/conc",
"github.com/spf13/cobra",
"github.com/spf13/viper",
"github.com/stretchr/testify",
"github.com/subosito/gotenv",
"go.etcd.io/etcd/client/pkg/v3",
"go.mongodb.org/mongo-driver",
"go.opentelemetry.io/auto/sdk",
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc",
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp",
"go.opentelemetry.io/otel",
"go.opentelemetry.io/otel/exporters/otlp/otlptrace",
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc",
"go.opentelemetry.io/otel/metric",
"go.opentelemetry.io/otel/sdk",
"go.opentelemetry.io/otel/trace",
"go.uber.org/multierr",
"go.uber.org/zap",
"helm.sh/helm/v3",
"k8s.io/apiextensions-apiserver",
"k8s.io/apimachinery",
"k8s.io/apiserver",
"k8s.io/cli-runtime",
"k8s.io/client-go",
"k8s.io/code-generator",
"k8s.io/component-base",
"k8s.io/kube-openapi",
"k8s.io/kubectl",
"kubesphere.io/kubesphere",
"oras.land/oras-go",
"sigs.k8s.io/controller-runtime",
"sigs.k8s.io/controller-tools",
"sigs.k8s.io/kustomize/api",
"sigs.k8s.io/kustomize/kyaml"
]
},
"unwantedVendored": [
"github.com/davecgh/go-spew",
"github.com/gogo/protobuf",
"github.com/golang/groupcache",
"github.com/golang/protobuf",
"github.com/google/gofuzz",
"github.com/google/shlex",
"github.com/gorilla/mux",
"github.com/gregjones/httpcache",
"github.com/grpc-ecosystem/go-grpc-prometheus",
"github.com/hashicorp/errwrap",
"github.com/hashicorp/go-multierror",
"github.com/json-iterator/go",
"github.com/klauspost/compress",
"github.com/mailru/easyjson",
"github.com/modern-go/concurrent",
"github.com/modern-go/reflect2",
"github.com/pkg/errors",
"github.com/xeipuuv/gojsonschema",
"go.mongodb.org/mongo-driver",
"golang.org/x/exp",
"gopkg.in/yaml.v2",
"gopkg.in/yaml.v3"
]
}
}

45
hack/update-gofmt.sh
View File

@@ -1,5 +1,19 @@
#!/usr/bin/env bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# GoFmt apparently is changing @ head...
set -o errexit
@@ -9,25 +23,22 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
kube::golang::verify_go_version
kube::golang::setup_env
cd "${KUBE_ROOT}"
find_files() {
find . -not \( \
\( \
-wholename './output' \
-o -wholename './.git' \
-o -wholename './_output' \
-o -wholename './_gopath' \
-o -wholename './release' \
-o -wholename './target' \
-o -wholename '*/third_party/*' \
-o -wholename '*/vendor/*' \
-o -wholename './staging/src/kubesphere.io/client-go/*vendor/*' \
-o -wholename './staging/src/kubesphere.io/api/*/zz_generated.deepcopy.go' \
\) -prune \
\) -name '*.go'
function git_find() {
# Similar to find but faster and easier to understand. We want to include
# modified and untracked files because this might be running against code
# which is not tracked by git yet.
git ls-files -cmo --exclude-standard \
':!:vendor/*' `# catches vendor/...` \
':!:*/vendor/*' `# catches any subdir/vendor/...` \
':!:third_party/*' `# catches third_party/...` \
':!:*/third_party/*' `# catches third_party/...` \
':!:*/testdata/*' `# catches any subdir/testdata/...` \
':(glob)**/*.go' \
"$@"
}
find_files | xargs gofmt -s -w
git_find -z | xargs -0 gofmt -s -w

18
hack/update-goimports.sh
View File

@@ -21,25 +21,17 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
kube::golang::verify_go_version
# Ensure that we find the binaries we build before anything else.
export GOBIN="${KUBE_OUTPUT_BINPATH}"
PATH="${GOBIN}:${PATH}"
# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on
kube::golang::setup_env
if ! command -v goimports ; then
# Install goimports
# Install goimports
echo 'installing goimports'
GO111MODULE=auto go install -mod=mod golang.org/x/tools/cmd/goimports@v0.7.0
go install -mod=mod golang.org/x/tools/cmd/goimports@v0.33.0
fi
cd "${KUBE_ROOT}" || exit 1
IFS=$'\n' read -r -d '' -a files < <( find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/client/*" -not -name "zz_generated.deepcopy.go" && printf '\0' )
IFS=$'\n' read -r -d '' -a files < <( find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./_output/*" -not -name "zz_generated.deepcopy.go" && printf '\0' )
"goimports" -w -local kubesphere.io/kubesphere "${files[@]}"
goimports -w -local kubesphere.io/kubesphere "${files[@]}"

47
hack/update-internal-modules.sh Executable file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
# These are "internal" modules. For various reasons, we want them to be
# decoupled from their parent modules.
MODULES=()
kube::util::read-array MODULES < <(
git ls-files -cmo --exclude-standard -- ':!:vendor/*' ':(glob)*/**/go.work' \
| while read -r F; do \
dirname "${F}"; \
done
)
# Detect problematic GOPROXY settings that prevent lookup of dependencies
if [[ "${GOPROXY:-}" == "off" ]]; then
kube::log::error "Cannot run hack/update-internal-modules.sh with \$GOPROXY=off"
exit 1
fi
kube::golang::setup_env
for mod in "${MODULES[@]}"; do
echo "=== tidying go.mod/go.sum in ${mod}"
echo "${KUBE_ROOT}/${mod}"
go -C "${KUBE_ROOT}/${mod}" mod edit -fmt
go -C "${KUBE_ROOT}/${mod}" mod tidy
done

31
hack/update-licenses.sh
View File

@@ -7,24 +7,23 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
if ! command -v license-eye &> /dev/null
then
# Ensure that we find the binaries we build before anything else.
export GOBIN="${KUBE_OUTPUT_BINPATH}"
PATH="${GOBIN}:${PATH}"
kube::golang::setup_env
# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on
# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
export GOFLAGS=
# Install skywalking-eyes
echo 'installing skywalking-eyes '
go install -mod=mod github.com/apache/skywalking-eyes/cmd/license-eye@v0.6.0
if ! command -v goimports ; then
# Install goimports
echo 'installing goimports'
go install -mod=mod github.com/apache/skywalking-eyes/cmd/license-eye@v0.7.0
fi
cd "${KUBE_ROOT}"
cd "${KUBE_ROOT}" || exit 1
echo 'running skywalking-eyes fix '
license-eye header fix
exit 0
mapfile -t files < <(git diff --name-only HEAD~1 | grep -v '^vendor/' | grep -E '\.go$' || true)
if [ "${#files[@]}" -eq 0 ]; then
echo "✅ No files changed."
exit 0
fi
license-eye header fix "${files[@]}"

16
hack/update-vendor-licenses.sh
View File

@@ -103,6 +103,7 @@ process_content () {
IFS=" " read -r -a local_files <<< "$(
for dir_root in ${package} ${package_root}; do
[[ -d ${DEPS_DIR}/${dir_root} ]] || continue
# One (set) of these is fine
find "${DEPS_DIR}/${dir_root}" \
-xdev -follow -maxdepth ${find_maxdepth} \
@@ -134,7 +135,8 @@ process_content () {
#############################################################################
# use modules, and use module info rather than the vendor dir for computing dependencies
export GO111MODULE=on
kube::golang::setup_env
export GOWORK=off
export GOFLAGS=-mod=mod
# Check bash version
@@ -200,8 +202,8 @@ for PACKAGE in ${modules}; do
# if there are no files vendored under this package...
if [[ -z "$(find "${DEPS_DIR}/${PACKAGE}" -mindepth 1 -maxdepth 1 -type f)" ]]; then
# and we have the same number of submodules as subdirectories...
if [[ "$(find "${DEPS_DIR}/${PACKAGE}/" -mindepth 1 -maxdepth 1 -type d | wc -l)" -gt 0 ]]; then
# and we have at least the same number of submodules as subdirectories...
if [[ "$(find "${DEPS_DIR}/${PACKAGE}/" -mindepth 1 -maxdepth 1 -type d | wc -l)" -le "$(echo "${modules}" | grep -cE "^${PACKAGE}/")" ]]; then
echo "Only submodules of ${PACKAGE} are vendored, skipping" >&2
continue
fi
@@ -229,6 +231,7 @@ for PACKAGE in ${modules}; do
if [[ -z "${file}" ]]; then
cat >&2 << __EOF__
No license could be found for ${PACKAGE} - aborting.
Options:
1. Check if the upstream repository has a newer version with LICENSE, COPYRIGHT and/or
COPYING files.
@@ -249,7 +252,14 @@ __EOF__
mv "${TMP_LICENSE_FILE}" "${dest_dir}/LICENSE"
done
# copy licenses for forked code from vendor and third_party directories
(cd "${KUBE_ROOT}" && \
find vendor third_party -iname 'licen[sc]e*' -o -iname 'notice*' -o -iname 'copying*' | \
grep -E 'third_party|forked' | \
xargs tar -czf - | tar -C "${TMP_LICENSES_DIR}" -xzf -)
# Leave things like OWNERS alone.
rm -f "${LICENSES_DIR}/LICENSE"
rm -rf "${LICENSES_DIR}/vendor"
rm -rf "${LICENSES_DIR}/third_party"
mv "${TMP_LICENSES_DIR}"/* "${LICENSES_DIR}"

447
hack/update-vendor.sh
View File

@@ -1,6 +1,18 @@
#!/usr/bin/env bash
# This script is a modified version of Kubernetes
# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
@@ -12,7 +24,12 @@ cd "$(pwd -P)"
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
# Explicitly opt into go modules, even though we're inside a GOPATH directory
# Get all the default Go environment.
kube::golang::setup_env
# Turn off workspaces until we are ready for them later
export GOWORK=off
# Explicitly opt into go modules
export GO111MODULE=on
# Explicitly set GOFLAGS to ignore vendor, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
export GOFLAGS=-mod=mod
@@ -25,65 +42,30 @@ if [[ "${GOPROXY:-}" == "off" ]]; then
exit 1
fi
kube::golang::verify_go_version
kube::util::require-jq
TMP_DIR="${TMP_DIR:-$(mktemp -d /tmp/update-vendor.XXXX)}"
LOG_FILE="${LOG_FILE:-${TMP_DIR}/update-vendor.log}"
kube::log::status "logfile at ${LOG_FILE}"
# Set up some FDs for this script to use, while capturing everything else to
# the log. NOTHING ELSE should write to $LOG_FILE directly.
exec 11>&1 # Real stdout, use this explicitly
exec 22>&2 # Real stderr, use this explicitly
exec 1>"${LOG_FILE}" # Automatic stdout
exec 2>&1 # Automatic stderr
set -x # Trace this script to stderr
go env # For the log
function finish {
ret=$?
if [[ ${ret} != 0 ]]; then
echo "An error has occurred. Please see more details in ${LOG_FILE}"
echo "An error has occurred. Please see more details in ${LOG_FILE}" >&22
fi
exit ${ret}
}
trap finish EXIT
if [ -z "${BASH_XTRACEFD:-}" ]; then
exec 19> "${LOG_FILE}"
export BASH_XTRACEFD="19"
set -x
fi
# ensure_require_replace_directives_for_all_dependencies:
# - ensures all existing 'require' directives have an associated 'replace' directive pinning a version
# - adds explicit 'require' directives for all transitive dependencies
# - adds explicit 'replace' directives for all require directives (existing 'replace' directives take precedence)
function ensure_require_replace_directives_for_all_dependencies() {
local local_tmp_dir
local_tmp_dir=$(mktemp -d "${TMP_DIR}/pin_replace.XXXX")
# collect 'require' directives that actually specify a version
local require_filter='(.Version != null) and (.Version != "v0.0.0") and (.Version != "v0.0.0-00010101000000-000000000000")'
# collect 'replace' directives that unconditionally pin versions (old=new@version)
local replace_filter='(.Old.Version == null) and (.New.Version != null)'
# Capture local require/replace directives before running any go commands that can modify the go.mod file
local require_json="${local_tmp_dir}/require.json"
local replace_json="${local_tmp_dir}/replace.json"
go mod edit -json \
| jq -r ".Require // [] | sort | .[] | select(${require_filter})" \
> "${require_json}"
go mod edit -json \
| jq -r ".Replace // [] | sort | .[] | select(${replace_filter})" \
> "${replace_json}"
# Propagate root replace/require directives into staging modules, in case we are downgrading, so they don't bump the root required version back up
for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/kubesphere.io/${repo}" >/dev/null 2>&1
jq -r '"-require \(.Path)@\(.Version)"' < "${require_json}" \
| xargs -L 100 go mod edit -fmt
jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"' < "${replace_json}" \
| xargs -L 100 go mod edit -fmt
popd >/dev/null 2>&1
done
# tidy to ensure require directives are added for indirect dependencies
go mod tidy
}
function print_go_mod_section() {
local directive="$1"
local file="$2"
@@ -160,197 +142,135 @@ function add_generated_comments() {
go mod edit -fmt
}
function add_staging_replace_directives() {
local path_to_staging_kubesphere_io="$1"
# Prune
go mod edit -json \
| jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' \
| xargs -L 100 go mod edit -fmt
go mod edit -json \
| jq -r '.Replace[]? | select(.New.Path | startswith("'"${path_to_staging_kubesphere_io}"'")) | "-dropreplace \(.Old.Path)"' \
| xargs -L 100 go mod edit -fmt
# Re-add
kube::util::list_staging_repos \
| while read -r X; do echo "-require kubesphere.io/${X}@v0.0.0"; done \
| xargs -L 100 go mod edit -fmt
kube::util::list_staging_repos \
| while read -r X; do echo "-replace kubesphere.io/${X}=${path_to_staging_kubesphere_io}/${X}"; done \
| xargs -L 100 go mod edit -fmt
}
# Phase 1: ensure go.mod files for staging modules and main module
for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/kubesphere.io/${repo}" >/dev/null 2>&1
if [[ ! -f go.mod ]]; then
kube::log::status "go.mod: initialize ${repo}"
rm -f Godeps/Godeps.json # remove before initializing, staging Godeps are not authoritative
go mod init "kubesphere.io/${repo}"
go mod edit -fmt
fi
popd >/dev/null 2>&1
done
if [[ ! -f go.mod ]]; then
kube::log::status "go.mod: initialize kubesphere.io/kubesphere"
go mod init "kubesphere.io/kubesphere"
rm -f Godeps/Godeps.json # remove after initializing
# === Capture go / godebug directives from root go.mod
go_directive_value=$(grep '^go 1.' go.mod | awk '{print $2}' || true)
if [[ -z "${go_directive_value}" ]]; then
kube::log::error "root go.mod must have 'go 1.x.y' directive" >&22 2>&1
exit 1
fi
# Phase 2: ensure staging repo require/replace directives
kube::log::status "go.mod: update staging references"
# Prune
go mod edit -json \
| jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' \
| xargs -L 100 go mod edit -fmt
go mod edit -json \
| jq -r '.Replace[]? | select(.New.Path | startswith("./staging/")) | "-dropreplace \(.Old.Path)"' \
| xargs -L 100 go mod edit -fmt
# Readd
kube::util::list_staging_repos \
| while read -r X; do echo "-require kubesphere.io/${X}@v0.0.0"; done \
| xargs -L 100 go mod edit -fmt
kube::util::list_staging_repos \
| while read -r X; do echo "-replace kubesphere.io/${X}=./staging/src/kubesphere.io/${X}"; done \
| xargs -L 100 go mod edit -fmt
# Phase 3: capture required (minimum) versions from all modules, and replaced (pinned) versions from the root module
# pin referenced versions
ensure_require_replace_directives_for_all_dependencies
# resolves/expands references in the root go.mod (if needed)
go mod tidy >>"${LOG_FILE}" 2>&1
# pin expanded versions
ensure_require_replace_directives_for_all_dependencies
# group require/replace directives
group_directives
# Phase 4: copy root go.mod to staging dirs and rewrite
kube::log::status "go.mod: propagate to staging modules"
for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/kubesphere.io/${repo}" >/dev/null 2>&1
echo "=== propagating to ${repo}" >> "${LOG_FILE}"
# copy root go.mod, changing module name
sed "s#module kubesphere.io/kubesphere#module kubesphere.io/${repo}#" \
< "${KUBE_ROOT}/go.mod" \
> "${KUBE_ROOT}/staging/src/kubesphere.io/${repo}/go.mod"
# remove `require` directives for staging components (will get re-added as needed by `go list`)
kube::util::list_staging_repos \
| while read -r X; do echo "-droprequire kubesphere.io/${X}"; done \
| xargs -L 100 go mod edit
# rewrite `replace` directives for staging components to point to peer directories
kube::util::list_staging_repos \
| while read -r X; do echo "-replace kubesphere.io/${X}=../${X}"; done \
| xargs -L 100 go mod edit
popd >/dev/null 2>&1
done
# Phase 5: sort and tidy staging components
kube::log::status "go.mod: sorting staging modules"
# tidy staging repos in reverse dependency order.
# the content of dependencies' go.mod files affects what `go mod tidy` chooses to record in a go.mod file.
tidy_unordered="${TMP_DIR}/tidy_unordered.txt"
kube::util::list_staging_repos \
| xargs -I {} echo "kubesphere.io/{}" > "${tidy_unordered}"
rm -f "${TMP_DIR}/tidy_deps.txt"
# SC2094 checks that you do not read and write to the same file in a pipeline.
# We do read from ${tidy_unordered} in the pipeline and mention it within the
# pipeline (but only ready it again) so we disable the lint to assure shellcheck
# that :this-is-fine:
# shellcheck disable=SC2094
while IFS= read -r repo; do
# record existence of the repo to ensure modules with no peer relationships still get included in the order
echo "${repo} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"
pushd "${KUBE_ROOT}/staging/src/${repo}" >/dev/null 2>&1
# save the original go.mod, since go list doesn't just add missing entries, it also removes specific required versions from it
tmp_go_mod="${TMP_DIR}/tidy_${repo/\//_}_go.mod.original"
tmp_go_deps="${TMP_DIR}/tidy_${repo/\//_}_deps.txt"
cp go.mod "${tmp_go_mod}"
{
echo "=== sorting ${repo}"
# 'go list' calculates direct imports and updates go.mod so that go list -m lists our module dependencies
echo "=== computing imports for ${repo}"
go list all
echo "=== computing tools imports for ${repo}"
go list -tags=tools all
} >> "${LOG_FILE}" 2>&1
# capture module dependencies
go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}"
# restore the original go.mod file
cp "${tmp_go_mod}" go.mod
# list all module dependencies
for dep in $(join "${tidy_unordered}" "${tmp_go_deps}"); do
# record the relationship (put dep first, because we want to sort leaves first)
echo "${dep} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"
# switch the required version to an explicit v0.0.0 (rather than an unknown v0.0.0-00010101000000-000000000000)
go mod edit -require "${dep}@v0.0.0"
done
popd >/dev/null 2>&1
done < "${tidy_unordered}"
kube::log::status "go.mod: tidying"
for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
pushd "${KUBE_ROOT}/staging/src/${repo}" >/dev/null 2>&1
echo "=== tidying ${repo}" >> "${LOG_FILE}"
# prune replace directives that pin to the naturally selected version.
# do this before tidying, since tidy removes unused modules that
# don't provide any relevant packages, which forgets which version of the
# unused transitive dependency we had a require directive for,
# and prevents pruning the matching replace directive after tidying.
go list -m -json all |
jq -r 'select(.Replace != null) |
select(.Path == .Replace.Path) |
select(.Version == .Replace.Version) |
"-dropreplace \(.Replace.Path)"' |
xargs -L 100 go mod edit -fmt
go mod tidy -v >>"${LOG_FILE}" 2>&1
# disallow transitive dependencies on kubesphere.io/kubesphere
loopback_deps=()
kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep kubesphere.io/kubesphere/ || true)
if [[ -n ${loopback_deps[*]:+"${loopback_deps[*]}"} ]]; then
kube::log::error "Disallowed ${repo} -> kubesphere.io/kubesphere dependencies exist via the following imports:
$(go mod why "${loopback_deps[@]}")"
exit 1
fi
# prune unused pinned replace directives
comm -23 \
<(go mod edit -json | jq -r '.Replace[] | .Old.Path' | sort) \
<(go list -m -json all | jq -r .Path | sort) |
while read -r X; do echo "-dropreplace=${X}"; done |
xargs -L 100 go mod edit -fmt
# prune replace directives that pin to the naturally selected version
go list -m -json all |
jq -r 'select(.Replace != null) |
select(.Path == .Replace.Path) |
select(.Version == .Replace.Version) |
"-dropreplace \(.Replace.Path)"' |
xargs -L 100 go mod edit -fmt
# group require/replace directives
group_directives
popd >/dev/null 2>&1
done
echo "=== tidying root" >> "${LOG_FILE}"
go mod tidy >>"${LOG_FILE}" 2>&1
# prune unused pinned non-local replace directives
comm -23 \
<(go mod edit -json | jq -r '.Replace[] | select(.New.Path | startswith("./") | not) | .Old.Path' | sort) \
<(go list -m -json all | jq -r .Path | sort) |
while read -r X; do echo "-dropreplace=${X}"; done |
xargs -L 100 go mod edit -fmt
# disallow transitive dependencies on kubesphere.io/kubesphere
loopback_deps=()
kube::util::read-array loopback_deps < <(go mod graph | grep ' kubesphere.io/kubesphere' || true)
if [[ -n ${loopback_deps[*]:+"${loopback_deps[*]}"} ]]; then
kube::log::error "Disallowed transitive kubesphere.io/kubesphere dependencies exist via the following imports:"
kube::log::error "${loopback_deps[@]}"
godebug_directive_value=$(grep 'godebug default=go' go.mod | awk '{print $2}' || true)
if [[ -z "${godebug_directive_value}" ]]; then
kube::log::error "root go.mod must have 'godebug default=go1.x' directive" >&22 2>&1
exit 1
fi
# Phase 6: add generated comments to go.mod files
kube::log::status "go.mod: adding generated comments"
# === Ensure staging go.mod files exist
for repo in $(kube::util::list_staging_repos); do
(
cd "staging/src/kubesphere.io/${repo}"
if [[ ! -f go.mod ]]; then
kube::log::status "go.mod: initialize ${repo}" >&11
go mod init "kubesphere.io/${repo}"
fi
go mod edit -go "${go_directive_value}" -godebug "${godebug_directive_value}"
)
done
# === Ensure root and staging go.mod files refer to each other using v0.0.0 and local path replaces
kube::log::status "go.mod: update staging module references" >&11
add_staging_replace_directives "./staging/src/kubesphere.io"
for repo in $(kube::util::list_staging_repos); do
(
cd "staging/src/kubesphere.io/${repo}"
add_staging_replace_directives ".."
)
done
# === Ensure all root and staging modules are included in go.work
kube::log::status "go.mod: go work use" >&11
(
cd "${KUBE_ROOT}"
unset GOWORK
unset GOFLAGS
if [[ ! -f go.work ]]; then
kube::log::status "go.work: initialize" >&11
go work init
fi
# Prune use directives
go work edit -json \
| jq -r '.Use[]? | "-dropuse \(.DiskPath)"' \
| xargs -L 100 go work edit -fmt
# Ensure go and godebug directives
go work edit -go "${go_directive_value}" -godebug "${godebug_directive_value}"
# Re-add use directives
go work use .
for repo in $(kube::util::list_staging_repos); do
go work use "./staging/src/kubesphere.io/${repo}"
done
)
# === Propagate MVS across all root / staging modules (calculated by `go work`) back into root / staging modules
kube::log::status "go.mod: go work sync" >&11
(
cd "${KUBE_ROOT}"
unset GOWORK
unset GOFLAGS
go work sync
)
# === Tidy
kube::log::status "go.mod: tidy" >&11
for repo in $(kube::util::list_staging_repos); do
(
echo "=== tidying kubesphere.io/${repo}"
cd "staging/src/kubesphere.io/${repo}"
go mod tidy -v
group_directives
)
done
echo "=== tidying root"
go mod tidy -v
group_directives
# === Prune unused replace directives, format modules
kube::log::status "go.mod: prune" >&11
for repo in $(kube::util::list_staging_repos); do
(
echo "=== pruning kubesphere.io/${repo}"
cd "staging/src/kubesphere.io/${repo}"
# drop all unused replace directives
comm -23 \
<(go mod edit -json | jq -r '.Replace[] | .Old.Path' | sort) \
<(go list -m -json all | jq -r 'select(.Main | not) | .Path' | sort) |
while read -r X; do echo "-dropreplace=${X}"; done |
xargs -L 100 go mod edit -fmt
group_directives
)
done
echo "=== pruning root"
# drop unused replace directives other than to local paths
comm -23 \
<(go mod edit -json | jq -r '.Replace[] | select(.New.Path | startswith("./") | not) | .Old.Path' | sort) \
<(go list -m -json all | jq -r 'select(.Main | not) | .Path' | sort) |
while read -r X; do echo "-dropreplace=${X}"; done |
xargs -L 100 go mod edit -fmt
group_directives
# === Add generated comments to go.mod files
kube::log::status "go.mod: adding generated comments" >&11
add_generated_comments "
// This is a generated file. Do not edit directly.
// Ensure you've carefully read
@@ -359,31 +279,60 @@ add_generated_comments "
// Run hack/update-vendor.sh to update go.mod files and the vendor directory.
"
for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/kubesphere.io/${repo}" >/dev/null 2>&1
(
cd "staging/src/kubesphere.io/${repo}"
add_generated_comments "// This is a generated file. Do not edit directly."
popd >/dev/null 2>&1
)
done
# === Update internal modules
kube::log::status "vendor: updating internal modules" >&11
hack/update-internal-modules.sh
# Phase 7: rebuild vendor directory
kube::log::status "vendor: running 'go mod vendor'"
go mod vendor >>"${LOG_FILE}" 2>&1
# create a symlink in vendor directory pointing to the staging components.
# This lets other packages and tools use the local staging components as if they were vendored.
for repo in $(kube::util::list_staging_repos); do
rm -fr "${KUBE_ROOT}/vendor/kubesphere.io/${repo}"
ln -s "../../staging/src/kubesphere.io/${repo}" "${KUBE_ROOT}/vendor/kubesphere.io/${repo}"
done
# === Rebuild vendor directory
(
kube::log::status "vendor: running 'go work vendor'" >&11
unset GOWORK
unset GOFLAGS
# rebuild go.work.sum
rm -f go.work.sum
go mod download
# rebuild vendor
go work vendor
)
kube::log::status "vendor: updating vendor/LICENSES"
hack/update-vendor-licenses.sh >>"${LOG_FILE}" 2>&1
kube::log::status "vendor: updating vendor/LICENSES" >&11
hack/update-vendor-licenses.sh
kube::log::status "vendor: creating OWNERS file"
kube::log::status "vendor: creating OWNERS file" >&11
rm -f "vendor/OWNERS"
cat <<__EOF__ > "vendor/OWNERS"
# See the OWNERS docs at https://github.com/kubesphere/kubesphere/blob/master/OWNERS
__EOF__
kube::log::status "NOTE: don't forget to handle vendor/* and LICENSE/* files that were added or removed"
# === Disallow transitive dependencies on kubesphere.io/kubesphere
kube::log::status "go.mod: prevent staging --> kubesphere.io/kubesphere dep" >&11
for repo in $(kube::util::list_staging_repos); do
(
echo "=== checking kubesphere.io/${repo}"
cd "staging/src/kubesphere.io/${repo}"
loopback_deps=()
kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep kubesphere.io/kubesphere/ || true)
if (( "${#loopback_deps[@]}" > 0 )); then
kube::log::error "${#loopback_deps[@]} disallowed ${repo} -> kubesphere.io/kubesphere dependencies exist via the following imports: $(go mod why "${loopback_deps[@]}")" >&22 2>&1
exit 1
fi
)
done
kube::log::status "go.mod: prevent kubesphere.io/kubesphere --> * --> kubesphere.io/kubesphere dep" >&11
loopback_deps=()
kube::util::read-array loopback_deps < <(go mod graph | grep ' kubesphere.io/kubesphere' || true)
if (( "${#loopback_deps[@]}" > 0 )); then
kube::log::error "${#loopback_deps[@]} disallowed transitive kubesphere.io/kubesphere dependencies exist via the following imports:" >&22 2>&1
kube::log::error "${loopback_deps[@]}" >&22 2>&1
exit 1
fi
kube::log::status "NOTE: don't forget to handle vendor/* and LICENSE/* files that were added or removed" >&11

6
hack/verify-all.sh
View File

@@ -25,11 +25,9 @@ source "${KUBE_ROOT}/hack/lib/util.sh"
# Excluded check patterns are always skipped.
EXCLUDED_PATTERNS=(
"verify-all.sh" # this script calls the make rule and would cause a loop
"verify-linkcheck.sh" # runs in separate Jenkins job once per day due to high network usage
"verify-*-dockerized.sh" # Don't run any scripts that intended to be run dockerized
"verify-govet-levee.sh" # Do not run levee analysis by default while KEP-1933 implementation is in alpha.
# "verify-licenses.sh"
"verify-licenses.sh"
"verify-shellcheck.sh"
"verify-vendor-licenses.sh"
)
while IFS='' read -r line; do EXCLUDED_CHECKS+=("$line"); done < <(ls "${EXCLUDED_PATTERNS[@]/#/${KUBE_ROOT}/hack/}" 2>/dev/null || true)

23
hack/verify-gofmt.sh
View File

@@ -1,5 +1,19 @@
#!/usr/bin/env bash
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script checks whether the source code needs to be formatted or not by
# `gofmt`. Run `hack/update-gofmt.sh` to actually format sources.
#
@@ -16,21 +30,18 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
cd "${KUBE_ROOT}"
kube::golang::verify_go_version
kube::golang::setup_env
find_files() {
find . -not \( \
\( \
-wholename './output' \
-o -wholename './.git' \
-wholename './.git' \
-o -wholename './_output' \
-o -wholename './_gopath' \
-o -wholename './release' \
-o -wholename './target' \
-o -wholename '*/third_party/*' \
-o -wholename '*/vendor/*' \
-o -wholename './staging/src/kubesphere.io/client-go/*vendor/*' \
-o -wholename './staging/src/kubesphere.io/api/*/zz_generated.deepcopy.go' \
-o -wholename '*/testdata/*' \
-o -wholename '*/bindata.go' \
\) -prune \
\) -name '*.go'

20
hack/verify-goimports.sh
View File

@@ -20,26 +20,24 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
kube::golang::verify_go_version
# Detect problematic GOPROXY settings that prevent lookup of dependencies
if [[ "${GOPROXY:-}" == "off" ]]; then
kube::log::error "Cannot run with \$GOPROXY=off"
exit 1
fi
# Ensure that we find the binaries we build before anything else.
export GOBIN="${KUBE_OUTPUT_BINPATH}"
PATH="${GOBIN}:${PATH}"
# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on
kube::golang::setup_env
if ! command -v goimports ; then
# Install goimports
# Install goimports
echo 'installing goimports'
GO111MODULE=auto go install -mod=mod golang.org/x/tools/cmd/goimports@v0.7.0
go install -mod=mod golang.org/x/tools/cmd/goimports@v0.33.0
fi
cd "${KUBE_ROOT}" || exit 1
IFS=$'\n' read -r -d '' -a files < <( find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/apis/*" -not -path "./pkg/client/*" -not -name "zz_generated.deepcopy.go" && printf '\0' )
IFS=$'\n' read -r -d '' -a files < <( find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./_output/*" -not -name "zz_generated.deepcopy.go" && printf '\0' )
output=$(goimports -local kubesphere.io/kubesphere -l "${files[@]}")

14
hack/verify-golangci-lint.sh
View File

@@ -10,21 +10,13 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
kube::golang::verify_go_version
# Ensure that we find the binaries we build before anything else.
export GOBIN="${KUBE_OUTPUT_BINPATH}"
PATH="${GOBIN}:${PATH}"
# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on
kube::golang::setup_env
if ! command -v golangci-lint ; then
# Install golangci-lint
# Install golangci-lint
echo 'installing golangci-lint'
GO111MODULE=auto go install -mod=mod github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8
go install -mod=mod github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8
fi
cd "${KUBE_ROOT}"

136
hack/verify-licenses.sh
View File

@@ -1,30 +1,134 @@
#!/usr/bin/env bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Usage: `hack/verify-licenses.sh`.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
if ! command -v license-eye &> /dev/null
then
# Ensure that we find the binaries we build before anything else.
export GOBIN="${KUBE_OUTPUT_BINPATH}"
PATH="${GOBIN}:${PATH}"
# This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
kube::golang::setup_env
kube::util::ensure-temp-dir
# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on
# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
export GOFLAGS=
ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
mkdir -p "$ARTIFACTS/logs/"
# Install skywalking-eyes
echo 'installing skywalking-eyes '
go install -mod=mod github.com/apache/skywalking-eyes/cmd/license-eye@v0.6.0
# Creating a new repository tree
# Deleting vendor directory to make go-licenses fetch license URLs from go-packages source repository
git worktree add -f "${KUBE_TEMP}"/tmp_test_licenses/kubernetes HEAD >/dev/null 2>&1 || true
cd "${KUBE_TEMP}"/tmp_test_licenses/kubernetes && rm -rf vendor
# Ensure that we find the binaries we build before anything else.
export GOBIN="${KUBE_OUTPUT_BIN}"
PATH="${GOBIN}:${PATH}"
function http_code() {
curl -I -s -o /dev/null -w "%{http_code}" "$1"
}
packages_flagged=()
packages_url_missing=()
exit_code=0
# Install go-licenses
echo '[INFO] Installing go-licenses...'
go install github.com/google/go-licenses@latest
# Fetching CNCF Approved List Of Licenses
# Refer: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md
curl -s 'https://spdx.org/licenses/licenses.json' -o "${ARTIFACTS}"/licenses.json
echo '[INFO] Fetching current list of CNCF approved licenses...'
jq -r '.licenses[] | select(.isDeprecatedLicenseId==false) .licenseId' "${ARTIFACTS}"/licenses.json | sort | uniq > "${ARTIFACTS}"/licenses.txt
# Scanning go-packages under the project & verifying against the CNCF approved list of licenses
echo '[INFO] Starting license scan on go-packages...'
go-licenses report ./... >> "${ARTIFACTS}"/licenses.csv 2>"${ARTIFACTS}"/logs/go-licenses.log
echo -e 'PACKAGE_NAME LICENSE_NAME LICENSE_URL\n' >> "${ARTIFACTS}"/approved_licenses.dump
while IFS=, read -r GO_PACKAGE LICENSE_URL LICENSE_NAME; do
if ! grep -q "^${LICENSE_NAME}$" "${ARTIFACTS}"/licenses.txt; then
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${ARTIFACTS}"/notapproved_licenses.dump
packages_flagged+=("${GO_PACKAGE}")
continue
fi
if [[ "${LICENSE_URL}" == 'Unknown' ]]; then
if [[ "${GO_PACKAGE}" != kubesphere.io/* ]]; then
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${ARTIFACTS}"/approved_licenses_with_missing_urls.dump
packages_url_missing+=("${GO_PACKAGE}")
else
LICENSE_URL='https://github.com/kubesphere/kubesphere/blob/master/LICENSE'
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${ARTIFACTS}"/approved_licenses.dump
fi
continue
fi
if [[ "$(http_code "${LICENSE_URL}")" != 404 ]]; then
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${ARTIFACTS}"/approved_licenses.dump
continue
fi
# The URL 404'ed. Try parent-paths.
#echo -e "DBG: err 404 ${LICENSE_URL}"
dir="$(dirname "${LICENSE_URL}")"
file="$(basename "${LICENSE_URL}")"
while [[ "${dir}" != "." ]]; do
dir="$(dirname "${dir}")"
#echo "DBG: try ${dir}/${file}"
if [[ "$(http_code "${dir}/${file}")" != 404 ]]; then
#echo "DBG: it worked"
echo "${GO_PACKAGE} ${LICENSE_NAME} ${dir}/${file}" >> "${ARTIFACTS}"/approved_licenses.dump
break
fi
#echo "DBG: still 404"
done
if [[ "${dir}" == "." ]];then
#echo "DBG: failed to find a license"
packages_url_missing+=("${GO_PACKAGE}")
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${ARTIFACTS}"/approved_licenses_with_missing_urls.dump
fi
done < "${ARTIFACTS}"/licenses.csv
awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${ARTIFACTS}"/approved_licenses.dump
if [[ ${#packages_url_missing[@]} -gt 0 ]]; then
echo -e '\n[ERROR] The following go-packages in the project have unknown or unreachable license URL:'
awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${ARTIFACTS}"/approved_licenses_with_missing_urls.dump
exit_code=1
fi
cd "${KUBE_ROOT}"
echo 'running skywalking-eyes check '
license-eye header check
exit 0
if [[ ${#packages_flagged[@]} -gt 0 ]]; then
echo -e "\n[ERROR] The following go-packages in the project are using non-CNCF approved licenses. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md"
awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${ARTIFACTS}"/notapproved_licenses.dump
exit_code=1
elif [[ "${exit_code}" -eq 1 ]]; then
echo -e "\n[ERROR] Project is using go-packages with unknown or unreachable license URLs. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md"
else
echo -e "\n[SUCCESS] Scan complete! All go-packages under the project are using current CNCF approved licenses!"
fi
exit "${exit_code}"

12
hack/verify-spelling.sh
View File

@@ -25,18 +25,14 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
export KUBE_ROOT
source "${KUBE_ROOT}/hack/lib/init.sh"
# Ensure that we find the binaries we build before anything else.
export GOBIN="${KUBE_OUTPUT_BINPATH}"
PATH="${GOBIN}:${PATH}"
kube::golang::setup_env
# Install tools we need
if ! command -v misspell ; then
# As GOFLAGS may equal to `-mod=vendor`, we must download the modules to vendor or go install will fail.
GO111MODULE=on go install -mod=mod github.com/client9/misspell/cmd/misspell@v0.3.4
# Install misspell
echo 'installing misspell'
go install -mod=mod github.com/golangci/misspell/cmd/misspell@v0.7.0
fi
cd "${KUBE_ROOT}"
# Spell checking
# All the skipping files are defined in hack/.spelling_failures
skipping_file="${KUBE_ROOT}/hack/.spelling_failures"

30
hack/verify-vendor-licenses.sh Executable file
View File

@@ -0,0 +1,30 @@
#!/usr/bin/env bash
# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script checks whether updating of licenses files is needed
# or not. We should run `hack/update-vendor-licenses.sh` and commit the results,
# if actually updates them.
# Usage: `hack/verify-vendor-licenses.sh`.
set -o errexit
set -o nounset
set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/verify-generated.sh"
kube::verify::generated "Generated files need to be updated" "Please run 'hack/update-vendor-licenses.sh'" hack/update-vendor-licenses.sh "$@"