#!/usr/bin/env bash
# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
# Go tools really don't like it if you have a symlink in `pwd`.
cd "$(pwd -P)"
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"
# Get all the default Go environment.
kube::golang::setup_env
# Turn off workspaces until we are ready for them later
export GOWORK=off
# Explicitly opt into go modules
export GO111MODULE=on
# Explicitly set GOFLAGS to ignore vendor, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
export GOFLAGS=-mod=mod
# Ensure sort order doesn't depend on locale
export LANG=C
export LC_ALL=C
# Detect problematic GOPROXY settings that prevent lookup of dependencies
if [[ "${GOPROXY:-}" == "off" ]]; then
kube::log::error "Cannot run hack/update-vendor.sh with \$GOPROXY=off"
exit 1
fi
kube::util::require-jq
TMP_DIR="${TMP_DIR:-$(mktemp -d /tmp/update-vendor.XXXX)}"
LOG_FILE="${LOG_FILE:-${TMP_DIR}/update-vendor.log}"
kube::log::status "logfile at ${LOG_FILE}"
# Set up some FDs for this script to use, while capturing everything else to
# the log. NOTHING ELSE should write to $LOG_FILE directly.
exec 11>&1 # Real stdout, use this explicitly
exec 22>&2 # Real stderr, use this explicitly
exec 1>"${LOG_FILE}" # Automatic stdout
exec 2>&1 # Automatic stderr
set -x # Trace this script to stderr
go env # For the log
function finish {
ret=$?
if [[ ${ret} != 0 ]]; then
echo "An error has occurred. Please see more details in ${LOG_FILE}" >&22
fi
exit ${ret}
}
trap finish EXIT
function print_go_mod_section() {
local directive="$1"
local file="$2"
if [ -s "${file}" ]; then
echo "${directive} ("
cat "$file"
echo ")"
fi
}
function group_directives() {
local local_tmp_dir
local_tmp_dir=$(mktemp -d "${TMP_DIR}/group_replace.XXXX")
local go_mod_require_direct="${local_tmp_dir}/go.mod.require_direct.tmp"
local go_mod_require_indirect="${local_tmp_dir}/go.mod.require_indirect.tmp"
local go_mod_replace="${local_tmp_dir}/go.mod.replace.tmp"
local go_mod_other="${local_tmp_dir}/go.mod.other.tmp"
# separate replace and non-replace directives
awk "
# print lines between 'require (' ... ')' lines
/^require [(]/ { inrequire=1; next }
inrequire && /^[)]/ { inrequire=0; next }
inrequire && /\/\/ indirect/ { print > \"${go_mod_require_indirect}\"; next }
inrequire { print > \"${go_mod_require_direct}\"; next }
# print lines between 'replace (' ... ')' lines
/^replace [(]/ { inreplace=1; next }
inreplace && /^[)]/ { inreplace=0; next }
inreplace { print > \"${go_mod_replace}\"; next }
# print ungrouped replace directives with the replace directive trimmed
/^replace [^(]/ { sub(/^replace /,\"\"); print > \"${go_mod_replace}\"; next }
# print ungrouped require directives with the require directive trimmed
/^require [^(].*\/\/ indirect/ { sub(/^require /,\"\"); print > \"${go_mod_require_indirect}\"; next }
/^require [^(]/ { sub(/^require /,\"\"); print > \"${go_mod_require_direct}\"; next }
# otherwise print to the other file
{ print > \"${go_mod_other}\" }
" < go.mod
{
cat "${go_mod_other}";
print_go_mod_section "require" "${go_mod_require_direct}"
print_go_mod_section "require" "${go_mod_require_indirect}"
print_go_mod_section "replace" "${go_mod_replace}"
} > go.mod
go mod edit -fmt
}
function add_generated_comments() {
local local_tmp_dir
local_tmp_dir=$(mktemp -d "${TMP_DIR}/add_generated_comments.XXXX")
local go_mod_nocomments="${local_tmp_dir}/go.mod.nocomments.tmp"
# drop comments before the module directive
awk "
BEGIN { dropcomments=1 }
/^module / { dropcomments=0 }
dropcomments && /^\/\// { next }
{ print }
" < go.mod > "${go_mod_nocomments}"
# Add the specified comments
local comments="${1}"
{
echo "${comments}"
echo ""
cat "${go_mod_nocomments}"
} > go.mod
# Format
go mod edit -fmt
}
function add_staging_replace_directives() {
local path_to_staging_k8s_io="$1"
# Prune
go mod edit -json \
| jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' \
| xargs -L 100 go mod edit -fmt
go mod edit -json \
| jq -r '.Replace[]? | select(.New.Path | startswith("'"${path_to_staging_k8s_io}"'")) | "-dropreplace \(.Old.Path)"' \
| xargs -L 100 go mod edit -fmt
# Re-add
kube::util::list_staging_repos \
| while read -r X; do echo "-require k8s.io/${X}@v0.0.0"; done \
| xargs -L 100 go mod edit -fmt
kube::util::list_staging_repos \
| while read -r X; do echo "-replace k8s.io/${X}=${path_to_staging_k8s_io}/${X}"; done \
| xargs -L 100 go mod edit -fmt
}
# === Capture go / godebug directives from root go.mod
go_directive_value=$(grep '^go 1.' go.mod | awk '{print $2}' || true)
if [[ -z "${go_directive_value}" ]]; then
kube::log::error "root go.mod must have 'go 1.x.y' directive" >&22 2>&1
exit 1
fi
godebug_directive_value=$(grep 'godebug default=go' go.mod | awk '{print $2}' || true)
if [[ -z "${godebug_directive_value}" ]]; then
kube::log::error "root go.mod must have 'godebug default=go1.x' directive" >&22 2>&1
exit 1
fi
# === Ensure staging go.mod files exist
for repo in $(kube::util::list_staging_repos); do
(
cd "staging/src/k8s.io/${repo}"
if [[ ! -f go.mod ]]; then
kube::log::status "go.mod: initialize ${repo}" >&11
go mod init "k8s.io/${repo}"
fi
go mod edit -go "${go_directive_value}" -godebug "${godebug_directive_value}"
)
done
# === Ensure root and staging go.mod files refer to each other using v0.0.0 and local path replaces
kube::log::status "go.mod: update staging module references" >&11
add_staging_replace_directives "./staging/src/k8s.io"
for repo in $(kube::util::list_staging_repos); do
(
cd "staging/src/k8s.io/${repo}"
add_staging_replace_directives ".."
)
done
# === Ensure all root and staging modules are included in go.work
kube::log::status "go.mod: go work use" >&11
(
cd "${KUBE_ROOT}"
unset GOWORK
unset GOFLAGS
if [[ ! -f go.work ]]; then
kube::log::status "go.work: initialize" >&11
go work init
fi
# Prune use directives
go work edit -json \
| jq -r '.Use[]? | "-dropuse \(.DiskPath)"' \
| xargs -L 100 go work edit -fmt
# Ensure go and godebug directives
go work edit -go "${go_directive_value}" -godebug "${godebug_directive_value}"
# Re-add use directives
go work use .
for repo in $(kube::util::list_staging_repos); do
go work use "./staging/src/k8s.io/${repo}"
done
)
# === Propagate MVS across all root / staging modules (calculated by `go work`) back into root / staging modules
kube::log::status "go.mod: go work sync" >&11
(
cd "${KUBE_ROOT}"
unset GOWORK
unset GOFLAGS
go work sync
)
# === Tidy
kube::log::status "go.mod: tidy" >&11
for repo in $(kube::util::list_staging_repos); do
(
echo "=== tidying k8s.io/${repo}"
cd "staging/src/k8s.io/${repo}"
go mod tidy -v
group_directives
)
done
echo "=== tidying root"
go mod tidy -v
group_directives
# === Prune unused replace directives, format modules
kube::log::status "go.mod: prune" >&11
for repo in $(kube::util::list_staging_repos); do
(
echo "=== pruning k8s.io/${repo}"
cd "staging/src/k8s.io/${repo}"
# drop all unused replace directives
comm -23 \
<(go mod edit -json | jq -r '.Replace[] | .Old.Path' | sort) \
<(go list -m -json all | jq -r 'select(.Main | not) | .Path' | sort) |
while read -r X; do echo "-dropreplace=${X}"; done |
xargs -L 100 go mod edit -fmt
group_directives
)
done
echo "=== pruning root"
# drop unused replace directives other than to local paths
comm -23 \
<(go mod edit -json | jq -r '.Replace[] | select(.New.Path | startswith("./") | not) | .Old.Path' | sort) \
<(go list -m -json all | jq -r 'select(.Main | not) | .Path' | sort) |
while read -r X; do echo "-dropreplace=${X}"; done |
xargs -L 100 go mod edit -fmt
group_directives
# === Add generated comments to go.mod files
kube::log::status "go.mod: adding generated comments" >&11
add_generated_comments "
// This is a generated file. Do not edit directly.
// Ensure you've carefully read
// https://git.k8s.io/community/contributors/devel/sig-architecture/vendor.md
// Run hack/pin-dependency.sh to change pinned dependency versions.
// Run hack/update-vendor.sh to update go.mod files and the vendor directory.
"
for repo in $(kube::util::list_staging_repos); do
(
cd "staging/src/k8s.io/${repo}"
add_generated_comments "// This is a generated file. Do not edit directly."
)
done
# === Update internal modules
kube::log::status "vendor: updating internal modules" >&11
hack/update-internal-modules.sh
# === Rebuild vendor directory
(
kube::log::status "vendor: running 'go work vendor'" >&11
unset GOWORK
unset GOFLAGS
# rebuild go.work.sum
rm -f go.work.sum
go mod download
# rebuild vendor
go work vendor
)
kube::log::status "vendor: updating vendor/LICENSES" >&11
hack/update-vendor-licenses.sh
kube::log::status "vendor: creating OWNERS file" >&11
rm -f "vendor/OWNERS"
cat <<__EOF__ > "vendor/OWNERS"
# See the OWNERS docs at https://go.k8s.io/owners
options:
# make root approval non-recursive
no_parent_owners: true
approvers:
- dep-approvers
reviewers:
- dep-reviewers
__EOF__
# === Disallow transitive dependencies on k8s.io/kubernetes
kube::log::status "go.mod: prevent staging --> k8s.io/kubernetes dep" >&11
for repo in $(kube::util::list_staging_repos); do
(
echo "=== checking k8s.io/${repo}"
cd "staging/src/k8s.io/${repo}"
loopback_deps=()
kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true)
if (( "${#loopback_deps[@]}" > 0 )); then
kube::log::error "${#loopback_deps[@]} disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports: $(go mod why "${loopback_deps[@]}")" >&22 2>&1
exit 1
fi
)
done
kube::log::status "go.mod: prevent k8s.io/kubernetes --> * --> k8s.io/kubernetes dep" >&11
loopback_deps=()
kube::util::read-array loopback_deps < <(go mod graph | grep ' k8s.io/kubernetes' || true)
if (( "${#loopback_deps[@]}" > 0 )); then
kube::log::error "${#loopback_deps[@]} disallowed transitive k8s.io/kubernetes dependencies exist via the following imports:" >&22 2>&1
kube::log::error "${loopback_deps[@]}" >&22 2>&1
exit 1
fi
kube::log::status "NOTE: don't forget to handle vendor/* and LICENSE/* files that were added or removed" >&11
Codebase Browser
kubernetes