Skip to content

Commit

Permalink
Merge pull request #1199 from justaugustus/vdf
Browse files Browse the repository at this point in the history 
[VDF] anago: Publish container images on K8s Infra (k8s-staging-kubernetes)
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Apr 1, 2020
2 parents 5748d59 + 9df2381 commit 01df93c
Show file tree
Hide file tree
Showing 2 changed files with 134 additions and 45 deletions.
62 changes: 36 additions & 26 deletions anago
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,12 +259,11 @@ copy_logs_to_workdir () {
}

###############################################################################
# Ensures all registries that will be used during both mock and --nomock
# runs allow write access so we don't fall over later
# @param registries - A space separated list of registries
# Ensures we have write access to a specified registry
# @param registry - A registry to check the ACLs for
#
ensure_registry_acls () {
local registries=($1)
local registry="$1"
local emptyfile="$TMPDIR/empty-file.$$"
local gs_path
local r
Expand All @@ -276,27 +275,30 @@ ensure_registry_acls () {

# Short of creating a hardcoded map of project-id to registry, translating
# _ to - seems to be a simple rule to keep this, well, simple.
for r in ${registries[*]//_/-}; do
# In this context, "google-containers" is still used
if [[ "$r" == "$GCRIO_PATH_PROD" ]]; then
artifact_namespace="google-containers"
else
artifact_namespace="${r/gcr.io\//}"
fi
r=${registry//_/-}

gs_path="gs://artifacts.$artifact_namespace.appspot.com/containers"
logecho -n "Checking write access to registry $r: "
if logrun $GSUTIL -q cp $emptyfile $gs_path && \
logrun $GSUTIL -q rm $gs_path/${emptyfile##*/}; then
logecho $OK
else
logecho $FAILED
((retcode++))
fi
# When we are no-mock mode we need to perform an image promotion, so it's
# unnecessary to check for write access to the production container registry.
if ((FLAGS_nomock)); then
logecho -n "Skipping container registry ACL check on $GCRIO_PATH_PROD in no-mock mode: "
logecho $OK
return 0
else
artifact_namespace="${r/gcr.io\//}"
fi

# Always reset back to $USER
((FLAGS_gcb)) || logrun $GCLOUD config set account $GCP_USER
done
gs_path="gs://artifacts.$artifact_namespace.appspot.com/containers"
logecho -n "Checking write access to registry $r: "
if logrun $GSUTIL -q cp $emptyfile $gs_path && \
logrun $GSUTIL -q rm $gs_path/${emptyfile##*/}; then
logecho $OK
else
logecho $FAILED
((retcode++))
fi

# Always reset back to $USER
((FLAGS_gcb)) || logrun $GCLOUD config set account $GCP_USER

logrun rm -f $emptyfile

Expand Down Expand Up @@ -378,7 +380,7 @@ check_prerequisites () {

# Verify write access to all container registries that might be used
# to ensure both mock and --nomock runs will work.
ensure_registry_acls "${ALL_CONTAINER_REGISTRIES[*]}" || return 1
ensure_registry_acls "$GCRIO_PATH" || return 1

logecho -n "Checking cloud project state: "
GCLOUD_PROJECT=$($GCLOUD config get-value project 2>/dev/null)
Expand Down Expand Up @@ -1444,8 +1446,16 @@ push_all_artifacts () {
gs://$RELEASE_BUCKET/$BUCKET_TYPE/$version || return 1
fi

common::runstep release::docker::release \
$KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1
# When we are no-mock mode we need to perform an image promotion, so
# instead of pushing to the production container registry, we validate
# that the manifest is populated on the remote registry.
if ! ((FLAGS_nomock)); then
common::runstep release::docker::release \
$KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1
fi

common::runstep release::docker::validate_remote_manifests \
$KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1

common::runstep release::gcs::publish_version \
$BUCKET_TYPE $version $BUILD_OUTPUT-$version $RELEASE_BUCKET || return 1
Expand Down
117 changes: 98 additions & 19 deletions lib/releaselib.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ readonly CI_BUCKET="kubernetes-release-dev"

# TODO(vdf): Need to reference K8s Infra registries here
readonly GCRIO_PATH_PROD="k8s.gcr.io"
# TODO(vdf): Remove all GCRIO_PATH_PROD_PUSH logic once the k8s.gcr.io vanity
# domain flip (VDF) is successful
readonly GCRIO_PATH_PROD_PUSH="gcr.io/google-containers"
readonly GCRIO_PATH_TEST="gcr.io/$TEST_PROJECT"
readonly GCRIO_PATH_TEST="gcr.io/k8s-staging-kubernetes"

readonly KUBE_CROSS_REGISTRY="us.gcr.io/k8s-artifacts-prod/build-image"
readonly KUBE_CROSS_IMAGE="${KUBE_CROSS_REGISTRY}/kube-cross"
Expand Down Expand Up @@ -650,7 +652,6 @@ release::gcs::locally_stage_release_artifacts() {
# --release-kind used by push-build.sh
local release_kind=${4:-"kubernetes"}
local platform
local platforms
local release_stage=$build_output/release-stage
local release_tars=$build_output/release-tars
local gcs_stage=$build_output/gcs-stage/$version
Expand Down Expand Up @@ -745,7 +746,7 @@ release::gcs::locally_stage_release_artifacts() {

# Upload the "naked" binaries to GCS. This is useful for install scripts that
# download the binaries directly and don't need tars.
platforms=($(cd "$release_stage/client"; echo *))
mapfile -t platforms < <(find "${release_stage}/client" -maxdepth 1 -mindepth 1 -type f -exec basename {} \;)
for platform in "${platforms[@]}"; do
src="$release_stage/client/$platform/$release_kind/client/bin/*"
dst="bin/${platform/-//}/"
Expand Down Expand Up @@ -1020,30 +1021,31 @@ release::gcs::publish () {
# @param build_output - build output directory
# @return 1 on failure
release::docker::release () {
local registry=$1
local push_registry=$registry
local version=$2
local build_output=$3
local release_images=$build_output/release-images
local docker_target
local registry="$1"
local push_registry="$registry"
local version="$2"
local build_output="$3"
local release_images="$build_output/release-images"
local arch
local -a arches
local tarfile
local orig_tag
local -a new_tags
local new_tag
local binary
local -A manifest_images

common::argc_validate 3

# TODO(vdf): Remove all GCRIO_PATH_PROD_PUSH logic once the k8s.gcr.io vanity
# domain flip (VDF) is successful
if [[ "$registry" == "$GCRIO_PATH_PROD" ]]; then
# Switch to the push alias if using the $GCRIO_PATH_PROD alias
push_registry="$GCRIO_PATH_PROD_PUSH"
fi

logecho "Send docker containers from release-images to $push_registry..."

arches=($(cd "$release_images"; echo *))
for arch in ${arches[@]}; do
mapfile -t arches < <(find "${release_images}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)
for arch in "${arches[@]}"; do
for tarfile in $release_images/$arch/*.tar; do
# There may be multiple tags; just get the first
orig_tag=$(tar xf $tarfile manifest.json -O | jq -r '.[0].RepoTags[0]')
Expand Down Expand Up @@ -1099,6 +1101,89 @@ release::docker::release () {
return 0
}

# TODO(vdf): Consider collapsing this into release::docker::release and renaming
# that function AFTER the k8s.gcr.io Vanity Domain Flip (VDF).
###############################################################################
# Validates that image manifests have been pushed to a specified remote registry.
# Uses 'skopeo inspect'.
#
# @param registry - docker registry
# @param version - version tag
# @param build_output - build output directory
# @return 1 on failure
release::docker::validate_remote_manifests () {
local registry="$1"
local push_registry="$registry"
local version="$2"
local build_output="$3"
local release_images="$build_output/release-images"
local arch
local tarfile
local orig_tag
local new_tag
local binary
local -A manifest_images

common::argc_validate 3

# TODO(vdf): Remove all GCRIO_PATH_PROD_PUSH logic once the k8s.gcr.io vanity
# domain flip (VDF) is successful
if [[ "$registry" == "$GCRIO_PATH_PROD" ]]; then
# Switch to the push alias if using the $GCRIO_PATH_PROD alias
push_registry="$GCRIO_PATH_PROD_PUSH"
fi

logecho "Validating image manifests in $push_registry..."

mapfile -t arches < <(find "${release_images}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)
for arch in "${arches[@]}"; do
for tarfile in $release_images/$arch/*.tar; do
# There may be multiple tags; just get the first
orig_tag=$(tar xf $tarfile manifest.json -O | jq -r '.[0].RepoTags[0]')
if [[ ! "$orig_tag" =~ ^.+/(.+):.+$ ]]; then
logecho "$FAILED: malformed tag in $tarfile:"
logecho $orig_tag
return 1
fi
binary=${BASH_REMATCH[1]}

new_tag="$push_registry/${binary/-$arch/}"
manifest_images["${new_tag}"]+=" $arch"
done
done

for image in "${!manifest_images[@]}"; do
local archs
local manifest
local digest

logecho "Validating manifest list exists for ${image}:${version}..."

archs=$(echo "${manifest_images[$image]}" | sed -e 's/^[[:space:]]*//')

if ! manifest=$(skopeo inspect "docker://${image}:${version}" --raw); then
logecho "Could not find manifest list for ${image}:${version}"
return 1
fi

for arch in ${archs}; do
logecho "Checking image digest for ${image} on ${arch} architecture..."

digest=$(echo -n "${manifest}" \
| jq --arg a "${arch}" -r '.manifests[] | select(.platform.architecture==$a)' \
| jq -r '.digest')

if [[ -n "$digest" ]]; then
logecho "Digest for ${image} on ${arch}: ${digest}"
else
logecho "Could not find the image digest for ${image} on ${arch}. Exiting..."
return 1
fi
done
done

return 0
}

###############################################################################
# Get the kubecross image version for a given release branch.
Expand Down Expand Up @@ -1277,9 +1362,6 @@ release::send_announcement () {
# READ_RELEASE_BUCKETS - array of readable buckets for multiple sourcing of
# mock staged builds
# GCRIO_PATH - GCR path based on mock or --nomock
# ALL_CONTAINER_REGISTRIES - when running mock (via GCB) this array also
# contains k8s.gcr.io so we can check access in mock
# mode before an actual release occurs
release::set_globals () {
logecho -n "Setting global variables: "

Expand All @@ -1306,7 +1388,6 @@ release::set_globals () {
fi

GCRIO_PATH="${FLAGS_gcrio_path:-$GCRIO_PATH_TEST}"
ALL_CONTAINER_REGISTRIES=("$GCRIO_PATH")

if ((FLAGS_nomock)); then
RELEASE_BUCKET="$PROD_BUCKET"
Expand Down Expand Up @@ -1339,8 +1420,6 @@ release::set_globals () {
WRITE_RELEASE_BUCKETS=("$RELEASE_BUCKET")
READ_RELEASE_BUCKETS+=("$RELEASE_BUCKET")

ALL_CONTAINER_REGISTRIES=("$GCRIO_PATH")

# TODO:
# These KUBE_ globals extend beyond the scope of the new release refactored
# tooling so to pass these through as flags will require fixes across
Expand Down

0 comments on commit 01df93c

Please sign in to comment.