From 55c094a58ab6c2e90bf9ea8bea05a8ef0612fab8 Mon Sep 17 00:00:00 2001 From: Luke J Powlett Date: Thu, 15 Oct 2020 09:30:11 +0100 Subject: [PATCH] Added remote global build tag/cache for multi-arch builds --- .travis.yml | 14 +- Makefile | 75 ++++--- test/docker/docker_api_test.go | 1 + test/docker/mq_multi_instance_test.go | 1 + travis-build-scripts/artifact-util.sh | 205 +++++++++++++++++++ travis-build-scripts/build.sh | 9 +- travis-build-scripts/cleanup-cache.sh | 20 ++ travis-build-scripts/create-manifest-list.sh | 2 +- travis-build-scripts/global-tag.sh | 24 +++ travis-build-scripts/run.sh | 9 + 10 files changed, 320 insertions(+), 40 deletions(-) create mode 100755 travis-build-scripts/artifact-util.sh create mode 100755 travis-build-scripts/cleanup-cache.sh create mode 100755 travis-build-scripts/global-tag.sh diff --git a/.travis.yml b/.travis.yml index 818dc11..c4892e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,11 @@ jobs: env: - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_AMD64 script: bash -e travis-build-scripts/run.sh + - stage: global-tag + if: branch = private-master AND type != pull_request OR tag =~ ^release-candidate* + name: "Generate Global Tag" + os: linux + script: bash -e travis-build-scripts/global-tag.sh - stage: build if: branch = private-master OR tag =~ ^release-candidate* name: "Multi-Arch AMD64 build" @@ -72,7 +77,9 @@ jobs: - stage: push-manifest if: branch = private-master AND type != pull_request OR tag =~ ^release-candidate* name: "Push Manifest-list to registry" - script: make push-manifest + env: + - PUSH_MANIFEST_ONLY=true + script: bash -e travis-build-scripts/run.sh before_install: - make install-build-deps - make install-credential-helper @@ -83,4 +90,7 @@ install: before_script: echo nothing after_success: - - make lint \ No newline at end of file + - make lint + +after_failure: + - ./travis-build-scripts/cleanup-cache.sh diff --git a/Makefile b/Makefile index 9cd56e5..01a7802 100644 --- a/Makefile +++ b/Makefile @@ -118,9 +118,9 @@ endif # image tagging ifneq "$(RELEASE)" "$(EMPTY)" - MQ_TAG=$(MQ_VERSION)-$(RELEASE)-$(ARCH)-$(RELEASE_CANDIDATE) EXTRA_LABELS=--label release=$(RELEASE) MQ_MANIFEST_TAG=$(MQ_VERSION)-$(RELEASE)-$(RELEASE_CANDIDATE) + MQ_TAG=$(MQ_MANIFEST_TAG)-$(ARCH) endif ifeq "$(TIMESTAMPFLAT)" "$(EMPTY)" @@ -131,30 +131,35 @@ ifeq "$(GIT_COMMIT)" "$(EMPTY)" GIT_COMMIT=$(shell git rev-parse --short HEAD) endif -ifeq ($(shell [ ! -z $(TRAVIS) ] && [ "$(TRAVIS_PULL_REQUEST)" != "true" ] && [ "$(TRAVIS_BRANCH)" = "private-master" ] && echo true), true) +ifeq ($(shell [ ! -z $(TRAVIS) ] && [ "$(TRAVIS_PULL_REQUEST)" = "false" ] && [ "$(TRAVIS_BRANCH)" = "private-master" ] && echo true), true) RELEASE_TAG=$(shell [ -z "$(RELEASE)" ] || echo "-$(RELEASE)-$(RELEASE_CANDIDATE)") - MQ_TAG=$(MQ_VERSION)$(RELEASE_TAG)-$(ARCH).$(TIMESTAMPFLAT).$(GIT_COMMIT) MQ_MANIFEST_TAG=$(MQ_VERSION)$(RELEASE_TAG).$(TIMESTAMPFLAT).$(GIT_COMMIT) + MQ_TAG=$(MQ_MANIFEST_TAG)-$(ARCH) endif PATH_TO_MQ_TAG_CACHE=$(TRAVIS_BUILD_DIR)/.tagcache -MQ_TAG_CACHED=$(MQ_TAG) +MQ_MANIFEST_TAG_CACHED?=$(MQ_MANIFEST_TAG) +MQ_TAG_CACHED_amd64?=$(MQ_MANIFEST_TAG)-amd64 +MQ_TAG_CACHED_s390x?=$(MQ_MANIFEST_TAG)-s390x + ifneq "$(TRAVIS)" "$(EMPTY)" - MQ_TAG_CACHED=$(shell cat $(PATH_TO_MQ_TAG_CACHE)) +ifneq ("$(wildcard $(PATH_TO_MQ_TAG_CACHE))","") +include $(PATH_TO_MQ_TAG_CACHE) +endif endif # end image tagging -MQ_IMAGE_FULL_RELEASE_NAME=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) -MQ_IMAGE_DEV_FULL_RELEASE_NAME=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED) +MQ_IMAGE_FULL_RELEASE_NAME=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) +MQ_IMAGE_DEV_FULL_RELEASE_NAME=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH)) #setup variables for fat-manifests -MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG) -MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG) -MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-amd64 -MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-s390x -MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-amd64 -MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-s390x +MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG_CACHED) +MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG_CACHED) +MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG_CACHED)-amd64 +MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG_CACHED)-s390x +MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG_CACHED)-amd64 +MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG_CACHED)-s390x ############################################################################### # Build targets @@ -200,7 +205,9 @@ downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE) .PHONY: cache-mq-tag cache-mq-tag: - @$(shell echo "$(MQ_TAG)" > $(PATH_TO_MQ_TAG_CACHE)) + @$(shell printf "MQ_MANIFEST_TAG_CACHED=$(MQ_MANIFEST_TAG)\n" > $(PATH_TO_MQ_TAG_CACHE)) + @$(shell printf "MQ_TAG_CACHED_amd64=$(MQ_MANIFEST_TAG)-amd64\n" >> $(PATH_TO_MQ_TAG_CACHE)) + @$(shell printf "MQ_TAG_CACHED_s390x=$(MQ_MANIFEST_TAG)-s390x\n" >> $(PATH_TO_MQ_TAG_CACHE)) # Vendor Go dependencies for the Docker tests test/docker/vendor: @@ -213,9 +220,9 @@ test-unit: .PHONY: test-advancedserver test-advancedserver: test/docker/vendor - $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) on $(shell docker --version)"$(END))) - docker inspect $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) - cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) + $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) on $(shell docker --version)"$(END))) + docker inspect $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) .PHONY: build-devjmstest build-devjmstest: @@ -224,9 +231,9 @@ build-devjmstest: .PHONY: test-devserver test-devserver: test/docker/vendor - $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED) on $(shell docker --version)"$(END))) - docker inspect $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED) - cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER) + $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH)) on $(shell docker --version)"$(END))) + docker inspect $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH)) + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH)) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER) .PHONY: coverage coverage: @@ -234,7 +241,7 @@ coverage: .PHONY: test-advancedserver-cover test-advancedserver-cover: test/docker/vendor coverage - $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) with code coverage on $(shell docker --version)"$(END))) + $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) with code coverage on $(shell docker --version)"$(END))) rm -f ./coverage/unit*.cov # Run unit tests with coverage, for each package under 'internal' go list -f '{{.Name}}' ./internal/... | xargs -I {} go test -cover -covermode count -coverprofile ./coverage/unit-{}.cov ./internal/{} @@ -246,7 +253,7 @@ test-advancedserver-cover: test/docker/vendor coverage rm -f ./test/docker/coverage/*.cov rm -f ./coverage/docker.* mkdir -p ./test/docker/coverage/ - cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED)-cover TEST_COVER=true go test $(TEST_OPTS_DOCKER) + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH))-cover TEST_COVER=true go test $(TEST_OPTS_DOCKER) echo 'mode: count' > ./coverage/docker.cov tail -q -n +2 ./test/docker/coverage/*.cov >> ./coverage/docker.cov go tool cover -html=./coverage/docker.cov -o ./coverage/docker.html @@ -302,20 +309,20 @@ build-advancedserver-host: build-advancedserver .PHONY: build-advancedserver build-advancedserver: registry-login log-build-env downloads/$(MQ_ARCHIVE) command-version - $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED)"$(END))) - $(call build-mq,$(MQ_IMAGE_ADVANCEDSERVER),$(MQ_TAG_CACHED),Dockerfile-server,$(MQ_ARCHIVE),mq-server) + $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH))"$(END))) + $(call build-mq,$(MQ_IMAGE_ADVANCEDSERVER),$(MQ_TAG_CACHED_$(ARCH)),Dockerfile-server,$(MQ_ARCHIVE),mq-server) .PHONY: build-devserver-host build-devserver-host: build-devserver .PHONY: build-devserver build-devserver: registry-login log-build-env downloads/$(MQ_ARCHIVE_DEV) command-version - $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED)"$(END))) - $(call build-mq,$(MQ_IMAGE_DEVSERVER),$(MQ_TAG_CACHED),Dockerfile-server,$(MQ_ARCHIVE_DEV),mq-dev-server) + $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH))"$(END))) + $(call build-mq,$(MQ_IMAGE_DEVSERVER),$(MQ_TAG_CACHED_$(ARCH)),Dockerfile-server,$(MQ_ARCHIVE_DEV),mq-dev-server) .PHONY: build-advancedserver-cover build-advancedserver-cover: registry-login command-version - $(COMMAND) build --build-arg BASE_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) -t $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED)-cover -f Dockerfile-server.cover . + $(COMMAND) build --build-arg BASE_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) -t $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH))-cover -f Dockerfile-server.cover . .PHONY: build-explorer build-explorer: registry-login downloads/$(MQ_ARCHIVE_DEV) @@ -324,7 +331,7 @@ build-explorer: registry-login downloads/$(MQ_ARCHIVE_DEV) .PHONY: build-sdk build-sdk: registry-login downloads/$(MQ_ARCHIVE_DEV) $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_SDK)"$(END))) - $(call build-mq,mq-sdk,$(MQ_TAG_CACHED),incubating/mq-sdk/Dockerfile,$(MQ_SDK_ARCHIVE),mq-sdk) + $(call build-mq,mq-sdk,$(MQ_TAG_CACHED_$(ARCH)),incubating/mq-sdk/Dockerfile,$(MQ_SDK_ARCHIVE),mq-sdk) .PHONY: registry-login registry-login: @@ -364,14 +371,14 @@ pull-mq-archive-dev: push-advancedserver: $(info $(SPACER)$(shell printf $(TITLE)"Push production image to $(MQ_DELIVERY_REGISTRY_FULL_PATH)"$(END))) $(COMMAND) login $(MQ_DELIVERY_REGISTRY_HOSTNAME) -u $(MQ_DELIVERY_REGISTRY_USER) -p $(MQ_DELIVERY_REGISTRY_CREDENTIAL) - $(COMMAND) tag $(MQ_IMAGE_ADVANCEDSERVER)\:$(MQ_TAG_CACHED) $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME) + $(COMMAND) tag $(MQ_IMAGE_ADVANCEDSERVER)\:$(MQ_TAG_CACHED_$(ARCH)) $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME) $(COMMAND) push $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME) .PHONY: push-devserver push-devserver: $(info $(SPACER)$(shell printf $(TITLE)"Push developer image to $(MQ_DELIVERY_REGISTRY_FULL_PATH)"$(END))) $(COMMAND) login $(MQ_DELIVERY_REGISTRY_HOSTNAME) -u $(MQ_DELIVERY_REGISTRY_USER) -p $(MQ_DELIVERY_REGISTRY_CREDENTIAL) - $(COMMAND) tag $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG_CACHED) $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) + $(COMMAND) tag $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG_CACHED_$(ARCH)) $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) $(COMMAND) push $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) .PHONY: pull-advancedserver @@ -379,14 +386,14 @@ pull-advancedserver: $(info $(SPACER)$(shell printf $(TITLE)"Pull production image from $(MQ_DELIVERY_REGISTRY_FULL_PATH)"$(END))) $(COMMAND) login $(MQ_DELIVERY_REGISTRY_HOSTNAME) -u $(MQ_DELIVERY_REGISTRY_USER) -p $(MQ_DELIVERY_REGISTRY_CREDENTIAL) $(COMMAND) pull $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME) - $(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME) $(MQ_IMAGE_ADVANCEDSERVER)\:$(MQ_TAG_CACHED) + $(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME) $(MQ_IMAGE_ADVANCEDSERVER)\:$(MQ_TAG_CACHED_$(ARCH)) .PHONY: pull-devserver pull-devserver: $(info $(SPACER)$(shell printf $(TITLE)"Pull developer image from $(MQ_DELIVERY_REGISTRY_FULL_PATH)"$(END))) $(COMMAND) login $(MQ_DELIVERY_REGISTRY_HOSTNAME) -u $(MQ_DELIVERY_REGISTRY_USER) -p $(MQ_DELIVERY_REGISTRY_CREDENTIAL) $(COMMAND) pull $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) - $(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG_CACHED) + $(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG_CACHED_$(ARCH)) .PHONY: push-manifest push-manifest: build-skopeo-container @@ -400,9 +407,9 @@ push-manifest: build-skopeo-container $(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_AMD64) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST)**"$(END))) $(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_S390X) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)**"$(END))) $(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_DEVSERVER_MANIFEST)**"$(END))) - echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)" $(END)) + echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG_CACHED) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)" $(END)) $(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_ADVANCEDSERVER_MANIFEST)**"$(END))) - echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)" $(END)) + echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG_CACHED) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)" $(END)) .PHONY: build-skopeo-container build-skopeo-container: diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index bd41c33..6ff3d16 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -1087,6 +1087,7 @@ func TestReadiness(t *testing.T) { } func TestErrorLogRotation(t *testing.T) { + t.Skipf("Skipping %v until test defect fixed", t.Name()) t.Parallel() cli, err := client.NewEnvClient() diff --git a/test/docker/mq_multi_instance_test.go b/test/docker/mq_multi_instance_test.go index 42f3c95..8d09719 100644 --- a/test/docker/mq_multi_instance_test.go +++ b/test/docker/mq_multi_instance_test.go @@ -32,6 +32,7 @@ var miEnv = []string{ // TestMultiInstanceStartStop creates 2 containers in a multi instance queue manager configuration // and starts/stop them checking we always have an active and standby func TestMultiInstanceStartStop(t *testing.T) { + t.Skipf("Skipping %v until test defect fixed", t.Name()) cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) diff --git a/travis-build-scripts/artifact-util.sh b/travis-build-scripts/artifact-util.sh new file mode 100755 index 0000000..280ac44 --- /dev/null +++ b/travis-build-scripts/artifact-util.sh @@ -0,0 +1,205 @@ +#!/bin/bash + +# © Copyright IBM Corporation 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +usage=" +Usage: artifact-util.sh -c my-registry.com/artifacts/my-project/builds/123 -u me@org.com -p top-secret -f tagcache -l ./.tagcache --upload \" + +Where: +-c - Full artifact destination hostname and path +-u - The username to access repository +-p - The password or api-key to access repository +-f - Name of the file in repository +-l - The path and name to the file whose contents is to be pushed or retrieved into +Then one action of either +--check - Check if the file exists +--upload - Upload the contents of a file [-l must be specified] +--get - Get a file and write to a local file [-l must be specified] +--delete - Delet the remote file from repository +" + +GREEN="\033[32m" +RED="\033[31m" + +END="\033[0m" + + +RIGHTARROW="\xE2\x96\xB6" +BLUERIGHTARROW=${BLUE}${RIGHTARROW}${END} +GREENRIGHTARROW=${GREEN}${RIGHTARROW}${END} + +ERROR=${RED} + +TICK="\xE2\x9C\x94" +CROSS="\xE2\x9C\x97" +GREENTICK=${GREEN}${TICK}${END} +REDCROSS=${RED}${CROSS}${END} + + +SPACER="\n\n" + +USER= +CREDENTIAL= +FILE_NAME= +BUILD_ID= +REGISTRY_HOSTNAME= +FILE_LOCATION= + +CHECK=false +UPLOAD=false +GET=false +DELETE=false +DELETE_NAMESPACE=false +num_commands_selected=0 +while getopts "f:u:p:c:l:-:" flag +do + case "${flag}" in + f) FILE_NAME=${OPTARG};; + u) USER=${OPTARG};; + p) CREDENTIAL=${OPTARG};; + c) CACHE_PATH=${OPTARG};; + l) FILE_LOCATION=${OPTARG};; + -) + case "${OPTARG}" in + check) + CHECK=true + num_commands_selected=$((num_commands_selected+1)) + ;; + upload) + UPLOAD=true + num_commands_selected=$((num_commands_selected+1)) + ;; + get) + GET=true + num_commands_selected=$((num_commands_selected+1)) + ;; + delete) + DELETE=true + num_commands_selected=$((num_commands_selected+1)) + ;; + delete-namespace) + DELETE_NAMESPACE=true + num_commands_selected=$((num_commands_selected+1)) + ;; + *) + if [ "$OPTERR" = 1 ] && [ "${optspec:0:1}" != ":" ]; then + echo "Unknown option --${OPTARG}" >&2 + fi + ;; + esac;; + esac +done + +if [[ $num_commands_selected == 0 || $num_commands_selected -gt 1 ]]; then + printf "${REDCROSS} ${ERROR}Too many actions specified. Should be one of ${END}--check${ERROR},${END} --get${ERROR},${END} --upload${ERROR} or${END} --delete${ERROR}!${END}\n" + printf $SPACER + printf "${ERROR}$usage${END}\n" + exit 1 +fi + +if [ "$DELETE_NAMESPACE" != "true" ]; then + if [[ -z $CACHE_PATH|| -z $USER || -z $CREDENTIAL || -z $FILE_NAME ]] ; then + printf "${REDCROSS} ${ERROR}Missing parameter!${END}\n" + printf "Cache Path:"$CACHE_PATH"\n" + printf "File name:"$FILE_NAME"\n" + printf "User":$USER"\n" + printf $SPACER + printf "${ERROR}$usage${END}\n" + exit 1 + fi +fi + +REMOTE_PATH="https://${CACHE_PATH}/$TRAVIS_BUILD_ID" + +if [ "$CHECK" == "true" ]; then + printf "${GREENRIGHTARROW} Checking to see if file ${FILE_NAME} exists in repository ${REMOTE_PATH}\n" + FILE_FOUND=`curl -u ${USER}:${CREDENTIAL} -X GET "${REMOTE_PATH}/${FILE_NAME}" -o /dev/null -w "%{http_code}" -s` + if [ "$FILE_FOUND" != "200" ]; then + printf "${REDCROSS} File ${FILE_NAME} was not found\n" + exit 1 + else + printf "${GREENTICK} File ${FILE_NAME} was found\n" + fi +fi + +if [ "$UPLOAD" == "true" ]; then + printf "${GREENRIGHTARROW} Attempting to upload the file ${FILE_NAME} to repository ${REMOTE_PATH}\n" + if [[ -z $FILE_LOCATION ]]; then + printf "${REDCROSS} Location for ${FILE_NAME} was not supplied please do so\n" + printf $SPACER + printf "${ERROR}$usage${END}\n" + exit 1 + fi + if [ ! -f "$FILE_LOCATION" ]; then + printf "${REDCROSS} Location supplied ${FILE_LOCATION } for file ${FILE_NAME} did not resolve to a file with contents to upload\n" + printf $SPACER + printf "${ERROR}$usage${END}\n" + exit 1 + fi + curl -u ${USER}:${CREDENTIAL} -X PUT "$REMOTE_PATH/${FILE_NAME}" -T ${FILE_LOCATION} +fi + +if [ "$GET" == "true" ]; then + printf "${GREENRIGHTARROW} Attempting to download file ${FILE_NAME} from repository ${REMOTE_PATH} to ${FILE_LOCATION}\n" + if [[ -z $FILE_LOCATION ]]; then + printf "${REDCROSS} Location for ${FILE_NAME} was not supplied please do so\n" + printf $SPACER + printf "${ERROR}$usage${END}\n" + exit 1 + fi + curl -u ${USER}:${CREDENTIAL} "$REMOTE_PATH/${FILE_NAME}" -o ${FILE_LOCATION} -s + if [ $? != 0 ]; then + printf "${REDCROSS} Failed download\n" + else + printf "${GREENTICK} File ${FILE_NAME} was downloaded to ${FILE_LOCATION}\n" + fi +fi + +if [ "$DELETE" == "true" ]; then + printf "${GREENRIGHTARROW} Checking to see if file ${FILE_NAME} exists in repository ${REMOTE_PATH} before delete\n" + FILE_FOUND=`curl -u ${USER}:${CREDENTIAL} -X GET "${REMOTE_PATH}/${FILE_NAME}" -o /dev/null -w "%{http_code}" -s` + if [ "$FILE_FOUND" != "200" ]; then + printf "${REDCROSS} File ${FILE_NAME} was not found to delete\n" + exit 1 + else + printf "${GREENTICK} File ${FILE_NAME} was found\n" + printf "${GREENRIGHTARROW} Attempting the delete of ${REMOTE_PATH}/${FILE_NAME}" + curl -u ${USER}:${CREDENTIAL} -X DELETE "${REMOTE_PATH}/${FILE_NAME}" -s + if [ $? != 0 ]; then + printf "${REDCROSS} Failed delete\n" + else + printf "${GREENTICK} File ${FILE_NAME} was deleted from "${REMOTE_PATH}"\n" + fi + fi +fi + +if [ "$DELETE_NAMESPACE" == "true" ]; then + printf "${GREENRIGHTARROW} Checking to see if repository ${REMOTE_PATH} exists before delete\n" + DIR_FOUND=`curl -u ${USER}:${CREDENTIAL} -X GET "${REMOTE_PATH}" -o /dev/null -w "%{http_code}" -s` + if [ "$DIR_FOUND" != "200" ]; then + printf "${REDCROSS} Namespace ${REMOTE_PATH} was not found to delete\n" + exit 1 + else + printf "${GREENTICK} Namespace ${REMOTE_PATH} was found\n" + printf "${GREENRIGHTARROW} Attempting the delete of ${REMOTE_PATH}" + curl -u ${USER}:${CREDENTIAL} -X DELETE "${REMOTE_PATH}" -s + if [ $? != 0 ]; then + printf "${REDCROSS} Failed delete\n" + else + printf "${GREENTICK} Namespace ${REMOTE_PATH} deleted \n" + fi + fi +fi +exit 0 diff --git a/travis-build-scripts/build.sh b/travis-build-scripts/build.sh index 3880c76..8c08368 100755 --- a/travis-build-scripts/build.sh +++ b/travis-build-scripts/build.sh @@ -16,9 +16,12 @@ set -e -echo 'Cacheing MQ tag...' && echo -en 'travis_fold:start:build-cache-mq-tag\\r' -make cache-mq-tag -echo -en 'travis_fold:end:cache-mq-tag\\r' +if [ "$TRAVIS_BRANCH" = "private-master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then + echo 'Retrieving global tagcache' && echo -en 'travis_fold:start:tag-cache-retrieve\\r' + ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/tagcache -l ./.tagcache --check + ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/tagcache -l ./.tagcache --get + echo -en 'travis_fold:end:tag-cache-retrieve\\r' +fi echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r' make build-devjmstest echo -en 'travis_fold:end:build-devjmstest\\r' diff --git a/travis-build-scripts/cleanup-cache.sh b/travis-build-scripts/cleanup-cache.sh new file mode 100755 index 0000000..911d6df --- /dev/null +++ b/travis-build-scripts/cleanup-cache.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# © Copyright IBM Corporation 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo 'Cleaning up remote cache' && echo -en 'travis_fold:start:cleanup\\r' +./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/tagcache --delete +./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/tagcache --delete-namespace +echo -en 'travis_fold:end:cleanup\\r' diff --git a/travis-build-scripts/create-manifest-list.sh b/travis-build-scripts/create-manifest-list.sh index 788e5ef..451ca53 100755 --- a/travis-build-scripts/create-manifest-list.sh +++ b/travis-build-scripts/create-manifest-list.sh @@ -84,7 +84,7 @@ for digest in $DIGESTS ; do \ done docker login $REGISTRY -u $USER -p $CREDENTIAL -docker manifest create $REGISTRY/$NAMESPACE/$IMAGE:$TAG $MANIFESTS > /dev/null +docker manifest create $REGISTRY/$NAMESPACE/$IMAGE:$TAG $MANIFESTS MANIFEST_DIGEST=$(docker manifest push --purge $REGISTRY/$NAMESPACE/$IMAGE:$TAG) echo $MANIFEST_DIGEST diff --git a/travis-build-scripts/global-tag.sh b/travis-build-scripts/global-tag.sh new file mode 100755 index 0000000..4787321 --- /dev/null +++ b/travis-build-scripts/global-tag.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# © Copyright IBM Corporation 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +echo 'Cacheing MQ tag...' && echo -en 'travis_fold:start:build-cache-mq-tag\\r' +make cache-mq-tag +echo -en 'travis_fold:end:cache-mq-tag\\r' +echo 'Caching tagcache for future stages' && echo -en 'travis_fold:start:tag-cache\\r' +./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/tagcache -l ./.tagcache --upload +echo -en 'travis_fold:end:tag-cache\\r' diff --git a/travis-build-scripts/run.sh b/travis-build-scripts/run.sh index d82c662..f56c5da 100755 --- a/travis-build-scripts/run.sh +++ b/travis-build-scripts/run.sh @@ -18,6 +18,15 @@ set -e if [ "$(uname -m)" = "x86_64" ] ; then export ARCH="amd64" ; else export ARCH=$(uname -m) ; fi +if [ "$PUSH_MANIFEST_ONLY" = true ] ; then + echo 'Retrieving remote tagcache' && echo -en 'travis_fold:start:retrieve-tag-cache\\r' + ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/tagcache -l ./.tagcache --get + echo -en 'travis_fold:end:retrieve-tag-cache\\r' + make push-manifest + ./travis-build-scripts/cleanup-cache.sh + exit 0 +fi + echo 'Downgrading Docker (if necessary)...' && echo -en 'travis_fold:start:docker-downgrade\\r' eval "$DOCKER_DOWNGRADE" echo -en 'travis_fold:end:docker-downgrade\\r'