From dd1d534045a6286bc130dca2edabec2a80c09c30 Mon Sep 17 00:00:00 2001 From: Luke J Powlett Date: Tue, 17 Nov 2020 13:33:29 +0000 Subject: [PATCH] Added support for internal build levels * Dummy change to see if things are working * Initial internal build support * Add SSH key support * Add SSH key support pt 2 * Add SSH key support pt 3 * Add Push support * Full package support * Correct typo * Correct typo 2 * Disable some tests as per master * Change rsync variable names * Fix build issues * Remove the -full package support for now Co-authored-by: Jason Edmeades --- Makefile | 31 +++++++++++++++++++++------ travis-build-scripts/build.sh | 40 ++++++++++++++++++++++++----------- travis-build-scripts/push.sh | 4 ++-- travis-build-scripts/run.sh | 12 +++++++++-- travis-build-scripts/test.sh | 34 +++++++++++++++++++---------- 5 files changed, 88 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index aa3ced7..f9d230f 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ MQ_ARCHIVE_DEV ?= $(MQ_VERSION)-IBM-MQ-Advanced-for-Developers-Non-Install-$(MQ_ # MQ_SDK_ARCHIVE specifies the archive to use for building the golang programs. Defaults vary on developer or advanced. MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) # Options to `go test` for the Docker tests -TEST_OPTS_DOCKER ?= +TEST_OPTS_DOCKER ?= # Timeout for the Docker tests TEST_TIMEOUT_DOCKER ?= 30m # MQ_IMAGE_ADVANCEDSERVER is the name of the built MQ Advanced image @@ -113,7 +113,7 @@ endif # LTS_TAG is the tag modifier for an LTS container build LTS_TAG= -ifeq "$(LTS)" "true" +ifeq "$(LTS)" "true" ifneq "$(LTS_TAG_OVERRIDE)" "$(EMPTY)" LTS_TAG=$(LTS_TAG_OVERRIDE) else @@ -201,18 +201,37 @@ incubating: build-explorer downloads/$(MQ_ARCHIVE_DEV): $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END))) mkdir -p downloads +ifneq "$(BUILD_RSYNC_SERVER)" "$(EMPTY)" +# Use key which is not stored in the repository to fetch the files from the fileserver + curl -L $(BUILD_RSYNC_ENCRYPTED_KEY_URL) -o ./host.key.gpg + @echo $(BUILD_RSYNC_ENCRYPTION_PASSWORD)|gpg --passphrase-fd 0 ./host.key.gpg + chmod 600 ./host.key + rsync -rv -e "ssh -o BatchMode=yes -q -o StrictHostKeyChecking=no -i ./host.key" --include="*/" --include="*.tar.gz" --exclude="*" $(BUILD_RSYNC_USER)@$(BUILD_RSYNC_SERVER):"$(BUILD_RSYNC_PATH)" downloads/$(MQ_ARCHIVE_DEV) + -@rm host.key.gpg host.key +else ifneq "$(MQ_ARCHIVE_REPOSITORY_DEV)" "$(EMPTY)" curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY_DEV)" -o downloads/$(MQ_ARCHIVE_DEV) else curl -L https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_ARCHIVE_DEV) -o downloads/$(MQ_ARCHIVE_DEV) endif +endif downloads/$(MQ_ARCHIVE): $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced "$(MQ_VERSION)$(END))) mkdir -p downloads +ifneq "$(BUILD_RSYNC_SERVER)" "$(EMPTY)" +# Use key which is not stored in the repository to fetch the files from the fileserver + -@rm host.key.gpg host.key + curl -L $(BUILD_RSYNC_ENCRYPTED_KEY_URL) -o ./host.key.gpg + @echo $(BUILD_RSYNC_ENCRYPTION_PASSWORD)|gpg --passphrase-fd 0 ./host.key.gpg + chmod 600 ./host.key + rsync -rv -e "ssh -o BatchMode=yes -q -o StrictHostKeyChecking=no -i ./host.key" --include="*/" --include="*.tar.gz" --exclude="*" $(BUILD_RSYNC_USER)@$(BUILD_RSYNC_SERVER):"$(BUILD_RSYNC_PATH)" downloads/$(MQ_ARCHIVE) + -@rm host.key.gpg host.key +else ifneq "$(MQ_ARCHIVE_REPOSITORY)" "$(EMPTY)" curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY)" -o downloads/$(MQ_ARCHIVE) endif +endif .PHONY: downloads downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE) @@ -283,7 +302,7 @@ define build-mq $(COMMAND) build \ --tag $1:$2 \ --file $3 \ - $(EXTRA_ARGS) \ + $(EXTRA_ARGS) \ --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" \ --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" \ --build-arg IMAGE_TAG="$1:$2" \ @@ -415,15 +434,15 @@ ifneq "$(LTS)" "true" $(eval MQ_IMAGE_DEVSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_DEVSERVER_S390X) | jq -r .Digest)) $(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_AMD64) has a digest of $(MQ_IMAGE_DEVSERVER_AMD64_DIGEST)**"$(END))) $(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_S390X) has a digest of $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)**"$(END))) -endif +endif $(eval MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_AMD64) | jq -r .Digest)) $(eval MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_S390X) | jq -r .Digest)) $(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_AMD64) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST)**"$(END))) $(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_S390X) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)**"$(END))) -ifneq "$(LTS)" "true" +ifneq "$(LTS)" "true" $(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_DEVSERVER_MANIFEST)**"$(END))) echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)" $(END)) -endif +endif $(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_ADVANCEDSERVER_MANIFEST)**"$(END))) echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)" $(END)) diff --git a/travis-build-scripts/build.sh b/travis-build-scripts/build.sh index f68a253..10b60d3 100755 --- a/travis-build-scripts/build.sh +++ b/travis-build-scripts/build.sh @@ -20,20 +20,36 @@ if [ "$TRAVIS_BRANCH" = "$MAIN_BRANCH" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ] echo 'Retrieving global tagcache' && echo -en 'travis_fold:start:tag-cache-retrieve\\r' ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --check ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get - echo -en 'travis_fold:end:tag-cache-retrieve\\r' + echo -en 'travis_fold:end:tag-cache-retrieve\\r' fi -if [ "$LTS" != true ] ; then +if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then + if [ "$LTS" != true ] ; then + echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r' + make build-devjmstest + echo -en 'travis_fold:end:build-devjmstest\\r' + echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r' + make build-devserver + echo -en 'travis_fold:end:build-devserver\\r' + fi + if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then + if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then + echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r' + make build-advancedserver + echo -en 'travis_fold:end:build-advancedserver\\r' + fi + fi +else echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r' make build-devjmstest echo -en 'travis_fold:end:build-devjmstest\\r' - echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r' - make build-devserver - echo -en 'travis_fold:end:build-devserver\\r' -fi -if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then - if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then - echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r' - make build-advancedserver - echo -en 'travis_fold:end:build-advancedserver\\r' - fi + + if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then + echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r' + make build-devserver + echo -en 'travis_fold:end:build-devserver\\r' + else + echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r' + make build-advancedserver + echo -en 'travis_fold:end:build-advancedserver\\r' + fi fi diff --git a/travis-build-scripts/push.sh b/travis-build-scripts/push.sh index 5165cc2..bc73cef 100755 --- a/travis-build-scripts/push.sh +++ b/travis-build-scripts/push.sh @@ -21,7 +21,7 @@ if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then exit 0 fi -if [ ! -z $2 ]; then +if [ ! -z $2 ]; then export ARCH=$2 fi @@ -50,7 +50,7 @@ if [ ! -z $1 ]; then exit 1 ;; esac -else +else echo "ERROR: Type ( developer | production ) must be passed to push.sh" exit 1 fi diff --git a/travis-build-scripts/run.sh b/travis-build-scripts/run.sh index 470f1aa..b6dc973 100755 --- a/travis-build-scripts/run.sh +++ b/travis-build-scripts/run.sh @@ -21,7 +21,7 @@ if [ "$(uname -m)" = "x86_64" ] ; then export ARCH="amd64" ; else export ARCH=$( if [ "$PUSH_MANIFEST_ONLY" = true ] ; then echo 'Retrieving remote tagcache' && echo -en 'travis_fold:start:retrieve-tag-cache\\r' ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get - echo -en 'travis_fold:end:retrieve-tag-cache\\r' + echo -en 'travis_fold:end:retrieve-tag-cache\\r' make push-manifest ./travis-build-scripts/cleanup-cache.sh exit 0 @@ -38,9 +38,17 @@ echo -en 'travis_fold:end:docker-downgrade\\r' ./travis-build-scripts/test.sh ## Push images -if [ "$BUILD_ALL" = true ] ; then +if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then + if [ "$BUILD_ALL" = true ] ; then ./travis-build-scripts/push.sh developer ./travis-build-scripts/push.sh production + fi +else + if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then + ./travis-build-scripts/push.sh developer + else + ./travis-build-scripts/push.sh production + fi fi if [ "$LTS" = true ] ; then diff --git a/travis-build-scripts/test.sh b/travis-build-scripts/test.sh index 0baa241..81afef0 100755 --- a/travis-build-scripts/test.sh +++ b/travis-build-scripts/test.sh @@ -16,17 +16,29 @@ set -e -if [ "$LTS" != true ] ; then - echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r' - make test-devserver - echo -en 'travis_fold:end:test-devserver\\r' -fi -if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then - if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then - echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r' - make test-advancedserver - echo -en 'travis_fold:end:test-advancedserver\\r' - fi +if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then + if [ "$LTS" != true ] ; then + echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r' + make test-devserver + echo -en 'travis_fold:end:test-devserver\\r' + fi + if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then + if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then + echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r' + make test-advancedserver + echo -en 'travis_fold:end:test-advancedserver\\r' + fi + fi +else + if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then + echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r' + make test-devserver + echo -en 'travis_fold:end:test-devserver\\r' + else + echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r' + make test-advancedserver + echo -en 'travis_fold:end:test-advancedserver\\r' + fi fi echo 'Running gosec scan...' && echo -en 'travis_fold:start:gosec-scan\\r' if [ "$ARCH" = "amd64" ] ; then