academy/mq-container
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added support for internal build levels

Browse Source 
* Dummy change to see if things are working

* Initial internal build support

* Add SSH key support

* Add SSH key support pt 2

* Add SSH key support pt 3

* Add Push support

* Full package support

* Correct typo

* Correct typo 2

* Disable some tests as per master

* Change rsync variable names

* Fix build issues

* Remove the -full package support for now

Co-authored-by: Jason Edmeades <jason_edmeades@uk.ibm.com>
This commit is contained in:
Luke J Powlett
2020-11-17 13:33:29 +00:00
committed by Arthur J Barr
parent c39a532da9
commit dd1d534045
5 changed files with 88 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
Makefile
View File

@@ -38,7 +38,7 @@ MQ_ARCHIVE_DEV ?= $(MQ_VERSION)-IBM-MQ-Advanced-for-Developers-Non-Install-$(MQ_
# MQ_SDK_ARCHIVE specifies the archive to use for building the golang programs. Defaults vary on developer or advanced. # MQ_SDK_ARCHIVE specifies the archive to use for building the golang programs. Defaults vary on developer or advanced.
MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION))
# Options to `go test` for the Docker tests # Options to `go test` for the Docker tests
TEST_OPTS_DOCKER ?= TEST_OPTS_DOCKER ?=
# Timeout for the Docker tests # Timeout for the Docker tests
TEST_TIMEOUT_DOCKER ?= 30m TEST_TIMEOUT_DOCKER ?= 30m
# MQ_IMAGE_ADVANCEDSERVER is the name of the built MQ Advanced image # MQ_IMAGE_ADVANCEDSERVER is the name of the built MQ Advanced image
@@ -113,7 +113,7 @@ endif
# LTS_TAG is the tag modifier for an LTS container build # LTS_TAG is the tag modifier for an LTS container build
LTS_TAG= LTS_TAG=
ifeq "$(LTS)" "true" ifeq "$(LTS)" "true"
ifneq "$(LTS_TAG_OVERRIDE)" "$(EMPTY)" ifneq "$(LTS_TAG_OVERRIDE)" "$(EMPTY)"
LTS_TAG=$(LTS_TAG_OVERRIDE) LTS_TAG=$(LTS_TAG_OVERRIDE)
else else
@@ -201,18 +201,37 @@ incubating: build-explorer
downloads/$(MQ_ARCHIVE_DEV): downloads/$(MQ_ARCHIVE_DEV):
$(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END))) $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END)))
mkdir -p downloads mkdir -p downloads
ifneq "$(BUILD_RSYNC_SERVER)" "$(EMPTY)"
# Use key which is not stored in the repository to fetch the files from the fileserver
curl -L $(BUILD_RSYNC_ENCRYPTED_KEY_URL) -o ./host.key.gpg
@echo $(BUILD_RSYNC_ENCRYPTION_PASSWORD)|gpg --passphrase-fd 0 ./host.key.gpg
chmod 600 ./host.key
rsync -rv -e "ssh -o BatchMode=yes -q -o StrictHostKeyChecking=no -i ./host.key" --include="*/" --include="*.tar.gz" --exclude="*" $(BUILD_RSYNC_USER)@$(BUILD_RSYNC_SERVER):"$(BUILD_RSYNC_PATH)" downloads/$(MQ_ARCHIVE_DEV)
-@rm host.key.gpg host.key
else
ifneq "$(MQ_ARCHIVE_REPOSITORY_DEV)" "$(EMPTY)" ifneq "$(MQ_ARCHIVE_REPOSITORY_DEV)" "$(EMPTY)"
curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY_DEV)" -o downloads/$(MQ_ARCHIVE_DEV) curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY_DEV)" -o downloads/$(MQ_ARCHIVE_DEV)
else else
curl -L https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_ARCHIVE_DEV) -o downloads/$(MQ_ARCHIVE_DEV) curl -L https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_ARCHIVE_DEV) -o downloads/$(MQ_ARCHIVE_DEV)
endif endif
endif
downloads/$(MQ_ARCHIVE): downloads/$(MQ_ARCHIVE):
$(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced "$(MQ_VERSION)$(END))) $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced "$(MQ_VERSION)$(END)))
mkdir -p downloads mkdir -p downloads
ifneq "$(BUILD_RSYNC_SERVER)" "$(EMPTY)"
# Use key which is not stored in the repository to fetch the files from the fileserver
-@rm host.key.gpg host.key
curl -L $(BUILD_RSYNC_ENCRYPTED_KEY_URL) -o ./host.key.gpg
@echo $(BUILD_RSYNC_ENCRYPTION_PASSWORD)|gpg --passphrase-fd 0 ./host.key.gpg
chmod 600 ./host.key
rsync -rv -e "ssh -o BatchMode=yes -q -o StrictHostKeyChecking=no -i ./host.key" --include="*/" --include="*.tar.gz" --exclude="*" $(BUILD_RSYNC_USER)@$(BUILD_RSYNC_SERVER):"$(BUILD_RSYNC_PATH)" downloads/$(MQ_ARCHIVE)
-@rm host.key.gpg host.key
else
ifneq "$(MQ_ARCHIVE_REPOSITORY)" "$(EMPTY)" ifneq "$(MQ_ARCHIVE_REPOSITORY)" "$(EMPTY)"
curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY)" -o downloads/$(MQ_ARCHIVE) curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY)" -o downloads/$(MQ_ARCHIVE)
endif endif
endif
.PHONY: downloads .PHONY: downloads
downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE) downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE)
@@ -283,7 +302,7 @@ define build-mq
$(COMMAND) build \ $(COMMAND) build \
--tag $1:$2 \ --tag $1:$2 \
--file $3 \ --file $3 \
$(EXTRA_ARGS) \ $(EXTRA_ARGS) \
--build-arg IMAGE_REVISION="$(IMAGE_REVISION)" \ --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" \
--build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" \ --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" \
--build-arg IMAGE_TAG="$1:$2" \ --build-arg IMAGE_TAG="$1:$2" \
@@ -415,15 +434,15 @@ ifneq "$(LTS)" "true"
$(eval MQ_IMAGE_DEVSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_DEVSERVER_S390X) | jq -r .Digest)) $(eval MQ_IMAGE_DEVSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_DEVSERVER_S390X) | jq -r .Digest))
$(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_AMD64) has a digest of $(MQ_IMAGE_DEVSERVER_AMD64_DIGEST)**"$(END))) $(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_AMD64) has a digest of $(MQ_IMAGE_DEVSERVER_AMD64_DIGEST)**"$(END)))
$(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_S390X) has a digest of $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)**"$(END))) $(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_S390X) has a digest of $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)**"$(END)))
endif endif
$(eval MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_AMD64) | jq -r .Digest)) $(eval MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_AMD64) | jq -r .Digest))
$(eval MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_S390X) | jq -r .Digest)) $(eval MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_S390X) | jq -r .Digest))
$(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_AMD64) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST)**"$(END))) $(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_AMD64) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST)**"$(END)))
$(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_S390X) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)**"$(END))) $(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_S390X) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)**"$(END)))
ifneq "$(LTS)" "true" ifneq "$(LTS)" "true"
$(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_DEVSERVER_MANIFEST)**"$(END))) $(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_DEVSERVER_MANIFEST)**"$(END)))
echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)" $(END)) echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)" $(END))
endif endif
$(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_ADVANCEDSERVER_MANIFEST)**"$(END))) $(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_ADVANCEDSERVER_MANIFEST)**"$(END)))
echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)" $(END)) echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)" $(END))

40
travis-build-scripts/build.sh
View File

@@ -20,20 +20,36 @@ if [ "$TRAVIS_BRANCH" = "$MAIN_BRANCH" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]
echo 'Retrieving global tagcache' && echo -en 'travis_fold:start:tag-cache-retrieve\\r' echo 'Retrieving global tagcache' && echo -en 'travis_fold:start:tag-cache-retrieve\\r'
./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --check ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --check
./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get
echo -en 'travis_fold:end:tag-cache-retrieve\\r' echo -en 'travis_fold:end:tag-cache-retrieve\\r'
fi fi
if [ "$LTS" != true ] ; then if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then
if [ "$LTS" != true ] ; then
echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r'
make build-devjmstest
echo -en 'travis_fold:end:build-devjmstest\\r'
echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r'
make build-devserver
echo -en 'travis_fold:end:build-devserver\\r'
fi
if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then
if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then
echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r'
make build-advancedserver
echo -en 'travis_fold:end:build-advancedserver\\r'
fi
fi
else
echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r' echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r'
make build-devjmstest make build-devjmstest
echo -en 'travis_fold:end:build-devjmstest\\r' echo -en 'travis_fold:end:build-devjmstest\\r'
echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r'
make build-devserver if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then
echo -en 'travis_fold:end:build-devserver\\r' echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r'
fi make build-devserver
if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then echo -en 'travis_fold:end:build-devserver\\r'
if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then else
echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r' echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r'
make build-advancedserver make build-advancedserver
echo -en 'travis_fold:end:build-advancedserver\\r' echo -en 'travis_fold:end:build-advancedserver\\r'
fi fi
fi fi

4
travis-build-scripts/push.sh
View File

@@ -21,7 +21,7 @@ if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
exit 0 exit 0
fi fi
if [ ! -z $2 ]; then if [ ! -z $2 ]; then
export ARCH=$2 export ARCH=$2
fi fi
@@ -50,7 +50,7 @@ if [ ! -z $1 ]; then
exit 1 exit 1
;; ;;
esac esac
else else
echo "ERROR: Type ( developer | production ) must be passed to push.sh" echo "ERROR: Type ( developer | production ) must be passed to push.sh"
exit 1 exit 1
fi fi

12
travis-build-scripts/run.sh
View File

@@ -21,7 +21,7 @@ if [ "$(uname -m)" = "x86_64" ] ; then export ARCH="amd64" ; else export ARCH=$(
if [ "$PUSH_MANIFEST_ONLY" = true ] ; then if [ "$PUSH_MANIFEST_ONLY" = true ] ; then
echo 'Retrieving remote tagcache' && echo -en 'travis_fold:start:retrieve-tag-cache\\r' echo 'Retrieving remote tagcache' && echo -en 'travis_fold:start:retrieve-tag-cache\\r'
./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get
echo -en 'travis_fold:end:retrieve-tag-cache\\r' echo -en 'travis_fold:end:retrieve-tag-cache\\r'
make push-manifest make push-manifest
./travis-build-scripts/cleanup-cache.sh ./travis-build-scripts/cleanup-cache.sh
exit 0 exit 0
@@ -38,9 +38,17 @@ echo -en 'travis_fold:end:docker-downgrade\\r'
./travis-build-scripts/test.sh ./travis-build-scripts/test.sh
## Push images ## Push images
if [ "$BUILD_ALL" = true ] ; then if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then
if [ "$BUILD_ALL" = true ] ; then
./travis-build-scripts/push.sh developer ./travis-build-scripts/push.sh developer
./travis-build-scripts/push.sh production ./travis-build-scripts/push.sh production
fi
else
if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then
./travis-build-scripts/push.sh developer
else
./travis-build-scripts/push.sh production
fi
fi fi
if [ "$LTS" = true ] ; then if [ "$LTS" = true ] ; then

34
travis-build-scripts/test.sh
View File

@@ -16,17 +16,29 @@
set -e set -e
if [ "$LTS" != true ] ; then if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then
echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r' if [ "$LTS" != true ] ; then
make test-devserver echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r'
echo -en 'travis_fold:end:test-devserver\\r' make test-devserver
fi echo -en 'travis_fold:end:test-devserver\\r'
if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then fi
if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then
echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r' if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then
make test-advancedserver echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r'
echo -en 'travis_fold:end:test-advancedserver\\r' make test-advancedserver
fi echo -en 'travis_fold:end:test-advancedserver\\r'
fi
fi
else
if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then
echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r'
make test-devserver
echo -en 'travis_fold:end:test-devserver\\r'
else
echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r'
make test-advancedserver
echo -en 'travis_fold:end:test-advancedserver\\r'
fi
fi fi
echo 'Running gosec scan...' && echo -en 'travis_fold:start:gosec-scan\\r' echo 'Running gosec scan...' && echo -en 'travis_fold:start:gosec-scan\\r'
if [ "$ARCH" = "amd64" ] ; then if [ "$ARCH" = "amd64" ] ; then