academy/mq-container
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added remote global build tag/cache for multi-arch builds

Browse Source
This commit is contained in:
Luke J Powlett
2020-10-15 09:30:11 +01:00
committed by GitHub Enterprise
parent 3d6199e0d8
commit 55c094a58a
10 changed files with 320 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
Makefile
View File

@@ -118,9 +118,9 @@ endif
# image tagging
ifneq "$(RELEASE)" "$(EMPTY)"
MQ_TAG=$(MQ_VERSION)-$(RELEASE)-$(ARCH)-$(RELEASE_CANDIDATE)
EXTRA_LABELS=--label release=$(RELEASE)
MQ_MANIFEST_TAG=$(MQ_VERSION)-$(RELEASE)-$(RELEASE_CANDIDATE)
MQ_TAG=$(MQ_MANIFEST_TAG)-$(ARCH)
endif
ifeq "$(TIMESTAMPFLAT)" "$(EMPTY)"
@@ -131,30 +131,35 @@ ifeq "$(GIT_COMMIT)" "$(EMPTY)"
GIT_COMMIT=$(shell git rev-parse --short HEAD)
endif
ifeq ($(shell [ ! -z $(TRAVIS) ] && [ "$(TRAVIS_PULL_REQUEST)" != "true" ] && [ "$(TRAVIS_BRANCH)" = "private-master" ] && echo true), true)
ifeq ($(shell [ ! -z $(TRAVIS) ] && [ "$(TRAVIS_PULL_REQUEST)" = "false" ] && [ "$(TRAVIS_BRANCH)" = "private-master" ] && echo true), true)
RELEASE_TAG=$(shell [ -z "$(RELEASE)" ] || echo "-$(RELEASE)-$(RELEASE_CANDIDATE)")
MQ_TAG=$(MQ_VERSION)$(RELEASE_TAG)-$(ARCH).$(TIMESTAMPFLAT).$(GIT_COMMIT)
MQ_MANIFEST_TAG=$(MQ_VERSION)$(RELEASE_TAG).$(TIMESTAMPFLAT).$(GIT_COMMIT)
MQ_TAG=$(MQ_MANIFEST_TAG)-$(ARCH)
endif
PATH_TO_MQ_TAG_CACHE=$(TRAVIS_BUILD_DIR)/.tagcache
MQ_TAG_CACHED=$(MQ_TAG)
MQ_MANIFEST_TAG_CACHED?=$(MQ_MANIFEST_TAG)
MQ_TAG_CACHED_amd64?=$(MQ_MANIFEST_TAG)-amd64
MQ_TAG_CACHED_s390x?=$(MQ_MANIFEST_TAG)-s390x
ifneq "$(TRAVIS)" "$(EMPTY)"
MQ_TAG_CACHED=$(shell cat $(PATH_TO_MQ_TAG_CACHE))
ifneq ("$(wildcard $(PATH_TO_MQ_TAG_CACHE))","")
include $(PATH_TO_MQ_TAG_CACHE)
endif
endif
# end image tagging
MQ_IMAGE_FULL_RELEASE_NAME=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED)
MQ_IMAGE_DEV_FULL_RELEASE_NAME=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED)
MQ_IMAGE_FULL_RELEASE_NAME=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH))
MQ_IMAGE_DEV_FULL_RELEASE_NAME=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH))
#setup variables for fat-manifests
MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)
MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)
MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-amd64
MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-s390x
MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-amd64
MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-s390x
MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG_CACHED)
MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG_CACHED)
MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG_CACHED)-amd64
MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG_CACHED)-s390x
MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG_CACHED)-amd64
MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG_CACHED)-s390x
###############################################################################
# Build targets
@@ -200,7 +205,9 @@ downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE)
.PHONY: cache-mq-tag
cache-mq-tag:
@$(shell echo "$(MQ_TAG)" > $(PATH_TO_MQ_TAG_CACHE))
@$(shell printf "MQ_MANIFEST_TAG_CACHED=$(MQ_MANIFEST_TAG)\n" > $(PATH_TO_MQ_TAG_CACHE))
@$(shell printf "MQ_TAG_CACHED_amd64=$(MQ_MANIFEST_TAG)-amd64\n" >> $(PATH_TO_MQ_TAG_CACHE))
@$(shell printf "MQ_TAG_CACHED_s390x=$(MQ_MANIFEST_TAG)-s390x\n" >> $(PATH_TO_MQ_TAG_CACHE))
# Vendor Go dependencies for the Docker tests
test/docker/vendor:
@@ -213,9 +220,9 @@ test-unit:
.PHONY: test-advancedserver
test-advancedserver: test/docker/vendor
$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) on $(shell docker --version)"$(END)))
docker inspect $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED)
cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER)
$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) on $(shell docker --version)"$(END)))
docker inspect $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH))
cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER)
.PHONY: build-devjmstest
build-devjmstest:
@@ -224,9 +231,9 @@ build-devjmstest:
.PHONY: test-devserver
test-devserver: test/docker/vendor
$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED) on $(shell docker --version)"$(END)))
docker inspect $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED)
cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER)
$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH)) on $(shell docker --version)"$(END)))
docker inspect $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH))
cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH)) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER)
.PHONY: coverage
coverage:
@@ -234,7 +241,7 @@ coverage:
.PHONY: test-advancedserver-cover
test-advancedserver-cover: test/docker/vendor coverage
$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) with code coverage on $(shell docker --version)"$(END)))
$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) with code coverage on $(shell docker --version)"$(END)))
rm -f ./coverage/unit*.cov
# Run unit tests with coverage, for each package under 'internal'
go list -f '{{.Name}}' ./internal/... | xargs -I {} go test -cover -covermode count -coverprofile ./coverage/unit-{}.cov ./internal/{}
@@ -246,7 +253,7 @@ test-advancedserver-cover: test/docker/vendor coverage
rm -f ./test/docker/coverage/*.cov
rm -f ./coverage/docker.*
mkdir -p ./test/docker/coverage/
cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED)-cover TEST_COVER=true go test $(TEST_OPTS_DOCKER)
cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH))-cover TEST_COVER=true go test $(TEST_OPTS_DOCKER)
echo 'mode: count' > ./coverage/docker.cov
tail -q -n +2 ./test/docker/coverage/*.cov >> ./coverage/docker.cov
go tool cover -html=./coverage/docker.cov -o ./coverage/docker.html
@@ -302,20 +309,20 @@ build-advancedserver-host: build-advancedserver
.PHONY: build-advancedserver
build-advancedserver: registry-login log-build-env downloads/$(MQ_ARCHIVE) command-version
$(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED)"$(END)))
$(call build-mq,$(MQ_IMAGE_ADVANCEDSERVER),$(MQ_TAG_CACHED),Dockerfile-server,$(MQ_ARCHIVE),mq-server)
$(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH))"$(END)))
$(call build-mq,$(MQ_IMAGE_ADVANCEDSERVER),$(MQ_TAG_CACHED_$(ARCH)),Dockerfile-server,$(MQ_ARCHIVE),mq-server)
.PHONY: build-devserver-host
build-devserver-host: build-devserver
.PHONY: build-devserver
build-devserver: registry-login log-build-env downloads/$(MQ_ARCHIVE_DEV) command-version
$(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED)"$(END)))
$(call build-mq,$(MQ_IMAGE_DEVSERVER),$(MQ_TAG_CACHED),Dockerfile-server,$(MQ_ARCHIVE_DEV),mq-dev-server)
$(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER):$(MQ_TAG_CACHED_$(ARCH))"$(END)))
$(call build-mq,$(MQ_IMAGE_DEVSERVER),$(MQ_TAG_CACHED_$(ARCH)),Dockerfile-server,$(MQ_ARCHIVE_DEV),mq-dev-server)
.PHONY: build-advancedserver-cover
build-advancedserver-cover: registry-login command-version
$(COMMAND) build --build-arg BASE_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED) -t $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED)-cover -f Dockerfile-server.cover .
$(COMMAND) build --build-arg BASE_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH)) -t $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG_CACHED_$(ARCH))-cover -f Dockerfile-server.cover .
.PHONY: build-explorer
build-explorer: registry-login downloads/$(MQ_ARCHIVE_DEV)
@@ -324,7 +331,7 @@ build-explorer: registry-login downloads/$(MQ_ARCHIVE_DEV)
.PHONY: build-sdk
build-sdk: registry-login downloads/$(MQ_ARCHIVE_DEV)
$(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_SDK)"$(END)))
$(call build-mq,mq-sdk,$(MQ_TAG_CACHED),incubating/mq-sdk/Dockerfile,$(MQ_SDK_ARCHIVE),mq-sdk)
$(call build-mq,mq-sdk,$(MQ_TAG_CACHED_$(ARCH)),incubating/mq-sdk/Dockerfile,$(MQ_SDK_ARCHIVE),mq-sdk)
.PHONY: registry-login
registry-login:
@@ -364,14 +371,14 @@ pull-mq-archive-dev:
push-advancedserver:
$(info $(SPACER)$(shell printf $(TITLE)"Push production image to $(MQ_DELIVERY_REGISTRY_FULL_PATH)"$(END)))
$(COMMAND) login $(MQ_DELIVERY_REGISTRY_HOSTNAME) -u $(MQ_DELIVERY_REGISTRY_USER) -p $(MQ_DELIVERY_REGISTRY_CREDENTIAL)
$(COMMAND) tag $(MQ_IMAGE_ADVANCEDSERVER)\:$(MQ_TAG_CACHED) $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME)
$(COMMAND) tag $(MQ_IMAGE_ADVANCEDSERVER)\:$(MQ_TAG_CACHED_$(ARCH)) $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME)
$(COMMAND) push $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME)
.PHONY: push-devserver
push-devserver:
$(info $(SPACER)$(shell printf $(TITLE)"Push developer image to $(MQ_DELIVERY_REGISTRY_FULL_PATH)"$(END)))
$(COMMAND) login $(MQ_DELIVERY_REGISTRY_HOSTNAME) -u $(MQ_DELIVERY_REGISTRY_USER) -p $(MQ_DELIVERY_REGISTRY_CREDENTIAL)
$(COMMAND) tag $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG_CACHED) $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME)
$(COMMAND) tag $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG_CACHED_$(ARCH)) $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME)
$(COMMAND) push $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME)
.PHONY: pull-advancedserver
@@ -379,14 +386,14 @@ pull-advancedserver:
$(info $(SPACER)$(shell printf $(TITLE)"Pull production image from $(MQ_DELIVERY_REGISTRY_FULL_PATH)"$(END)))
$(COMMAND) login $(MQ_DELIVERY_REGISTRY_HOSTNAME) -u $(MQ_DELIVERY_REGISTRY_USER) -p $(MQ_DELIVERY_REGISTRY_CREDENTIAL)
$(COMMAND) pull $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME)
$(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME) $(MQ_IMAGE_ADVANCEDSERVER)\:$(MQ_TAG_CACHED)
$(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_FULL_RELEASE_NAME) $(MQ_IMAGE_ADVANCEDSERVER)\:$(MQ_TAG_CACHED_$(ARCH))
.PHONY: pull-devserver
pull-devserver:
$(info $(SPACER)$(shell printf $(TITLE)"Pull developer image from $(MQ_DELIVERY_REGISTRY_FULL_PATH)"$(END)))
$(COMMAND) login $(MQ_DELIVERY_REGISTRY_HOSTNAME) -u $(MQ_DELIVERY_REGISTRY_USER) -p $(MQ_DELIVERY_REGISTRY_CREDENTIAL)
$(COMMAND) pull $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME)
$(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG_CACHED)
$(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG_CACHED_$(ARCH))
.PHONY: push-manifest
push-manifest: build-skopeo-container
@@ -400,9 +407,9 @@ push-manifest: build-skopeo-container
$(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_AMD64) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST)**"$(END)))
$(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_S390X) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)**"$(END)))
$(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_DEVSERVER_MANIFEST)**"$(END)))
echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)" $(END))
echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG_CACHED) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)" $(END))
$(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_ADVANCEDSERVER_MANIFEST)**"$(END)))
echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)" $(END))
echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG_CACHED) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)" $(END))
.PHONY: build-skopeo-container
build-skopeo-container: