Merge pull request #247 from mq-cloudpak/tadj-update-ubi

udpdate ubi

.gitignore

@@ -13,3 +13,10 @@ vendor/github.com/prometheus/client_model/.project
 vendor/github.com/prometheus/client_model/.settings*
 gosec_results.json
 internal/qmgrauth/qmgroam/patch
+.tagcache
+
+# Nix
+*.nix
+.envrc
+.direnv
+result

.travis.yml

@@ -12,63 +12,82 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Temporarily removing dist tag as not supported by power build
+dist: bionic
-# dist: xenial
+group: beta
-
 sudo: required
 language: go
 
 go:
-  - "1.12"
+  - "1.16.12"
 
 services:
   - docker
 
+env:
+  global:
+    - MAIN_BRANCH=v9.2.5
+    - TAGCACHE_FILE=tagcache
+    - RELEASE=r3
+
 go_import_path: "github.com/ibm-messaging/mq-container"
 
 # cache:
 #   directories:
 #     - downloads
 
-env:
-  global:
-    - RELEASE=""
-
 jobs:
   include:
-    - stage: build
+    - stage: basic-build
-      name: "Basic build"
+      if: branch != v9.2.5 AND tag IS blank
-      if: branch != private-master AND tag IS blank
+      name: "Basic AMD64 build"
       os: linux
       env:
-        - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_AMD64
+        - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_925_ARCHIVE_REPOSITORY_DEV_AMD64
       script: bash -e travis-build-scripts/run.sh
-    - if: branch = private-master OR tag =~ ^release-candidate*
+
+    # CD Build
+
+    - stage: global-tag
+      if: branch = v9.2.5 AND type != pull_request OR tag =~ ^release-candidate*
+      name: "Generate Global Tag"
+      os: linux
+      script: bash -e travis-build-scripts/global-tag.sh
+    - stage: build
+      if: branch = v9.2.5 OR tag =~ ^release-candidate*
       name: "Multi-Arch AMD64 build"
       os: linux
       env:
         - BUILD_ALL=true
-        - MQ_ARCHIVE_REPOSITORY=$MQ_920_ARCHIVE_REPOSITORY_AMD64
+        - MQ_ARCHIVE_REPOSITORY=$MQ_925_ARCHIVE_REPOSITORY_AMD64
-        - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_AMD64
+        - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_925_ARCHIVE_REPOSITORY_DEV_AMD64
       script: bash -e travis-build-scripts/run.sh
-    # - if: branch = private-master OR tag =~ ^release-candidate*
+    - stage: build
-    #   name: "Multi-Arch PPC64LE build"
+      if: branch = v9.2.5 OR tag =~ ^release-candidate*
-    #   os: linux-ppc64le
+      name: "Multi-Arch S390X build"
-    #   env:
+      os: linux-s390
-    #     - BUILD_ALL=true
+      env:
-    #     - TEST_OPTS_DOCKER="-run TestGoldenPathWithMetrics"
+        - BUILD_ALL=true
-    #     # - MQ_ARCHIVE_REPOSITORY=$MQ_920_ARCHIVE_REPOSITORY_PPC64LE
+        - TEST_OPTS_DOCKER="-run TestGoldenPathWithMetrics"
-    #     - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_PPC64LE
+        - MQ_ARCHIVE_REPOSITORY=$MQ_925_ARCHIVE_REPOSITORY_S390X
-    #   script: bash -e travis-build-scripts/run.sh
+        - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_925_ARCHIVE_REPOSITORY_DEV_S390X
-    # - if: branch = private-master OR tag =~ ^release-candidate*
+      script: bash -e travis-build-scripts/run.sh
-    #   name: "Multi-Arch S390X build"
+    - stage: build
-    #   os: linux-s390
+      if: branch = v9.2.5 OR tag =~ ^release-candidate*
-    #   env:
+      name: "Multi-Arch PPC64LE build"
-    #     - BUILD_ALL=true
+      os: linux-ppc64le
-    #     - TEST_OPTS_DOCKER="-run TestGoldenPathWithMetrics"
+      env:
-    #     # - MQ_ARCHIVE_REPOSITORY=$MQ_920_ARCHIVE_REPOSITORY_S390X
+        - BUILD_ALL=true
-    #     - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_S390X
+        - TEST_OPTS_DOCKER="-run TestGoldenPathWithMetrics"
-    #   script: bash -e travis-build-scripts/run.sh
+        - MQ_ARCHIVE_REPOSITORY=$MQ_925_ARCHIVE_REPOSITORY_PPC64LE
+        - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_925_ARCHIVE_REPOSITORY_DEV_PPC64LE
+      script: bash -e travis-build-scripts/run.sh
+    - stage: push-manifest
+      if: branch = v9.2.5 AND type != pull_request OR tag =~ ^release-candidate*
+      name: "Push Manifest-list to registry"
+      env:
+        - PUSH_MANIFEST_ONLY=true
+      script: bash -e travis-build-scripts/run.sh
+
 
 before_install:
   - make install-build-deps

.whitesource

@@ -0,0 +1,6 @@
+{
+  "settingsInheritedFrom": "whitesource-config/whitesource-config@master",
+  "scanSettings": {
+      "baseBranches": ["private-master", "v9.2.0.x-eus"]
+  }
+}

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Change log
 
+## 9.2.5.0 (2021-12-06)
+
+* Updated to MQ version 9.2.5.0
+
+## 9.2.4.0 (2021-09-06)
+
+* Updated to MQ version 9.2.4.0
+
+## 9.2.3.0 (2021-07-22)
+
+* Updated to MQ version 9.2.3.0
+
+## 9.2.2.0 (2021-03-26)
+
+* Updated to MQ version 9.2.2.0
+
+## 9.2.1.0 (2020-02-18)
+
+* Updated to MQ version 9.2.1.0
+
+
+## 9.2.0.1-LTS (2020-12-04)
+
+* Added support for MQ Long Term Support (production licensed only) in the mq-container
+
 ## 9.2.0.0 (2020-07-23)
 
 * Updated to [MQ version 9.2.0.0](https://www.ibm.com/support/knowledgecenter/SSFKSJ_9.2.0/com.ibm.mq.pro.doc/q113110_.htm)

Dockerfile-server

@@ -1,4 +1,4 @@
-# © Copyright IBM Corporation 2015, 2020
+# © Copyright IBM Corporation 2015, 2021
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,18 +13,22 @@
 # limitations under the License.
 
 ARG BASE_IMAGE=registry.redhat.io/ubi8/ubi-minimal
-ARG BASE_TAG=8.2-301.1593113563
+ARG BASE_TAG=8.6-751
+ARG BUILDER_IMAGE=registry.redhat.io/ubi8/go-toolset
+ARG BUILDER_TAG=1.16.12-10
+ARG GO_WORKDIR=/opt/app-root/src/go/src/github.com/ibm-messaging/mq-container
+ARG MQ_URL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/9.2.5.0-IBM-MQ-Advanced-for-Developers-Non-Install-LinuxX64.tar.gz"
 ###############################################################################
 # Build stage to build Go code
 ###############################################################################
-FROM registry.redhat.io/ubi8/go-toolset:1.13.4-22 as builder
+FROM $BUILDER_IMAGE:$BUILDER_TAG as builder
-# FROM docker.io/centos/go-toolset-7-centos7 as builder
 # The URL to download the MQ installer from in tar.gz format
-# This assumes an archive containing the MQ RPM install packages
+# This assumes an archive containing the MQ Non-Install packages
-ARG MQ_URL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev912_linux_x86-64.tar.gz"
+ARG MQ_URL
 ARG IMAGE_REVISION="Not specified"
 ARG IMAGE_SOURCE="Not specified"
 ARG IMAGE_TAG="Not specified"
+ARG GO_WORKDIR
 USER 0
 COPY install-mq.sh /usr/local/bin/
 RUN mkdir /opt/mqm \
@@ -32,23 +36,25 @@ RUN mkdir /opt/mqm \
   && sleep 1 \
   && INSTALL_SDK=1 install-mq.sh \
   && chown -R 1001:root /opt/mqm/*
-WORKDIR /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/
+WORKDIR $GO_WORKDIR/
+COPY go.mod go.sum ./
 COPY cmd/ ./cmd
 COPY internal/ ./internal
 COPY pkg/ ./pkg
 COPY vendor/ ./vendor
-ENV PATH="${PATH}:/opt/rh/go-toolset-1.11/root/usr/bin" \
+ENV CGO_CFLAGS="-I/opt/mqm/inc/" \
-    CGO_CFLAGS="-I/opt/mqm/inc/" \
     CGO_LDFLAGS_ALLOW="-Wl,-rpath.*"
+ENV PATH="${PATH}:/opt/mqm/bin"
 RUN go build -ldflags "-X \"main.ImageCreated=$(date --iso-8601=seconds)\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\" -X \"main.ImageTag=$IMAGE_TAG\"" ./cmd/runmqserver/
 RUN go build ./cmd/chkmqready/
 RUN go build ./cmd/chkmqhealthy/
+RUN go build ./cmd/chkmqstarted/
 RUN go build ./cmd/runmqdevserver/
-RUN go build -buildmode=c-shared -o amqpasdev.so ./internal/qmgrauth/pas.go
 RUN go test -v ./cmd/runmqdevserver/...
 RUN go test -v ./cmd/runmqserver/
 RUN go test -v ./cmd/chkmqready/
 RUN go test -v ./cmd/chkmqhealthy/
+RUN go test -v ./cmd/chkmqstarted/
 RUN go test -v ./pkg/...
 RUN go test -v ./internal/...
 RUN go vet ./cmd/... ./internal/...
@@ -58,11 +64,10 @@ RUN go vet ./cmd/... ./internal/...
 ###############################################################################
 FROM $BASE_IMAGE:$BASE_TAG AS mq-server
 # The MQ packages to install - see install-mq.sh for default value
-ARG MQ_URL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev914_linux_x86-64.tar.gz"
+ARG MQ_URL
-ARG MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesWeb*.rpm MQSeriesAMS-*.rpm"
-#ARG MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-web ibmmq-ams"
 ARG BASE_IMAGE
 ARG BASE_TAG
+ARG GO_WORKDIR
 LABEL summary="IBM MQ Advanced Server"
 LABEL description="Simplify, accelerate and facilitate the reliable exchange of data with a security-rich messaging solution — trusted by the world’s most successful enterprises"
 LABEL vendor="IBM"
@@ -89,15 +94,16 @@ RUN env \
 # Create a directory for runtime data from runmqserver
 RUN mkdir -p /run/runmqserver \
   && chown 1001:root /run/runmqserver
-COPY --from=builder /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/runmqserver /usr/local/bin/
+COPY --from=builder $GO_WORKDIR/runmqserver /usr/local/bin/
-COPY --from=builder /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/chkmq* /usr/local/bin/
+COPY --from=builder $GO_WORKDIR/chkmq* /usr/local/bin/
 COPY NOTICES.txt /opt/mqm/licenses/notices-container.txt
+COPY ha/native-ha.ini.tpl /etc/mqm/native-ha.ini.tpl
 # Copy web XML files
 COPY web /etc/mqm/web
 COPY etc/mqm/*.tpl /etc/mqm/
 RUN chmod ug+x /usr/local/bin/runmqserver \
   && chown 1001:root /usr/local/bin/*mq* \
-  && chmod ug+xs /usr/local/bin/chkmq* \
+  && chmod ug+x /usr/local/bin/chkmq* \
   && chown -R 1001:root /etc/mqm/* \
   && install --directory --mode 2775 --owner 1001 --group root /run/runmqserver \
   && touch /run/termination-log \
@@ -114,12 +120,34 @@ USER 1001
 ENV MQ_CONNAUTH_USE_HTP=false
 ENTRYPOINT ["runmqserver"]
 
+###############################################################################
+# Build stage to build C code for custom authorization service (developer-only)
+###############################################################################
+FROM registry.redhat.io/rhel8/gcc-toolset-9-toolchain as cbuilder
+# The URL to download the MQ installer from in tar.gz format
+# This assumes an archive containing the MQ Non-Install packages
+ARG MQ_URL
+USER 0
+# Install the Apache Portable Runtime code (used for htpasswd hash checking)
+RUN yum -y install apr-devel apr-util-openssl apr-util-devel
+# Install MQ client
+COPY install-mq.sh /usr/local/bin/
+RUN mkdir /opt/mqm \
+  && chmod a+x /usr/local/bin/install-mq.sh \
+  && sleep 1 \
+  && INSTALL_SDK=1 install-mq.sh \
+  && chown -R 1001:root /opt/mqm/*
+COPY authservice/ /opt/app-root/src/authservice/
+WORKDIR /opt/app-root/src/authservice/mqhtpass
+RUN make all
+
 ###############################################################################
 # Add default developer config
 ###############################################################################
 FROM mq-server AS mq-dev-server
 ARG BASE_IMAGE
 ARG BASE_TAG
+ARG GO_WORKDIR
 # Enable MQ developer default configuration
 ENV MQ_DEV=true
 LABEL summary="IBM MQ Advanced for Developers Server"
@@ -134,7 +162,7 @@ LABEL io.k8s.description="Simplify, accelerate and facilitate the reliable excha
 LABEL base-image=$BASE_IMAGE
 LABEL base-image-release=$BASE_TAG
 USER 0
-COPY --from=builder /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/amqpas* /opt/mqm/lib64/
+COPY --from=cbuilder /opt/app-root/src/authservice/mqhtpass/build/mqhtpass.so /opt/mqm/lib64/
 COPY etc/mqm/*.ini /etc/mqm/
 COPY etc/mqm/mq.htpasswd /etc/mqm/
 RUN chmod 0660 /etc/mqm/mq.htpasswd
@@ -145,7 +173,7 @@ RUN chmod u+x /usr/local/bin/install-extra-packages.sh \
 # Create a directory for runtime data from runmqserver
 RUN mkdir -p /run/runmqdevserver \
   && chown 1001:root /run/runmqdevserver
-COPY --from=builder /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/runmqdevserver /usr/local/bin/
+COPY --from=builder $GO_WORKDIR/runmqdevserver /usr/local/bin/
 # Copy template files
 COPY incubating/mqadvanced-server-dev/*.tpl /etc/mqm/
 # Copy web XML files for default developer configuration
@@ -156,7 +184,7 @@ RUN chown -R 1001:root /etc/mqm/* \
   && install --directory --mode 2775 --owner 1001 --group root /run/runmqdevserver
 ENV MQ_ENABLE_EMBEDDED_WEB_SERVER=1 MQ_GENERATE_CERTIFICATE_HOSTNAME=localhost
 ENV LD_LIBRARY_PATH=/opt/mqm/lib64
-ENV MQS_PERMIT_UNKNOWN_ID=true
 ENV MQ_CONNAUTH_USE_HTP=true
+ENV MQS_PERMIT_UNKNOWN_ID=true
 USER 1001
 ENTRYPOINT ["runmqdevserver"]

Makefile

@@ -1,4 +1,4 @@
-# © Copyright IBM Corporation 2017, 2020
+# © Copyright IBM Corporation 2017, 2021
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,10 +16,11 @@
 # Conditional variables - you can override the values of these variables from
 # the command line
 ###############################################################################
+
+include config.env
+
 # RELEASE shows what release of the container code has been built
 RELEASE ?=
-# MQ_VERSION is the fully qualified MQ version number to build
-MQ_VERSION ?= 9.2.0.0
 # MQ_ARCHIVE_REPOSITORY is a remote repository from which to pull the MQ_ARCHIVE (if required)
 MQ_ARCHIVE_REPOSITORY ?=
 # MQ_ARCHIVE_REPOSITORY_DEV is a remote repository from which to pull the MQ_ARCHIVE_DEV (if required)
@@ -38,12 +39,16 @@ MQ_ARCHIVE_DEV ?= $(MQ_VERSION)-IBM-MQ-Advanced-for-Developers-Non-Install-$(MQ_
 MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION))
 # Options to `go test` for the Docker tests
 TEST_OPTS_DOCKER ?=
+# Timeout for the Docker tests
+TEST_TIMEOUT_DOCKER ?= 30m
 # MQ_IMAGE_ADVANCEDSERVER is the name of the built MQ Advanced image
 MQ_IMAGE_ADVANCEDSERVER ?=ibm-mqadvanced-server
 # MQ_IMAGE_DEVSERVER is the name of the built MQ Advanced for Developers image
 MQ_IMAGE_DEVSERVER ?=ibm-mqadvanced-server-dev
+# MQ_MANIFEST_TAG is the tag to use for fat-manifest
+MQ_MANIFEST_TAG ?= $(MQ_VERSION)$(RELEASE_TAG)$(LTS_TAG)$(MQ_MANIFEST_TAG_SUFFIX)
 # MQ_TAG is the tag of the built MQ Advanced image & MQ Advanced for Developers image
-MQ_TAG ?=$(MQ_VERSION)-$(ARCH)
+MQ_TAG ?= $(MQ_MANIFEST_TAG)-$(ARCH)
 # COMMAND is the container command to run.  "podman" or "docker"
 COMMAND ?=$(shell type -p podman 2>&1 >/dev/null && echo podman || echo docker)
 # MQ_DELIVERY_REGISTRY_HOSTNAME is a remote registry to push the MQ Image to (if required)
@@ -60,10 +65,16 @@ REGISTRY_USER ?=
 REGISTRY_PASS ?=
 # ARCH is the platform architecture (e.g. amd64, ppc64le or s390x)
 ARCH ?= $(if $(findstring x86_64,$(shell uname -m)),amd64,$(shell uname -m))
+# LTS is a boolean value to enable/disable LTS container build
+LTS ?= false
+# VOLUME_MOUNT_OPTIONS is used when bind-mounting files from the "downloads" directory into the container.  By default, SELinux labels are automatically re-written, but this doesn't work on some filesystems with extended attributes (xattrs).  You can turn off the label re-writing by setting this variable to be blank.
+VOLUME_MOUNT_OPTIONS ?= :Z
 
 ###############################################################################
 # Other variables
 ###############################################################################
+# Build doesn't work if BuildKit is enabled
+DOCKER_BUILDKIT=0
 GO_PKG_DIRS = ./cmd ./internal ./test
 MQ_ARCHIVE_TYPE=LINUX
 MQ_ARCHIVE_DEV_TYPE=Linux
@@ -71,7 +82,7 @@ MQ_ARCHIVE_DEV_TYPE=Linux
 BUILD_SERVER_CONTAINER=build-server
 # NUM_CPU is the number of CPUs available to Docker.  Used to control how many
 # test run in parallel
-NUM_CPU = $(or $(shell docker info --format "{{ .NCPU }}"),2)
+NUM_CPU ?= $(or $(shell $(COMMAND) info --format "{{ .NCPU }}"),2)
 # BASE_IMAGE_TAG is a normalized version of BASE_IMAGE, suitable for use in a Docker tag
 BASE_IMAGE_TAG=$(lastword $(subst /, ,$(subst :,-,$(BASE_IMAGE))))
 #BASE_IMAGE_TAG=$(subst /,-,$(subst :,-,$(BASE_IMAGE)))
@@ -104,19 +115,79 @@ else ifeq "$(ARCH)" "s390x"
 	MQ_ARCHIVE_ARCH=S390X
 endif
 
+# LTS_TAG is the tag modifier for an LTS container build
+LTS_TAG=
+ifeq "$(LTS)" "true"
+ifneq "$(LTS_TAG_OVERRIDE)" "$(EMPTY)"
+	LTS_TAG=$(LTS_TAG_OVERRIDE)
+else
+	LTS_TAG=-lts
+endif
+	MQ_ARCHIVE:=$(MQ_VERSION)-IBM-MQ-Advanced-Non-Install-Linux$(MQ_ARCHIVE_ARCH).tar.gz
+	MQ_DELIVERY_REGISTRY_NAMESPACE:=$(MQ_DELIVERY_REGISTRY_NAMESPACE)$(LTS_TAG)
+endif
+
+ifneq (,$(findstring release-candidate,$(TRAVIS_TAG)))
+    MQ_DELIVERY_REGISTRY_NAMESPACE=release-candidates
+endif
+
 ifneq "$(MQ_DELIVERY_REGISTRY_NAMESPACE)" "$(EMPTY)"
 	MQ_DELIVERY_REGISTRY_FULL_PATH=$(MQ_DELIVERY_REGISTRY_HOSTNAME)/$(MQ_DELIVERY_REGISTRY_NAMESPACE)
 else
 	MQ_DELIVERY_REGISTRY_FULL_PATH=$(MQ_DELIVERY_REGISTRY_HOSTNAME)
 endif
 
+# image tagging
+
 ifneq "$(RELEASE)" "$(EMPTY)"
-	MQ_TAG=$(MQ_VERSION)-$(RELEASE)-$(ARCH)
+	EXTRA_LABELS_RELEASE=--label "release=$(RELEASE)"
-	EXTRA_LABELS=--label release=$(RELEASE)
+	RELEASE_TAG="-$(RELEASE)"
 endif
+
+ifneq "$(MQ_ARCHIVE_LEVEL)" "$(EMPTY)"
+	EXTRA_LABELS_LEVEL=--label "mq-build=$(MQ_ARCHIVE_LEVEL)"
+endif
+
+EXTRA_LABELS=$(EXTRA_LABELS_RELEASE) $(EXTRA_LABELS_LEVEL)
+
+ifeq "$(TIMESTAMPFLAT)" "$(EMPTY)"
+	TIMESTAMPFLAT=$(shell date "+%Y%m%d%H%M%S")
+endif
+
+ifeq "$(GIT_COMMIT)" "$(EMPTY)"
+	GIT_COMMIT=$(shell git rev-parse --short HEAD)
+endif
+
+ifeq ($(shell [ ! -z $(TRAVIS) ] && [ "$(TRAVIS_PULL_REQUEST)" = "false" ] && [ "$(TRAVIS_BRANCH)" = "$(MAIN_BRANCH)" ] && echo true), true)
+	MQ_MANIFEST_TAG_SUFFIX=.$(TIMESTAMPFLAT).$(GIT_COMMIT)
+endif
+
+PATH_TO_MQ_TAG_CACHE=$(TRAVIS_BUILD_DIR)/.tagcache
+ifneq "$(TRAVIS)" "$(EMPTY)"
+ifneq ("$(wildcard $(PATH_TO_MQ_TAG_CACHE))","")
+include $(PATH_TO_MQ_TAG_CACHE)
+endif
+endif
+
+MQ_AMD64_TAG=$(MQ_MANIFEST_TAG)-amd64
+MQ_S390X_TAG?=$(MQ_MANIFEST_TAG)-s390x
+MQ_PPC64LE_TAG?=$(MQ_MANIFEST_TAG)-ppc64le
+
+# end image tagging
+
 MQ_IMAGE_FULL_RELEASE_NAME=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG)
 MQ_IMAGE_DEV_FULL_RELEASE_NAME=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG)
 
+#setup variables for fat-manifests
+MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)
+MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)
+MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_AMD64_TAG)
+MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_S390X_TAG)
+MQ_IMAGE_DEVSERVER_PPC64LE=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_PPC64LE_TAG)
+MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_AMD64_TAG)
+MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_S390X_TAG)
+MQ_IMAGE_ADVANCEDSERVER_PPC64LE=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_PPC64LE_TAG)
+
 ###############################################################################
 # Build targets
 ###############################################################################
@@ -133,6 +204,9 @@ test-all: build-devjmstest test-devserver test-advancedserver
 .PHONY: devserver
 devserver: build-devserver build-devjmstest test-devserver
 
+.PHONY: advancedserver
+advancedserver: build-advancedserver test-advancedserver
+
 # Build incubating components
 .PHONY: incubating
 incubating: build-explorer
@@ -140,47 +214,70 @@ incubating: build-explorer
 downloads/$(MQ_ARCHIVE_DEV):
 	$(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END)))
 	mkdir -p downloads
+ifneq "$(BUILD_RSYNC_SERVER)" "$(EMPTY)"
+# Use key which is not stored in the repository to fetch the files from the fileserver
+	curl -L $(BUILD_RSYNC_ENCRYPTED_KEY_URL) -o ./host.key.gpg
+	@echo $(BUILD_RSYNC_ENCRYPTION_PASSWORD)|gpg --batch --passphrase-fd 0 ./host.key.gpg
+	chmod 600 ./host.key
+	rsync -rv -e "ssh -o BatchMode=yes -q -o StrictHostKeyChecking=no -i ./host.key" --include="*/" --include="*.tar.gz" --exclude="*" $(BUILD_RSYNC_USER)@$(BUILD_RSYNC_SERVER):"$(BUILD_RSYNC_PATH)" downloads/$(MQ_ARCHIVE_DEV)
+	-@rm host.key.gpg host.key
+else
 ifneq "$(MQ_ARCHIVE_REPOSITORY_DEV)" "$(EMPTY)"
 	curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY_DEV)" -o downloads/$(MQ_ARCHIVE_DEV)
 else
 	curl -L https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_ARCHIVE_DEV) -o downloads/$(MQ_ARCHIVE_DEV)
 endif
+endif
 
 downloads/$(MQ_ARCHIVE):
 	$(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced "$(MQ_VERSION)$(END)))
 	mkdir -p downloads
+ifneq "$(BUILD_RSYNC_SERVER)" "$(EMPTY)"
+# Use key which is not stored in the repository to fetch the files from the fileserver
+	-@rm host.key.gpg host.key
+	curl -L $(BUILD_RSYNC_ENCRYPTED_KEY_URL) -o ./host.key.gpg
+	@echo $(BUILD_RSYNC_ENCRYPTION_PASSWORD)|gpg --batch --passphrase-fd 0 ./host.key.gpg
+	chmod 600 ./host.key
+	rsync -rv -e "ssh -o BatchMode=yes -q -o StrictHostKeyChecking=no -i ./host.key" --include="*/" --include="*.tar.gz" --exclude="*" $(BUILD_RSYNC_USER)@$(BUILD_RSYNC_SERVER):"$(BUILD_RSYNC_PATH)" downloads/$(MQ_ARCHIVE)
+	-@rm host.key.gpg host.key
+else
 ifneq "$(MQ_ARCHIVE_REPOSITORY)" "$(EMPTY)"
 	curl -u $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -X GET "$(MQ_ARCHIVE_REPOSITORY)" -o downloads/$(MQ_ARCHIVE)
 endif
+endif
 
 .PHONY: downloads
 downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE)
 
+.PHONY: cache-mq-tag
+cache-mq-tag:
+	@printf "MQ_MANIFEST_TAG=$(MQ_MANIFEST_TAG)\n" | tee $(PATH_TO_MQ_TAG_CACHE)
+
 # Vendor Go dependencies for the Docker tests
 test/docker/vendor:
-	cd test/docker && dep ensure -vendor-only
+	cd test/docker && go mod vendor
 
 # Shortcut to just run the unit tests
 .PHONY: test-unit
 test-unit:
-	docker build --target builder --file Dockerfile-server .
+	$(COMMAND) build --target builder --file Dockerfile-server .
 
 .PHONY: test-advancedserver
 test-advancedserver: test/docker/vendor
-	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) on $(shell docker --version)"$(END)))
+	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) on $(shell $(COMMAND) --version)"$(END)))
-	docker inspect $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG)
+	$(COMMAND) inspect $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG)
-	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER)
+	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) -timeout $(TEST_TIMEOUT_DOCKER) $(TEST_OPTS_DOCKER)
 
 .PHONY: build-devjmstest
-build-devjmstest:
+build-devjmstest: registry-login
 	$(info $(SPACER)$(shell printf $(TITLE)"Build JMS tests for developer config"$(END)))
 	cd test/messaging && docker build --tag $(DEV_JMS_IMAGE) .
 
 .PHONY: test-devserver
 test-devserver: test/docker/vendor
-	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER):$(MQ_TAG) on $(shell docker --version)"$(END)))
+	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER):$(MQ_TAG) on $(shell $(COMMAND) --version)"$(END)))
-	docker inspect $(MQ_IMAGE_DEVSERVER):$(MQ_TAG)
+	$(COMMAND) inspect $(MQ_IMAGE_DEVSERVER):$(MQ_TAG)
-	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER)
+	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -timeout $(TEST_TIMEOUT_DOCKER) -tags mqdev $(TEST_OPTS_DOCKER)
 
 .PHONY: coverage
 coverage:
@@ -188,7 +285,7 @@ coverage:
 
 .PHONY: test-advancedserver-cover
 test-advancedserver-cover: test/docker/vendor coverage
-	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) with code coverage on $(shell docker --version)"$(END)))
+	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) with code coverage on $(shell $(COMMAND) --version)"$(END)))
 	rm -f ./coverage/unit*.cov
 	# Run unit tests with coverage, for each package under 'internal'
 	go list -f '{{.Name}}' ./internal/... | xargs -I {} go test -cover -covermode count -coverprofile ./coverage/unit-{}.cov ./internal/{}
@@ -212,13 +309,13 @@ test-advancedserver-cover: test/docker/vendor coverage
 # Build an MQ image.  The commands used are slightly different between Docker and Podman
 define build-mq
 	$(if $(findstring docker,$(COMMAND)), @docker network create build,)
-	$(if $(findstring docker,$(COMMAND)), @docker run --rm --name $(BUILD_SERVER_CONTAINER) --network build --network-alias build --volume $(DOWNLOADS_DIR):/usr/share/nginx/html:ro --detach docker.io/nginx:alpine,)
+	$(if $(findstring docker,$(COMMAND)), @docker run --rm --name $(BUILD_SERVER_CONTAINER) --network build --network-alias build --volume $(DOWNLOADS_DIR):/opt/app-root/src$(VOLUME_MOUNT_OPTIONS) --detach registry.redhat.io/ubi8/nginx-118 nginx -g "daemon off;",)
-	$(eval EXTRA_ARGS=$(if $(findstring docker,$(COMMAND)), --network build --build-arg MQ_URL=http://build:80/$4, --volume $(DOWNLOADS_DIR):/var/downloads --build-arg MQ_URL=file:///var/downloads/$4))
+	$(eval EXTRA_ARGS=$(if $(findstring docker,$(COMMAND)), --network build --build-arg MQ_URL=http://build:8080/$4, --volume $(DOWNLOADS_DIR):/var/downloads$(VOLUME_MOUNT_OPTIONS) --build-arg MQ_URL=file:///var/downloads/$4))
 	# Build the new image
 	$(COMMAND) build \
 	  --tag $1:$2 \
 	  --file $3 \
-		$(EXTRA_ARGS) \
+	  $(EXTRA_ARGS) \
 	  --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" \
 	  --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" \
 	  --build-arg IMAGE_TAG="$1:$2" \
@@ -226,7 +323,7 @@ define build-mq
 	  --label name=$1 \
 	  --label build-date=$(shell date +%Y-%m-%dT%H:%M:%S%z) \
 	  --label architecture="$(ARCH)" \
-	  --label run="docker run -d -e LICENSE=accept $1:$2" \
+	  --label run="podman run -d -e LICENSE=accept $1:$2" \
 	  --label vcs-ref=$(IMAGE_REVISION) \
 	  --label vcs-type=git \
 	  --label vcs-url=$(IMAGE_SOURCE) \
@@ -237,15 +334,15 @@ define build-mq
 	$(if $(findstring docker,$(COMMAND)), @docker network rm build)
 endef
 
-DOCKER_SERVER_VERSION=$(shell docker version --format "{{ .Server.Version }}")
+COMMAND_SERVER_VERSION=$(shell $(COMMAND) version --format "{{ .Server.Version }}")
-DOCKER_CLIENT_VERSION=$(shell docker version --format "{{ .Client.Version }}")
+COMMAND_CLIENT_VERSION=$(shell $(COMMAND) version --format "{{ .Client.Version }}")
 PODMAN_VERSION=$(shell podman version --format "{{ .Version }}")
 .PHONY: command-version
 command-version:
 # If we're using Docker, then check it's recent enough to support multi-stage builds
 ifneq (,$(findstring docker,$(COMMAND)))
-	@test "$(word 1,$(subst ., ,$(DOCKER_CLIENT_VERSION)))" -ge "17" || ("$(word 1,$(subst ., ,$(DOCKER_CLIENT_VERSION)))" -eq "17" && "$(word 2,$(subst ., ,$(DOCKER_CLIENT_VERSION)))" -ge "05") || (echo "Error: Docker client 17.05 or greater is required" && exit 1)
+	@test "$(word 1,$(subst ., ,$(COMMAND_CLIENT_VERSION)))" -ge "17" || ("$(word 1,$(subst ., ,$(COMMAND_CLIENT_VERSION)))" -eq "17" && "$(word 2,$(subst ., ,$(COMMAND_CLIENT_VERSION)))" -ge "05") || (echo "Error: Docker client 17.05 or greater is required" && exit 1)
-	@test "$(word 1,$(subst ., ,$(DOCKER_SERVER_VERSION)))" -ge "17" || ("$(word 1,$(subst ., ,$(DOCKER_SERVER_VERSION)))" -eq "17" && "$(word 2,$(subst ., ,$(DOCKER_CLIENT_VERSION)))" -ge "05") || (echo "Error: Docker server 17.05 or greater is required" && exit 1)
+	@test "$(word 1,$(subst ., ,$(COMMAND_SERVER_VERSION)))" -ge "17" || ("$(word 1,$(subst ., ,$(COMMAND_SERVER_VERSION)))" -eq "17" && "$(word 2,$(subst ., ,$(COMMAND_CLIENT_VERSION)))" -ge "05") || (echo "Error: Docker server 17.05 or greater is required" && exit 1)
 endif
 ifneq (,$(findstring podman,$(COMMAND)))
 	@test "$(word 1,$(subst ., ,$(PODMAN_VERSION)))" -ge "1" || (echo "Error: Podman version 1.0 or greater is required" && exit 1)
@@ -342,6 +439,34 @@ pull-devserver:
 	$(COMMAND) pull $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME)
 	$(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG)
 
+.PHONY: push-manifest
+push-manifest: build-skopeo-container
+	$(info $(SPACER)$(shell printf $(TITLE)"** Determining the image digests **"$(END)))
+ifneq "$(LTS)" "true"
+	$(eval MQ_IMAGE_DEVSERVER_AMD64_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_DEVSERVER_AMD64) | jq -r .Digest))
+	$(eval MQ_IMAGE_DEVSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_DEVSERVER_S390X) | jq -r .Digest))
+	$(eval MQ_IMAGE_DEVSERVER_PPC64LE_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_DEVSERVER_PPC64LE) | jq -r .Digest))
+	$(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_AMD64) has a digest of $(MQ_IMAGE_DEVSERVER_AMD64_DIGEST)**"$(END)))
+	$(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_S390X) has a digest of $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)**"$(END)))
+	$(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_PPC64LE) has a digest of $(MQ_IMAGE_DEVSERVER_PPC64LE_DIGEST)**"$(END)))
+endif
+	$(eval MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_AMD64) | jq -r .Digest))
+	$(eval MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_S390X) | jq -r .Digest))
+	$(eval MQ_IMAGE_ADVANCEDSERVER_PPC64LE_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_PPC64LE) | jq -r .Digest))
+	$(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_AMD64) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST)**"$(END)))
+	$(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_S390X) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)**"$(END)))
+	$(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_PPC64LE) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_PPC64LE_DIGEST)**"$(END)))
+ifneq "$(LTS)" "true"
+	$(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_DEVSERVER_MANIFEST)**"$(END)))
+	echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL)  -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST) $(MQ_IMAGE_DEVSERVER_PPC64LE_DIGEST)" $(END))
+endif
+	$(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_ADVANCEDSERVER_MANIFEST)**"$(END)))
+	echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_PPC64LE_DIGEST)" $(END))
+
+.PHONY: build-skopeo-container
+build-skopeo-container:
+	$(COMMAND) images | grep -q "skopeo"; if [ $$? != 0 ]; then $(COMMAND) build -t skopeo:latest ./docker-builds/skopeo/; fi
+
 .PHONY: clean
 clean:
 	rm -rf ./coverage
@@ -404,3 +529,16 @@ gosec:
 fi ;\
 
 include formatting.mk
+
+.PHONY: update-release-information
+update-release-information:
+	sed -i.bak 's/ARG MQ_URL=.*-LinuxX64.tar.gz"/ARG MQ_URL="https:\/\/public.dhe.ibm.com\/ibmdl\/export\/pub\/software\/websphere\/messaging\/mqadv\/$(MQ_VERSION)-IBM-MQ-Advanced-for-Developers-Non-Install-LinuxX64.tar.gz"/g' Dockerfile-server && rm Dockerfile-server.bak
+	$(eval MQ_VERSION_1=$(shell echo '${MQ_VERSION}' | rev | cut -c 3- | rev))
+	sed -i.bak 's/IBM_MQ_.*_LINUX_X86-64_NOINST.tar.gz/IBM_MQ_${MQ_VERSION_1}_LINUX_X86-64_NOINST.tar.gz/g' docs/building.md && rm docs/building.md.bak
+	sed -i.bak 's/ibm-mqadvanced-server:.*-amd64/ibm-mqadvanced-server:$(MQ_VERSION)-amd64/g' docs/security.md
+	sed -i.bak 's/ibm-mqadvanced-server-dev.*-amd64/ibm-mqadvanced-server-dev:$(MQ_VERSION)-amd64/g' docs/security.md && rm docs/security.md.bak
+	sed -i.bak 's/MQ_IMAGE_ADVANCEDSERVER=ibm-mqadvanced-server:.*-amd64/MQ_IMAGE_ADVANCEDSERVER=ibm-mqadvanced-server:$(MQ_VERSION)-amd64/g' docs/testing.md && rm docs/testing.md.bak
+	$(eval MQ_VERSION_2=$(shell echo '${MQ_VERSION_1}' | rev | cut -c 3- | rev))
+	sed -i.bak 's/knowledgecenter\/SSFKSJ_.*\/com/knowledgecenter\/SSFKSJ_${MQ_VERSION_2}.0\/com/g' docs/usage.md && rm docs/usage.md.bak
+	$(eval MQ_VERSION_3=$(shell echo '${MQ_VERSION_1}' | sed "s/\.//g"))
+	sed -i.bak 's/MQ_..._ARCHIVE_REPOSITORY/MQ_${MQ_VERSION_3}_ARCHIVE_REPOSITORY/g' .travis.yml && rm .travis.yml.bak

README.md

@@ -1,5 +1,6 @@
 # IBM MQ container
 
+
 [![Build Status](https://travis-ci.org/ibm-messaging/mq-container.svg?branch=master)](https://travis-ci.org/ibm-messaging/mq-container)
 
 **Note**: The `master` branch may be in an *unstable or even broken state* during development.
@@ -44,11 +45,12 @@ For issues relating specifically to the container image or Helm chart, please us
 The Dockerfiles and associated code and scripts are licensed under the [Apache License 2.0](http://www.apache.org/licenses/LICENSE-2.0.html).
 Licenses for the products installed within the images are as follows:
 
-- [IBM MQ Advanced for Developers](http://www14.software.ibm.com/cgi-bin/weblap/lap.pl?la_formnum=Z125-3301-14&li_formnum=L-APIG-BMKG5H) (International License Agreement for Non-Warranted Programs). This license may be viewed from an image using the `LICENSE=view` environment variable as described above or by following the link above.
+- [IBM MQ Advanced for Developers](http://www14.software.ibm.com/cgi-bin/weblap/lap.pl?la_formnum=Z125-3301-14&li_formnum=L-APIG-BYHCL7) (International License Agreement for Non-Warranted Programs). This license may be viewed from an image using the `LICENSE=view` environment variable as described above or by following the link above.
-- [IBM MQ Advanced](http://www14.software.ibm.com/cgi-bin/weblap/lap.pl?la_formnum=Z125-3301-14&li_formnum=L-APIG-BMJJBM) (International Program License Agreement). This license may be viewed from an image using the `LICENSE=view` environment variable as described above or by following the link above.
+- [IBM MQ Advanced](http://www14.software.ibm.com/cgi-bin/weblap/lap.pl?la_formnum=Z125-3301-14&li_formnum=L-APIG-BZDDDY) (International Program License Agreement). This license may be viewed from an image using the `LICENSE=view` environment variable as described above or by following the link above.
 
 Note: The IBM MQ Advanced for Developers license does not permit further distribution and the terms restrict usage to a developer machine.
 
+
 ## Copyright
 
 © Copyright IBM Corporation 2015, 2020

authservice/mqhtpass/Makefile

@@ -0,0 +1,50 @@
+# © Copyright IBM Corporation 2017, 2020
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This Makefile expects the following to be installed:
+# - gcc
+# - ldd
+# - MQ SDK (mqm_r library, plus header files)
+# - Apache Portable Runtime (apr-1 and aprutil-1 libraries, plus header files)
+
+SRC_DIR = src
+BUILD_DIR = ./build
+
+# Flags passed to the C compiler.  Need to use gnu11 to get POSIX functions needed for file locking.
+CFLAGS += -std=gnu11 -fPIC -Wall -m64
+
+LIB_APR = -L/usr/lib64 -lapr-1 -laprutil-1
+LIB_MQ = -L/opt/mqm/lib64 -lmqm_r
+
+all: $(BUILD_DIR)/mqhtpass.so $(BUILD_DIR)/htpass_test
+
+$(BUILD_DIR)/log.o : $(SRC_DIR)/log.c $(SRC_DIR)/log.h
+	mkdir -p ${dir $@}
+	gcc $(CFLAGS) -c $(SRC_DIR)/log.c -o $@
+
+$(BUILD_DIR)/htpass.o : $(SRC_DIR)/htpass.c $(SRC_DIR)/htpass.h
+	mkdir -p ${dir $@}
+	gcc $(CFLAGS) -c $(SRC_DIR)/htpass.c -I /usr/include/apr-1 -o $@
+
+$(BUILD_DIR)/htpass_test : $(BUILD_DIR)/htpass.o $(BUILD_DIR)/log.o
+	mkdir -p ${dir $@}
+	gcc $(CFLAGS) $(LIB_APR) -lpthread $(SRC_DIR)/htpass_test.c $^ -o $@
+# Run HTPasswd tests, and print log if they fail	
+	$@ || (cat htpass_test*.log && exit 1)
+
+$(BUILD_DIR)/mqhtpass.so : $(BUILD_DIR)/log.o $(BUILD_DIR)/htpass.o
+	mkdir -p ${dir $@}
+	# NOTE: rpath for libapr will be different on Ubuntu
+	gcc $(CFLAGS) -I/opt/mqm/inc -D_REENTRANT $(LIB_APR) $(LIB_MQ) -Wl,-rpath,/opt/mqm/lib64 -Wl,-rpath,/usr/lib64 -shared $(SRC_DIR)/mqhtpass.c $^ -o $@
+	ldd $@

authservice/mqhtpass/src/htpass.c

@@ -0,0 +1,145 @@
+/*
+© Copyright IBM Corporation 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "log.h"
+#include "htpass.h"
+#include <linux/limits.h>
+#include <apr_general.h>
+#include <apr_errno.h>
+#include <apr_md5.h>
+
+bool htpass_valid_file(char *filename)
+{
+  bool valid = true;
+  FILE *fp;
+  char *huser;
+
+  fp = fopen(filename, "r");
+  if (fp == NULL)
+  {
+    log_errorf("Error %d opening htpasswd file '%s'", errno, filename);
+  }
+  if (fp)
+  {
+    const size_t line_size = 1024;
+    char *line = malloc(line_size);
+    while (fgets(line, line_size, fp) != NULL)
+    {
+      char *saveptr;
+      // Need to use strtok_r to be safe for multiple threads
+      huser = strtok_r(line, ":", &saveptr);
+      if (strlen(huser) >= 12)
+      {
+        log_errorf("Invalid htpasswd file for use with IBM MQ.  User '%s' is longer than twelve characters", huser);
+        valid = false;
+        break;
+      }
+    }
+    fclose(fp);
+    if (line)
+    {
+      free(line);
+    }
+  }
+  return valid;
+}
+
+char *find_hash(char *filename, char *user)
+{
+  bool found = false;
+  FILE *fp;
+  char *huser;
+  char *hash;
+
+  fp = fopen(filename, "r");
+  if (fp == NULL)
+  {
+    log_errorf("Error %d opening htpasswd file '%s'", errno, filename);
+  }
+  if (fp)
+  {
+    const size_t line_size = 1024;
+    char *line = malloc(line_size);
+    while (fgets(line, line_size, fp) != NULL)
+    {
+      char *saveptr;
+      // Need to use strtok_r to be safe for multiple threads
+      huser = strtok_r(line, ":", &saveptr);
+      if (huser && (strcmp(user, huser) == 0))
+      {
+        // Make a duplicate of the string, because we'll be keeping it
+        hash = strdup(strtok_r(NULL, " \r\n\t", &saveptr));
+        found = true;
+        break;
+      }
+    }
+    fclose(fp);
+    if (line)
+    {
+      free(line);
+    }
+  }
+  if (!found)
+  {
+    hash = NULL;
+  }
+  return hash;
+}
+
+int htpass_authenticate_user(char *filename, char *user, char *password)
+{
+  char *hash = find_hash(filename, user);
+  int result = -1;
+  if (hash == NULL)
+  {
+    result = HTPASS_INVALID_USER;
+    log_debugf("User does not exist. user=%s", user);
+  }
+  else
+  {
+    // Use the Apache Portable Runtime utilities to validate the password against the hash.
+    // Supports multiple hashing algorithms, but we should only be using bcrypt
+    apr_status_t status = apr_password_validate(password, hash);
+    // status is usually either APR_SUCCESS or APR_EMISMATCH
+    if (status == APR_SUCCESS)
+    {
+      result = HTPASS_VALID;
+      log_debugf("Correct password supplied. user=%s", user);
+    }
+    else
+    {
+      result = HTPASS_INVALID_PASSWORD;
+      log_debugf("Incorrect password supplied. user=%s", user);
+    }
+  }
+  return result;
+}
+
+bool htpass_valid_user(char *filename, char *user)
+{
+  char *hash = find_hash(filename, user);
+  bool valid = false;
+  if (hash != NULL)
+  {
+    valid = true;
+  }
+  return valid;
+}

authservice/mqhtpass/src/htpass.h

@@ -0,0 +1,49 @@
+/*
+© Copyright IBM Corporation 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef _HTPASS_H
+#define _HTPASS_H
+
+#define HTPASS_VALID 0
+#define HTPASS_INVALID_USER 1
+#define HTPASS_INVALID_PASSWORD 2
+
+/**
+ * Validate an HTPasswd file for use with IBM MQ.
+ * 
+ * @param filename the HTPasswd file
+ */
+_Bool htpass_valid_file(char *filename);
+
+/**
+ * Authenticate a user, based on the supplied file name.
+ * 
+ * @param filename the HTPasswd file
+ * @param user the user name to authenticate
+ * @param password the password of the user
+ * @return HTPASS_VALID, HTPASS_INVALID_USER or HTPASS_INVALID_PASSWORD
+ */
+int htpass_authenticate_user(char *filename, char *user, char *password);
+
+/**
+ * Validate that a user exists in the password file.
+ * 
+ * @param filename the HTPasswd file
+ * @param user the user name to validate
+ */
+_Bool htpass_valid_user(char *filename, char *user);
+
+#endif

authservice/mqhtpass/src/htpass_test.c

@@ -0,0 +1,223 @@
+/*
+© Copyright IBM Corporation 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "log.h"
+#include "htpass.h"
+
+// Headers for multi-threaded tests
+#include <pthread.h>
+
+// Start a test and log the function name
+#define test_start() printf("=== RUN: %s\n", __func__)
+
+// Indicate test has passed
+#define test_pass() printf("--- PASS: %s\n", __func__)
+
+// Indicate test has failed
+void test_fail(const char *test_name)
+{
+  printf("--- FAIL: %s\n", test_name);
+  exit(1);
+}
+
+// ----------------------------------------------------------------------------
+// Simple tests for file validation
+// ----------------------------------------------------------------------------
+
+void test_htpass_valid_file_ok()
+{
+  test_start();
+  int ok = htpass_valid_file("./src/htpass_test.htpasswd");
+  if (!ok)
+    test_fail(__func__);
+  test_pass();
+}
+
+void test_htpass_valid_file_too_long()
+{
+  test_start();
+  int ok = htpass_valid_file("./src/htpass_test_invalid.htpasswd");
+  if (ok)
+    test_fail(__func__);
+  test_pass();
+}
+
+// ----------------------------------------------------------------------------
+// Simple tests for authentication
+// ----------------------------------------------------------------------------
+
+void test_htpass_authenticate_user_fred_valid()
+{
+  test_start();
+  int rc = htpass_authenticate_user("./src/htpass_test.htpasswd", "fred", "passw0rd");
+  printf("%s: fred - %d\n", __func__, rc);
+  if (rc != HTPASS_VALID)
+    test_fail(__func__);
+  test_pass();
+}
+
+void test_htpass_authenticate_user_fred_invalid1()
+{
+  test_start();
+  int rc = htpass_authenticate_user("./src/htpass_test.htpasswd", "fred", "passw0rd ");
+  printf("%s: fred - %d\n", __func__, rc);
+  if (rc != HTPASS_INVALID_PASSWORD)
+    test_fail(__func__);
+  test_pass();
+}
+
+void test_htpass_authenticate_user_fred_invalid2()
+{
+  test_start();
+  int rc = htpass_authenticate_user("./src/htpass_test.htpasswd", "fred", "");
+  printf("%s: fred - %d\n", __func__, rc);
+  if (rc != HTPASS_INVALID_PASSWORD)
+    test_fail(__func__);
+  test_pass();
+}
+
+void test_htpass_authenticate_user_fred_invalid3()
+{
+  test_start();
+  int rc = htpass_authenticate_user("./src/htpass_test.htpasswd", "fred", "clearlywrong");
+  printf("%s: fred - %d\n", __func__, rc);
+  if (rc != HTPASS_INVALID_PASSWORD)
+    test_fail(__func__);
+  test_pass();
+}
+
+void test_htpass_authenticate_user_barney_valid()
+{
+  test_start();
+  int rc = htpass_authenticate_user("./src/htpass_test.htpasswd", "barney", "s3cret");
+  printf("%s: barney - %d\n", __func__, rc);
+  if (rc != HTPASS_VALID)
+    test_fail(__func__);
+  test_pass();
+}
+
+void test_htpass_authenticate_user_unknown()
+{
+  test_start();
+  int rc = htpass_authenticate_user("./src/htpass_test.htpasswd", "george", "s3cret");
+  printf("%s: barney - %d\n", __func__, rc);
+  if (rc != HTPASS_INVALID_USER)
+    test_fail(__func__);
+  test_pass();
+}
+
+// ----------------------------------------------------------------------------
+// Multi-threaded test
+// ----------------------------------------------------------------------------
+
+#define NUM_THREADS 5
+// Number of tests to perform per thread.  Higher numbers are more likely to trigger timing issue.
+#define NUM_TESTS_PER_THREAD 1000
+// Maximum number of JSON errors to report (log can get flooded)
+#define MAX_JSON_ERRORS 10
+
+// Authenticate multiple users, multiple times
+void *authenticate_many_times(void *p)
+{
+  for (int i = 0; i < NUM_TESTS_PER_THREAD; i++)
+  {
+    int rc = htpass_authenticate_user("./src/htpass_test.htpasswd", "barney", "s3cret");
+    if (rc != HTPASS_VALID)
+      test_fail(__func__);
+    rc = htpass_authenticate_user("./src/htpass_test.htpasswd", "fred", "passw0rd");
+    if (rc != HTPASS_VALID)
+      test_fail(__func__);
+  }
+  pthread_exit(NULL);
+}
+
+void check_log_file_valid(char *filename)
+{
+  int errors = 0;
+  printf("--- Checking log file is valid\n");
+  // Check that the JSON log file isn't corrupted
+  FILE *log = fopen(filename, "r");
+  if (log == NULL)
+  {
+    test_fail(__func__);
+  }
+  const size_t line_size = 1024;
+  char *line = malloc(line_size);
+  while (fgets(line, line_size, log) != NULL)
+  {
+    if ((line[0] != '{') && (errors < MAX_JSON_ERRORS))
+    {
+      printf("*** Invalid JSON detected: %s\n", line);
+      errors++;
+    }
+  }
+  if (line)
+  {
+    free(line);
+  }
+  fclose(log);
+}
+
+// Test authenticate_user with multiple threads, each doing many authentications
+void test_htpass_authenticate_user_multithreaded(char *logfile)
+{
+  pthread_t threads[NUM_THREADS];
+  int rc;
+  test_start();
+  // Re-initialize the log to use a file for the multi-threaded test
+  log_init(logfile);
+  for (int i = 0; i < NUM_THREADS; i++)
+  {
+    printf("Creating thread %d\n", i);
+    rc = pthread_create(&threads[i], NULL, authenticate_many_times, NULL);
+    if (rc)
+    {
+      printf("Error: Unable to create thread, %d\n", rc);
+      test_fail(__func__);
+    }
+  }
+  // Wait for all the threads to complete
+  for (int i = 0; i < NUM_THREADS; i++)
+  {
+    pthread_join(threads[i], NULL);
+  }
+  check_log_file_valid(logfile);
+  test_pass();
+}
+
+// ----------------------------------------------------------------------------
+
+int main()
+{
+  // Turn on debugging for the tests
+  setenv("DEBUG", "true", true);
+  log_init("htpass_test.log");
+  test_htpass_valid_file_ok();
+  test_htpass_valid_file_too_long();
+  test_htpass_authenticate_user_fred_valid();
+  test_htpass_authenticate_user_fred_invalid1();
+  test_htpass_authenticate_user_fred_invalid2();
+  test_htpass_authenticate_user_fred_invalid3();
+  test_htpass_authenticate_user_barney_valid();
+  test_htpass_authenticate_user_unknown();
+  log_close();
+
+  // Call multi-threaded test last, because it re-initializes the log to use a file
+  test_htpass_authenticate_user_multithreaded("htpass_test_multithreaded.log");
+}

authservice/mqhtpass/src/htpass_test.htpasswd

@@ -0,0 +1,2 @@
+fred:$2y$05$3Fp9epsqEwWOHdyj9Ngf9.qfX34kzc9zNrdQ7kac0GmcCvQjIkAwy
+barney:$2y$05$l8EoyCQ9y2PyfUzIDDfTyu7SSaJEYB1TuHy07xZvN7xt/pR3SIw0a

authservice/mqhtpass/src/htpass_test_invalid.htpasswd

@@ -0,0 +1,3 @@
+fred:$2y$05$3Fp9epsqEwWOHdyj9Ngf9.qfX34kzc9zNrdQ7kac0GmcCvQjIkAwy
+barney:$2y$05$l8EoyCQ9y2PyfUzIDDfTyu7SSaJEYB1TuHy07xZvN7xt/pR3SIw0a
+namewhichisfartoolongformq:$2y$05$l8EoyCQ9y2PyfUzIDDfTyu7SSaJEYB1TuHy07xZvN7xt/pR3SIw0a

authservice/mqhtpass/src/log.c

@@ -0,0 +1,152 @@
+/*
+© Copyright IBM Corporation 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <time.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+FILE *fp = NULL;
+int pid;
+char hostname[255];
+bool debug = false;
+
+/**
+ * Determine whether debugging is enabled or not, using an environment variable.
+ */
+void init_debug(){
+  char *debug_env = getenv("DEBUG");
+  if (debug_env != NULL)
+  {
+    // Enable debug logging if the DEBUG environment variable is set
+    if (strncmp(debug_env, "true", 4) || strncmp(debug_env, "1", 1))
+    {
+      debug = true;
+    }
+  }
+}
+
+/**
+ * Internal function to initialize the log with the given file mode.
+ */
+int log_init_internal(char *filename, const char *mode)
+{
+  int result = 0;
+  pid = getpid();
+  hostname[254] = '\0';
+  gethostname(hostname, 254);
+  if (!fp)
+  {
+    fp = fopen(filename, "a");
+    if (fp)
+    {
+      // Disable buffering for this file
+      setbuf(fp, NULL);
+    }
+    else
+    {
+      result = 1;
+    }
+  }
+  init_debug();
+  return result;
+}
+
+int log_init_reset(char *filename)
+{
+  // Open the log file for writing (overwrite if it already exists)
+  return log_init_internal(filename, "w");
+}
+
+int log_init(char *filename)
+{
+  // Open the log file file for appending
+  return log_init_internal(filename, "a");
+}
+
+void log_init_file(FILE *f)
+{
+  fp = f;
+  init_debug();
+}
+
+void log_close()
+{
+  if (fp)
+  {
+    fclose(fp);
+    fp = NULL;
+  }
+}
+
+void log_printf(const char *source_file, int source_line, const char *level, const char *format, ...)
+{
+  if (fp)
+  {
+    // If this is a DEBUG message, and debugging is off
+    if ((strncmp(level, "DEBUG", 5) == 0) && !debug)
+    {
+      return;
+    }
+    char buf[1024] = "";
+    char *cur = buf;
+    char* const end = buf + sizeof buf;
+    char date_buf[70];
+    struct tm *utc;
+    time_t t;
+    struct timeval now;
+
+    gettimeofday(&now, NULL);
+    t = now.tv_sec;
+    t = time(NULL);
+    utc = gmtime(&t);
+
+    cur += snprintf(cur, end-cur, "{");
+    cur += snprintf(cur, end-cur, "\"loglevel\":\"%s\"", level);
+    // Print ISO-8601 time and date
+    if (strftime(date_buf, sizeof date_buf, "%FT%T", utc))
+    {
+       // Round microseconds down to milliseconds, for consistency
+       cur += snprintf(cur, end-cur, ", \"ibm_datetime\":\"%s.%03ldZ\"", date_buf, now.tv_usec / 1000);
+    }
+    cur += snprintf(cur, end-cur, ", \"ibm_processId\":\"%d\"", pid);
+    cur += snprintf(cur, end-cur, ", \"host\":\"%s\"", hostname);
+    cur += snprintf(cur, end-cur, ", \"module\":\"%s:%d\"", source_file, source_line);
+    cur += snprintf(cur, end-cur, ", \"message\":\"");
+
+    if (strncmp(level, "DEBUG", 5) == 0)
+    {
+      // Add a prefix on any debug messages
+      cur += snprintf(cur, end-cur, "mqhtpass: ");
+    }
+
+    // Print log message, using varargs
+    va_list args;
+    va_start(args, format);
+    cur += vsnprintf(cur, end-cur, format, args);
+    va_end(args);
+    cur += snprintf(cur, end-cur, "\"}\n");
+
+    // Important: Just do one file write, to prevent problems with multi-threading.
+    // This only works if the log message is not too long for the buffer.
+    fprintf(fp, buf);
+  }
+}
+

authservice/mqhtpass/src/log.h

@@ -0,0 +1,63 @@
+/*
+© Copyright IBM Corporation 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef _LOG_H
+#define _LOG_H
+
+/**
+ * Initialize the log to use the given file name, wiping any existing contents.
+ */
+int log_init_reset(char *filename);
+
+/**
+ * Initialize the log to use the given file name.
+ */
+int log_init(char *filename);
+
+/**
+ * Initialize the log with an existing file handle.
+ */
+void log_init_file(FILE *f);
+
+/**
+ * Write a message to the log file, based on a printf format string.
+ * 
+ * @param source_file the name of the source code file submitting this log message
+ * @param source_line the line of code in the source file
+ * @param level the log level, one of "DEBUG", "INFO" or "ERROR"
+ * @param format the printf format string for the message
+ */
+void log_printf(const char *source_file, int source_line, const char *level, const char *format, ...);
+
+void log_close();
+
+/**
+ * Variadic macro to write an informational message to the log file, based on a printf format string.
+ */
+#define log_infof(format,...) log_printf(__FILE__, __LINE__, "INFO", format, ##__VA_ARGS__)
+
+/**
+ * Variadic macro to write an error message to the log file, based on a printf format string.
+ */
+#define log_errorf(format,...) log_printf(__FILE__, __LINE__, "ERROR", format, ##__VA_ARGS__)
+
+/**
+ * Variadic macro to write a debug message to the log file, based on a printf format string.
+ */
+#define log_debugf(format,...) log_printf(__FILE__, __LINE__, "DEBUG", format, ##__VA_ARGS__)
+
+
+#endif

authservice/mqhtpass/src/mqhtpass.c

@@ -0,0 +1,351 @@
+/*
+© Copyright IBM Corporation 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This is a developer only configuration and not recommended for production usage.
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <cmqec.h>
+#include "log.h"
+#include "htpass.h"
+
+// Declare the internal functions that implement the interface
+MQZ_INIT_AUTHORITY MQStart;
+static MQZ_AUTHENTICATE_USER mqhtpass_authenticate_user;
+static MQZ_FREE_USER mqhtpass_free_user;
+static MQZ_TERM_AUTHORITY mqhtpass_terminate;
+
+#define LOG_FILE "/var/mqm/errors/mqhtpass.json"
+#define HTPASSWD_FILE "/etc/mqm/mq.htpasswd"
+#define NAME "MQ Advanced for Developers custom authentication service"
+
+static char *trim(char *s);
+
+/**
+ * Initialization and entrypoint for the dynamically loaded
+ * authorization installable service. It registers the addresses of the
+ * other functions which are to be called by the queue manager.
+ *
+ * This function is called whenever the module is loaded.  The Options
+ * field will show whether it's a PRIMARY (i.e. during qmgr startup) or
+ * SECONDARY.
+ */
+void MQENTRY MQStart(
+    MQHCONFIG hc,
+    MQLONG Options,
+    MQCHAR48 QMgrName,
+    MQLONG ComponentDataLength,
+    PMQBYTE ComponentData,
+    PMQLONG Version,
+    PMQLONG pCompCode,
+    PMQLONG pReason)
+{
+  MQLONG CC = MQCC_OK;
+  MQLONG Reason = MQRC_NONE;
+  int log_rc = 0;
+
+  if (Options == MQZIO_PRIMARY)
+  {
+    // Reset the log file.  The file could still get large if debug is turned on,
+    // but this is a simpler solution for now.
+    log_rc = log_init_reset(LOG_FILE);
+  }
+  else
+  {
+    log_rc = log_init(LOG_FILE);
+  }
+
+  if (log_rc != 0)
+  {
+    CC = MQCC_FAILED;
+    Reason = MQRC_INITIALIZATION_FAILED;
+  }
+
+  if (Options == MQZIO_PRIMARY)
+  {
+    log_infof("Initializing %s", NAME);
+  }
+  log_debugf("MQStart options=%s qmgr=%s", ((Options == MQZIO_SECONDARY) ? "Secondary" : "Primary"), trim(QMgrName));
+
+  if (!htpass_valid_file(HTPASSWD_FILE))
+  {
+    CC = MQCC_FAILED;
+    Reason = MQRC_INITIALIZATION_FAILED;
+  }
+
+  // Initialize the functions to use for each entry point
+  if (CC == MQCC_OK)
+  {
+    hc->MQZEP_Call(hc, MQZID_INIT_AUTHORITY, (PMQFUNC)MQStart, &CC, &Reason);
+  }
+  if (CC == MQCC_OK)
+  {
+    hc->MQZEP_Call(hc, MQZID_TERM_AUTHORITY, (PMQFUNC)mqhtpass_terminate, &CC, &Reason);
+  }
+  if (CC == MQCC_OK)
+  {
+    hc->MQZEP_Call(hc, MQZID_AUTHENTICATE_USER, (PMQFUNC)mqhtpass_authenticate_user, &CC, &Reason);
+  }
+  if (CC == MQCC_OK)
+  {
+    hc->MQZEP_Call(hc, MQZID_FREE_USER, (PMQFUNC)mqhtpass_free_user, &CC, &Reason);
+  }
+  *Version = MQZAS_VERSION_5;
+  *pCompCode = CC;
+  *pReason = Reason;
+  return;
+}
+
+/**
+ * Called during the connection of any application which supplies an MQCSP (Connection Security Parameters).
+ * This is the usual case.
+ * See https://www.ibm.com/support/knowledgecenter/SSFKSJ_latest/com.ibm.mq.ref.dev.doc/q095610_.html
+ */
+static void MQENTRY mqhtpass_authenticate_user_csp(
+    PMQCHAR pQMgrName,
+    PMQCSP pSecurityParms,
+    PMQZAC pApplicationContext,
+    PMQZIC pIdentityContext,
+    PMQPTR pCorrelationPtr,
+    PMQBYTE pComponentData,
+    PMQLONG pContinuation,
+    PMQLONG pCompCode,
+    PMQLONG pReason)
+{
+  char *csp_user = NULL;
+  char *csp_pass = NULL;
+
+  // Firstly, create null-terminated strings from the user credentials in the MQ CSP object
+  csp_user = malloc(pSecurityParms->CSPUserIdLength + 1);
+  if (!csp_user)
+  {
+    log_errorf("%s is unable to allocate memory for a user", NAME);
+    *pCompCode = MQCC_FAILED;
+    *pReason = MQRC_SERVICE_ERROR;
+    return;
+  }
+  strncpy(csp_user, pSecurityParms->CSPUserIdPtr, pSecurityParms->CSPUserIdLength);
+  csp_user[pSecurityParms->CSPUserIdLength] = 0;
+  csp_pass = malloc((pSecurityParms->CSPPasswordLength + 1));
+  if (!csp_pass)
+  {
+    log_errorf("%s is unable to allocate memory for a password", NAME);
+    *pCompCode = MQCC_FAILED;
+    *pReason = MQRC_SERVICE_ERROR;
+    if (csp_user)
+    {
+      free(csp_user);
+    }
+    return;
+  }
+  strncpy(csp_pass, pSecurityParms->CSPPasswordPtr, pSecurityParms->CSPPasswordLength);
+  csp_pass[pSecurityParms->CSPPasswordLength] = 0;
+  log_debugf("%s with CSP user set. user=%s", __func__, csp_user);
+  int auth_result = htpass_authenticate_user(HTPASSWD_FILE, csp_user, csp_pass);
+
+  if (auth_result == HTPASS_VALID)
+  {
+    // An OK completion code means MQ will accept this user is authenticated
+    *pCompCode = MQCC_OK;
+    *pReason = MQRC_NONE;
+    // Tell the queue manager to stop trying other authorization services.
+    *pContinuation = MQZCI_STOP;
+    memcpy(pIdentityContext->UserIdentifier, csp_user, sizeof(pIdentityContext->UserIdentifier));
+    log_debugf("Authenticated user=%s", pIdentityContext->UserIdentifier);
+  }
+  // If the htpasswd file does not have an entry for this user
+  else if (auth_result == HTPASS_INVALID_USER)
+  {
+    *pCompCode = MQCC_WARNING;
+    *pReason = MQRC_NONE;
+    // Tell the queue manager to continue trying other authorization services, as they might have the user.
+    *pContinuation = MQZCI_CONTINUE;
+    log_debugf(
+        "User authentication failed due to invalid user.  user=%s effuser=%s applname=%s csp_user=%s cc=%d reason=%d",
+        trim(pIdentityContext->UserIdentifier),
+        trim(pApplicationContext->EffectiveUserID),
+        trim(pApplicationContext->ApplName),
+        trim(csp_user),
+        *pCompCode,
+        *pReason);
+  }
+  // If the htpasswd file has an entry for this user, but the password supplied is incorrect
+  else if (auth_result == HTPASS_INVALID_PASSWORD)
+  {
+    *pCompCode = MQCC_WARNING;
+    *pReason = MQRC_NOT_AUTHORIZED;
+    // Tell the queue manager to stop trying other authorization services.
+    *pContinuation = MQZCI_STOP;
+    log_debugf(
+        "User authentication failed due to invalid password.  user=%s effuser=%s applname=%s csp_user=%s cc=%d reason=%d",
+        trim(pIdentityContext->UserIdentifier),
+        trim(pApplicationContext->EffectiveUserID),
+        trim(pApplicationContext->ApplName),
+        trim(csp_user),
+        *pCompCode,
+        *pReason);
+  }
+  if (csp_user)
+  {
+    free(csp_user);
+  }
+  if (csp_pass)
+  {
+    free(csp_pass);
+  }
+  return;
+}
+
+/**
+ * Called during the connection of any application.
+ * For more information on the parameters, see https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_latest/com.ibm.mq.ref.dev.doc/q110090_.html
+ */
+static void MQENTRY mqhtpass_authenticate_user(
+    PMQCHAR pQMgrName,
+    PMQCSP pSecurityParms,
+    PMQZAC pApplicationContext,
+    PMQZIC pIdentityContext,
+    PMQPTR pCorrelationPtr,
+    PMQBYTE pComponentData,
+    PMQLONG pContinuation,
+    PMQLONG pCompCode,
+    PMQLONG pReason)
+{
+  char *spuser = NULL;
+  char *sppass = NULL;
+  // By default, return a warning, which indicates to MQ that this
+  // authorization service hasn't authenticated the user.
+  *pCompCode = MQCC_WARNING;
+  *pReason = MQRC_NONE;
+  // By default, tell the queue manager to continue trying other
+  // authorization services.
+  *pContinuation = MQZCI_CONTINUE;
+
+  if ((pSecurityParms->AuthenticationType) == MQCSP_AUTH_USER_ID_AND_PWD)
+  {
+    mqhtpass_authenticate_user_csp(pQMgrName, pSecurityParms, pApplicationContext, pIdentityContext, pCorrelationPtr, pComponentData, pContinuation, pCompCode, pReason);
+  }
+  else
+  {
+    // Password not supplied, so just check that the user ID is valid
+    spuser = malloc(sizeof(PMQCHAR12) + 1);
+    if (!spuser)
+    {
+      log_errorf("%s is unable to allocate memory to check a user", NAME);
+      *pCompCode = MQCC_FAILED;
+      *pReason = MQRC_SERVICE_ERROR;
+      return;
+    }
+    strncpy(spuser, pApplicationContext->EffectiveUserID, strlen(pApplicationContext->EffectiveUserID));
+    spuser[sizeof(PMQCHAR12)] = 0;
+    log_debugf("%s without CSP user set.  effectiveuid=%s env=%d, callertype=%d, type=%d, accttoken=%d applidentitydata=%d", __func__, spuser, pApplicationContext->Environment, pApplicationContext->CallerType, pApplicationContext->AuthenticationType, pIdentityContext->AccountingToken, pIdentityContext->ApplIdentityData);
+    if (strncmp(spuser, "mqm", 3) == 0)
+    {
+      // Special case: pass the "mqm" user on for validation up the chain
+      // A warning in the completion code means MQ will pass this to other authorization services
+      *pCompCode = MQCC_WARNING;
+      *pReason = MQRC_NONE;
+      *pContinuation = MQZCI_CONTINUE;
+    }
+    else
+    {
+      bool valid_user = htpass_valid_user(HTPASSWD_FILE, spuser);
+      if (valid_user)
+      {
+        // An OK completion code means MQ will accept this user is authenticated
+        *pCompCode = MQCC_OK;
+        *pReason = MQRC_NONE;
+        *pContinuation = MQZCI_STOP;
+        memcpy(pIdentityContext->UserIdentifier, spuser, sizeof(pIdentityContext->UserIdentifier));
+      }
+      else
+      {
+        log_debugf(
+            "User authentication failed user=%s effuser=%s applname=%s cspuser=%s cc=%d reason=%d",
+            trim(pIdentityContext->UserIdentifier),
+            trim(pApplicationContext->EffectiveUserID),
+            trim(pApplicationContext->ApplName),
+            trim(spuser),
+            *pCompCode,
+            *pReason);
+      }
+      if (spuser)
+      {
+        free(spuser);
+      }
+    }
+  }
+  return;
+}
+
+/**
+ * Called during MQDISC, as the inverse of the call to authenticate.
+ */
+static void MQENTRY mqhtpass_free_user(
+    PMQCHAR pQMgrName,
+    PMQZFP pFreeParms,
+    PMQBYTE pComponentData,
+    PMQLONG pContinuation,
+
+    PMQLONG pCompCode,
+    PMQLONG pReason)
+{
+  log_debugf("mqhtpass_freeuser()");
+  *pCompCode = MQCC_WARNING;
+  *pReason = MQRC_NONE;
+  *pContinuation = MQZCI_CONTINUE;
+}
+
+/**
+ * Called when the authorization service is terminated.
+ */
+static void MQENTRY mqhtpass_terminate(
+    MQHCONFIG hc,
+    MQLONG Options,
+    PMQCHAR pQMgrName,
+    PMQBYTE pComponentData,
+    PMQLONG pCompCode,
+    PMQLONG pReason)
+{
+  if (Options == MQZTO_PRIMARY)
+  {
+    log_infof("Terminating %s", NAME);
+    log_close();
+  }
+  else {
+    log_debugf("Terminating secondary");
+  }
+  *pCompCode = MQCC_OK;
+  *pReason = MQRC_NONE;
+}
+
+/**
+ * Remove trailing spaces from a string.
+ */
+static char *trim(char *s)
+{
+  int i;
+  for (i = strlen(s) - 1; i >= 0; i--)
+  {
+    if (s[i] == ' ')
+      s[i] = 0;
+    else
+      break;
+  }
+  return s;
+}

cmd/chkmqhealthy/main.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2017, 2019
+© Copyright IBM Corporation 2017, 2020
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -36,12 +36,12 @@ func queueManagerHealthy() (bool, error) {
 	cmd := exec.Command("dspmq", "-n", "-m", name)
 	// Run the command and wait for completion
 	out, err := cmd.CombinedOutput()
+	fmt.Printf("%s", out)
 	if err != nil {
 		fmt.Println(err)
 		return false, err
 	}
-	fmt.Printf("%s", out)
+	if !strings.Contains(string(out), "(RUNNING)") && !strings.Contains(string(out), "(RUNNING AS STANDBY)") && !strings.Contains(string(out), "(STARTING)") && !strings.Contains(string(out), "(REPLICA)") {
-	if !strings.Contains(string(out), "(RUNNING)") && !strings.Contains(string(out), "(RUNNING AS STANDBY)") && !strings.Contains(string(out), "(STARTING)") {
 		return false, nil
 	}
 	return true, nil

cmd/chkmqready/main.go

@@ -39,7 +39,7 @@ func main() {
 	}
 
 	// Check if the queue manager has a running listener
-	if standby, _ := ready.IsRunningAsStandbyQM(name); !standby {
+	if active, _ := ready.IsRunningAsActiveQM(name); active {
 		conn, err := net.Dial("tcp", "127.0.0.1:1414")
 		if err != nil {
 			fmt.Println(err)
@@ -49,8 +49,13 @@ func main() {
 		if err != nil {
 			fmt.Println(err)
 		}
-	} else {
+	} else if standby, _ := ready.IsRunningAsStandbyQM(name); standby {
 		fmt.Printf("Detected queue manager running in standby mode")
 		os.Exit(10)
+	} else if replica, _ := ready.IsRunningAsReplicaQM(name); replica {
+		fmt.Printf("Detected queue manager running in replica mode")
+		os.Exit(20)
+	} else {
+		os.Exit(1)
 	}
 }

cmd/chkmqstarted/main.go

@@ -0,0 +1,58 @@
+/*
+© Copyright IBM Corporation 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// chkmqstarted checks that MQ has successfully started, by checking the output of the "dspmq" command
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/ibm-messaging/mq-container/pkg/name"
+)
+
+func queueManagerStarted() (bool, error) {
+	name, err := name.GetQueueManagerName()
+	if err != nil {
+		return false, err
+	}
+	// Specify the queue manager name, just in case someone's created a second queue manager
+	// #nosec G204
+	cmd := exec.Command("dspmq", "-n", "-m", name)
+	// Run the command and wait for completion
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		fmt.Println(err)
+		return false, err
+	}
+	if !strings.Contains(string(out), "(RUNNING)") && !strings.Contains(string(out), "(RUNNING AS STANDBY)") && !strings.Contains(string(out), "(STARTING)") && !strings.Contains(string(out), "(REPLICA)") {
+		return false, nil
+	}
+	return true, nil
+}
+
+func main() {
+	started, err := queueManagerStarted()
+	if err != nil {
+		os.Exit(2)
+	}
+	if !started {
+		os.Exit(1)
+	}
+	os.Exit(0)
+}

cmd/runmqdevserver/main.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2018, 2020
+© Copyright IBM Corporation 2018, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,7 +19,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"syscall"
 
 	"github.com/ibm-messaging/mq-container/internal/htpasswd"
@@ -30,27 +29,6 @@ import (
 
 var log *logger.Logger
 
-func setPassword(user string, password string) error {
-	// #nosec G204
-	cmd := exec.Command("sudo", "chpasswd")
-	stdin, err := cmd.StdinPipe()
-	if err != nil {
-		return err
-	}
-	fmt.Fprintf(stdin, "%s:%s", user, password)
-	err = stdin.Close()
-	if err != nil {
-		log.Errorf("Error closing password stdin: %v", err)
-	}
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		// Include the command output in the error
-		return fmt.Errorf("%v: %v", err.Error(), out)
-	}
-	log.Printf("Set password for \"%v\" user", user)
-	return nil
-}
-
 func getLogFormat() string {
 	return os.Getenv("LOG_FORMAT")
 }

cmd/runmqserver/logging.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2017, 2020
+© Copyright IBM Corporation 2017, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -79,6 +79,8 @@ func formatBasic(obj map[string]interface{}) string {
 	if len(inserts) > 0 {
 		return fmt.Sprintf("%s %s [%v]\n", obj["ibm_datetime"], obj["message"], strings.Join(inserts, ", "))
 	}
+	// Convert time zone information from some logs (e.g. Liberty) for consistency
+	obj["ibm_datetime"] = strings.Replace(obj["ibm_datetime"].(string), "+0000", "Z", 1)
 	return fmt.Sprintf("%s %s\n", obj["ibm_datetime"], obj["message"])
 }
 
@@ -100,6 +102,16 @@ func mirrorQueueManagerErrorLogs(ctx context.Context, wg *sync.WaitGroup, name s
 	return mirrorLog(ctx, wg, f, fromStart, mf, true)
 }
 
+// mirrorHTPasswdLogs starts a goroutine to mirror the contents of the MQ HTPasswd authorization service's log
+func mirrorHTPasswdLogs(ctx context.Context, wg *sync.WaitGroup, name string, fromStart bool, mf mirrorFunc) (chan error, error) {
+	return mirrorLog(ctx, wg, "/var/mqm/errors/mqhtpass.json", false, mf, true)
+}
+
+// mirrorWebServerLogs starts a goroutine to mirror the contents of the Liberty web server messages.log
+func mirrorWebServerLogs(ctx context.Context, wg *sync.WaitGroup, name string, fromStart bool, mf mirrorFunc) (chan error, error) {
+	return mirrorLog(ctx, wg, "/var/mqm/web/installations/Installation1/servers/mqweb/logs/messages.log", false, mf, true)
+}
+
 func getDebug() bool {
 	debug := os.Getenv("DEBUG")
 	if debug == "true" || debug == "1" {
@@ -119,14 +131,21 @@ func configureLogger(name string) (mirrorFunc, error) {
 			return nil, err
 		}
 		return func(msg string, isQMLog bool) bool {
-			obj, err := processLogMessage(msg)
+			// Check if the message is JSON
-			if err == nil && isQMLog && filterQMLogMessage(obj) {
+			if len(msg) > 0 && msg[0] == '{' {
-				return false
+				obj, err := processLogMessage(msg)
-			}
+				if err == nil && isQMLog && filterQMLogMessage(obj) {
-			if err != nil {
+					return false
-				log.Printf("Failed to unmarshall JSON - %v", msg)
+				}
+				if err != nil {
+					log.Printf("Failed to unmarshall JSON in log message - %v", msg)
+				} else {
+					fmt.Println(msg)
+				}
 			} else {
-				fmt.Println(msg)
+				// The log being mirrored isn't JSON, so wrap it in a simple JSON message
+				// MQ error logs are usually JSON, but this is useful for Liberty logs - usually expect WLP_LOGGING_MESSAGE_FORMAT=JSON to be set when mirroring Liberty logs.
+				fmt.Printf("{\"message\":\"%s\"}\n", msg)
 			}
 			return true
 		}, nil
@@ -136,16 +155,22 @@ func configureLogger(name string) (mirrorFunc, error) {
 			return nil, err
 		}
 		return func(msg string, isQMLog bool) bool {
-			// Parse the JSON message, and print a simplified version
+			// Check if the message is JSON
-			obj, err := processLogMessage(msg)
+			if len(msg) > 0 && msg[0] == '{' {
-			if err == nil && isQMLog && filterQMLogMessage(obj) {
+				// Parse the JSON message, and print a simplified version
-				return false
+				obj, err := processLogMessage(msg)
-			}
+				if err == nil && isQMLog && filterQMLogMessage(obj) {
-			if err != nil {
+					return false
-				log.Printf("Failed to unmarshall JSON - %v", err)
+				}
+				if err != nil {
+					log.Printf("Failed to unmarshall JSON in log message - %v", err)
+				} else {
+					fmt.Printf(formatBasic(obj))
+				}
 			} else {
-				fmt.Printf(formatBasic(obj))
+				// The log being mirrored isn't JSON, so just print it.
-				// fmt.Printf(formatSimple(obj["ibm_datetime"].(string), obj["message"].(string)))
+				// MQ error logs are usually JSON, but this is useful for Liberty logs - usually expect WLP_LOGGING_MESSAGE_FORMAT=JSON to be set when mirroring Liberty logs.
+				fmt.Println(msg)
 			}
 			return true
 		}, nil

cmd/runmqserver/main.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2017, 2020
+© Copyright IBM Corporation 2017, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@ import (
 	"os"
 	"sync"
 
+	"github.com/ibm-messaging/mq-container/internal/ha"
 	"github.com/ibm-messaging/mq-container/internal/metrics"
 	"github.com/ibm-messaging/mq-container/internal/ready"
 	"github.com/ibm-messaging/mq-container/internal/tls"
@@ -143,30 +144,55 @@ func doMain() error {
 	// Print out versioning information
 	logVersionInfo()
 
-	keyLabel, cmsKeystore, p12Truststore, err := tls.ConfigureTLSKeystores()
+	keyLabel, defaultCmsKeystore, defaultP12Truststore, err := tls.ConfigureDefaultTLSKeystores()
 	if err != nil {
 		logTermination(err)
 		return err
 	}
 
-	err = tls.ConfigureTLS(keyLabel, cmsKeystore, *devFlag, log)
+	err = tls.ConfigureTLS(keyLabel, defaultCmsKeystore, *devFlag, log)
 	if err != nil {
 		logTermination(err)
 		return err
 	}
 
-	err = postInit(name, keyLabel, p12Truststore)
+	err = postInit(name, keyLabel, defaultP12Truststore)
 	if err != nil {
 		logTermination(err)
 		return err
 	}
 
+	if os.Getenv("MQ_NATIVE_HA") == "true" {
+		err = ha.ConfigureNativeHA(log)
+		if err != nil {
+			logTermination(err)
+			return err
+		}
+	}
+
+	enableTraceCrtmqm := os.Getenv("MQ_ENABLE_TRACE_CRTMQM")
+	if enableTraceCrtmqm == "true" || enableTraceCrtmqm == "1" {
+		err = startMQTrace()
+		if err != nil {
+			logTermination(err)
+			return err
+		}
+	}
+
 	newQM, err := createQueueManager(name, *devFlag)
 	if err != nil {
 		logTermination(err)
 		return err
 	}
 
+	if enableTraceCrtmqm == "true" || enableTraceCrtmqm == "1" {
+		err = endMQTrace()
+		if err != nil {
+			logTermination(err)
+			return err
+		}
+	}
+
 	var wg sync.WaitGroup
 	defer func() {
 		log.Debug("Waiting for log mirroring to complete")
@@ -188,6 +214,23 @@ func doMain() error {
 		logTermination(err)
 		return err
 	}
+	if *devFlag {
+		_, err = mirrorHTPasswdLogs(ctx, &wg, name, newQM, mf)
+		if err != nil {
+			logTermination(err)
+			return err
+		}
+	}
+	// Recommended to use this option in conjunction with setting WLP_LOGGING_MESSAGE_FORMAT=JSON
+	mirrorWebLog := os.Getenv("MQ_ENABLE_EMBEDDED_WEB_SERVER_LOG")
+	if mirrorWebLog == "true" || mirrorWebLog == "1" {
+		_, err = mirrorWebServerLogs(ctx, &wg, name, newQM, mf)
+		if err != nil {
+			logTermination(err)
+			return err
+		}
+	}
+
 	err = updateCommandLevel()
 	if err != nil {
 		logTermination(err)

cmd/runmqserver/qmgr.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ibm-messaging/mq-container/internal/command"
 	containerruntime "github.com/ibm-messaging/mq-container/internal/containerruntime"
 	"github.com/ibm-messaging/mq-container/internal/mqscredact"
+	"github.com/ibm-messaging/mq-container/internal/mqversion"
 	"github.com/ibm-messaging/mq-container/internal/ready"
 )
 
@@ -112,9 +113,13 @@ func startQueueManager(name string) error {
 	out, rc, err := command.Run("strmqm", "-x", name)
 	if err != nil {
 		// 30=standby queue manager started, which is fine
+		// 94=native HA replica started, which is fine
 		if rc == 30 {
 			log.Printf("Started standby queue manager")
 			return nil
+		} else if rc == 94 {
+			log.Printf("Started replica queue manager")
+			return nil
 		}
 		log.Printf("Error %v starting queue manager: %v", rc, string(out))
 		return err
@@ -203,9 +208,28 @@ func getQueueManagerDataDir(mounts map[string]string, name string) string {
 }
 
 func getCreateQueueManagerArgs(mounts map[string]string, name string, devMode bool) []string {
+
+	mqversionBase := "9.2.1.0"
+
+	// use "UserExternal" only if we are 9.2.1.0 or above.
+	oaVal := "user"
+	mqVersionCheck, err := mqversion.Compare(mqversionBase)
+
+	if err != nil {
+		log.Printf("Error comparing MQ versions for oa,rc: %v", mqVersionCheck)
+	}
+	if mqVersionCheck >= 0 {
+		oaVal = "UserExternal"
+	}
+
+	//build args
 	args := []string{"-ii", "/etc/mqm/", "-ic", "/etc/mqm/", "-q", "-p", "1414"}
+
+	if os.Getenv("MQ_NATIVE_HA") == "true" {
+		args = append(args, "-lr", os.Getenv("HOSTNAME"))
+	}
 	if devMode {
-		args = append(args, "-oa", "user")
+		args = append(args, "-oa", oaVal)
 	}
 	if _, ok := mounts["/mnt/mqm-log"]; ok {
 		args = append(args, "-ld", "/mnt/mqm-log/log")

cmd/runmqserver/version.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"github.com/ibm-messaging/mq-container/internal/command"
+	"github.com/ibm-messaging/mq-container/internal/mqversion"
 )
 
 var (
@@ -50,7 +51,7 @@ func logImageTag() {
 }
 
 func logMQVersion() {
-	mqVersion, _, err := command.Run("dspmqver", "-b", "-f", "2")
+	mqVersion, err := mqversion.Get()
 	if err != nil {
 		log.Printf("Error Getting MQ version: %v", strings.TrimSuffix(string(mqVersion), "\n"))
 	}

cmd/runmqserver/webserver.go

@@ -62,15 +62,31 @@ func startWebServer(webKeystore, webkeystorePW, webTruststoreRef string) error {
 }
 
 func configureSSO(p12TrustStore tls.KeyStoreData, webKeystore string) (string, error) {
-	// Ensure all required environment variables are set for SSO
+	requiredEnvVars := []string{}
-	requiredEnvVars := []string{
+	_, set := os.LookupEnv("MQ_ZEN_INTERNAL_ENDPOINT")
-		"MQ_OIDC_CLIENT_ID",
+	if !set {
-		"MQ_OIDC_CLIENT_SECRET",
+		// Ensure all required environment variables are set for SSO
-		"MQ_OIDC_UNIQUE_USER_IDENTIFIER",
+		requiredEnvVars = []string{
-		"MQ_OIDC_AUTHORIZATION_ENDPOINT",
+			"MQ_OIDC_CLIENT_ID",
-		"MQ_OIDC_TOKEN_ENDPOINT",
+			"MQ_OIDC_CLIENT_SECRET",
-		"MQ_OIDC_JWK_ENDPOINT",
+			"MQ_OIDC_UNIQUE_USER_IDENTIFIER",
-		"MQ_OIDC_ISSUER_IDENTIFIER",
+			"MQ_OIDC_AUTHORIZATION_ENDPOINT",
+			"MQ_OIDC_TOKEN_ENDPOINT",
+			"MQ_OIDC_JWK_ENDPOINT",
+			"MQ_OIDC_ISSUER_IDENTIFIER",
+		}
+	} else {
+		// Ensure all required environment variables are set for Zen SSO
+		requiredEnvVars = []string{
+			"MQ_ZEN_UNIQUE_USER_IDENTIFIER",
+			"MQ_ZEN_INTERNAL_ENDPOINT",
+			"MQ_ZEN_ISSUER_IDENTIFIER",
+			"MQ_ZEN_AUDIENCES",
+			"MQ_ZEN_CONTEXT_NAME",
+			"MQ_ZEN_BASE_URI",
+			"MQ_ZEN_CONTEXT_NAMESPACE",
+			"IAM_URL",
+		}
 	}
 	for _, envVar := range requiredEnvVars {
 		if len(os.Getenv(envVar)) == 0 {

config.env

@@ -0,0 +1,6 @@
+###########################################################################################################################################################
+
+# MQ_VERSION is the fully qualified MQ version number to build
+MQ_VERSION ?= 9.2.5.0
+
+###########################################################################################################################################################

docker-builds/skopeo/Dockerfile

@@ -1,4 +1,4 @@
-# © Copyright IBM Corporation 2017, 2018
+# © Copyright IBM Corporation 2020
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,13 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-[[constraint]]
+FROM fedora:32
-  name = "github.com/docker/docker"
+RUN yum install skopeo -y -qq
-  version = "=v17.03.2-ce"
+ENTRYPOINT [ "skopeo" ]
-
-[[constraint]]
-  name = "github.com/docker/go-connections"
-  version = "0.4.0"
-
-[prune]
-  go-tests = true

docs/building.md

@@ -13,14 +13,18 @@ You will also need a [Red Hat Account](https://access.redhat.com) to be able to
 
 ## Building a production image
 
+From MQ 9.2.X, the MQ container adds support for MQ Long Term Support (LTS) **production licensed** releases.
+
+### MQ Continuous Delivery (CD)
+
 This procedure works for building the MQ Continuous Delivery release, on `amd64`, `ppc64le` and `s390x` architectures.
 
 1. Create a `downloads` directory in the root of this repository
-2. Download MQ from [IBM Passport Advantage](https://www.ibm.com/software/passportadvantage/) or [IBM Fix Central](https://www.ibm.com/support/fixcentral), and place the downloaded file (for example, `IBM_MQ_9.2.0_LINUX_X86-64_NOINST.tar.gz`) in the `downloads` directory
+2. Download MQ from [IBM Passport Advantage](https://www.ibm.com/software/passportadvantage/) or [IBM Fix Central](https://www.ibm.com/support/fixcentral), and place the downloaded file (for example, `IBM_MQ_9.2.5_LINUX_X86-64_NOINST.tar.gz`) in the `downloads` directory
 3. Login to the Red Hat Registry: `docker login registry.redhat.io` using your Customer Portal credentials.
 4. Run `make build-advancedserver`
 
-> **Warning**: Note that from MQ 9.2.X, the MQ container build uses a 'No-Install' MQ Package, available under `IBM MQ V9.2.x Continuous Delivery Release components eAssembly, part no. CJ7CNML`
+> **Warning**: Note that from MQ 9.2.X CD, the MQ container build uses a 'No-Install' MQ Package, available under `IBM MQ V9.2.x Continuous Delivery Release components eAssembly, part no. CJ7CNML`
 
 If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory).  You should also specify the MQ version, so that the resulting image is tagged correctly, for example:
 
@@ -28,6 +32,23 @@ If you have an MQ archive file with a different file name, you can specify a par
 MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 make build-advancedserver
 ```
 
+### MQ Long Term Support (LTS)
+
+This procedure works for building the MQ Long Term Support release, on `amd64`, `ppc64le` and `s390x` architectures.
+
+1. Create a `downloads` directory in the root of this repository
+2. Download MQ from [IBM Passport Advantage](https://www.ibm.com/software/passportadvantage/) or [IBM Fix Central](https://www.ibm.com/support/fixcentral), and place the downloaded file (for example, `9.2.0.4-IBM-MQ-Advanced-Non-Install-LinuxX86.tar.gz`) in the `downloads` directory
+3. Login to the Red Hat Registry: `docker login registry.redhat.io` using your Customer Portal credentials.
+4. Run `LTS=true make build-advancedserver`
+
+> **Warning**: Note that from MQ 9.2 LTS, the MQ container build uses a 'No-Install' MQ Package, available under `IBM MQ V9.2 Long Term Support Release components eAssembly, part no. CXXXXXX`
+
+If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory).  You should also specify the MQ version, so that the resulting image is tagged correctly, for example:
+
+```bash
+MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 LTS=true make build-advancedserver
+```
+
 ## Building a developer image
 
 Login to the Red Hat Registry: `docker login registry.redhat.io` using your Customer Portal credentials.

docs/internals.md

@@ -11,6 +11,7 @@ The resulting Docker image contains the following:
    - `runmqdevserver` - The main process for MQ Advanced for Developers
    - `chkmqhealthy` - Checks the health of the queue manager.  This can be used by (say) a Kubernetes liveness probe.
    - `chkmqready` - Checks if the queue manager is ready for work.  This can be used by (say) a Kubernetes readiness probe.
+   - `chkmqstarted` - Checks if the queue manager has successfully started.  This can be used by (say) a Kubernetes startup probe.
 
 ## runmqserver
 The `runmqserver` command has the following responsibilities:

docs/pluggable-connauth.md

@@ -24,6 +24,6 @@ Use an administrative tool or your application to connect to queue manager using
 
 #### Troubleshooting
 
-A log file named `amqpasdev.log` is generated under `/var/mqm/errors` directory path of the container.  This file will contain all the failed connection authentication requests.
+A log file named `mqhtpass.log` is generated under `/var/mqm/errors` directory path of the container.  This file will contain all the failed connection authentication requests.  Additional information is logged to this file if the environment variable `DEBUG` is set to `true`.
 
-**Please note**: This log file is based on circular logging and the maximum size is restricted to 1MB.
+**Please note**: This log file will be wiped when the queue manager is next started.

docs/security.md

@@ -16,20 +16,5 @@ docker run \
   --env LICENSE=accept \
   --env MQ_QMGR_NAME=QM1 \
   --detach \
-  ibm-mqadvanced-server:9.2.0.0-amd64
+  ibm-mqadvanced-server:9.2.5.0-amd64
-```
-
-The MQ Advanced for Developers image does require the "chown", "setuid", "setgid" and "audit_write" capabilities (plus "dac_override" if you're using an image based on Red Hat Enterprise Linux).  This is because it uses the "sudo" command to change passwords inside the container.  For example, in Docker, you could do the following:
-
-```sh
-docker run \
-  --cap-drop=ALL \
-  --cap-add=CHOWN \
-  --cap-add=SETUID \
-  --cap-add=SETGID \
-  --cap-add=AUDIT_WRITE \
-  --env LICENSE=accept \
-  --env MQ_QMGR_NAME=QM1 \
-  --detach \
-  ibm-mqadvanced-server-dev:9.2.0.0-amd64
 ```

docs/testing.md

@@ -14,17 +14,19 @@ There are two main sets of tests:
 2. Docker tests, which test a complete Docker image, using the Docker API
 
 ### Running the Docker tests
-The Docker tests can be run locally on a machine with Docker.  For example:
+
+The Docker tests can be run locally on a machine with Docker. For example:
 
 ```
-make test-devserver
+make devserver
-make test-advancedserver
+make advancedserver
 ```
 
 You can specify the image to use directly by using the `MQ_IMAGE_ADVANCEDSERVER` or `MQ_IMAGE_DEVSERVER` variables, for example:
 
 ```
-MQ_IMAGE_ADVANCEDSERVER=ibm-mqadvanced-server:9.2.0.0-amd64 make test-advancedserver
+MQ_IMAGE_ADVANCEDSERVER=ibm-mqadvanced-server:9.2.5.0-amd64 make test-advancedserver
+```
 
 You can pass parameters to `go test` with an environment variable.  For example, to run the "TestGoldenPath" test, run the following command:
 

etc/mqm/qm-service-component.ini

@@ -1,7 +1,7 @@
 ServiceComponent:
     Service=AuthorizationService
     Name=Dev.HtpAuth.Service
-    Module=/opt/mqm/lib64/amqpasdev.so
+    Module=/opt/mqm/lib64/mqhtpass.so
     ComponentDataSize=0
 ServiceComponent:
    Service=AuthorizationService

go.mod

@@ -0,0 +1,14 @@
+module github.com/ibm-messaging/mq-container
+
+go 1.15
+
+require (
+	github.com/genuinetools/amicontained v0.4.3
+	github.com/ibm-messaging/mq-golang v2.0.0+incompatible
+	github.com/prometheus/client_golang v1.11.1
+	github.com/prometheus/client_model v0.2.0
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f
+	golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40
+	software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001
+)

go.sum

@@ -0,0 +1,150 @@
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/genuinetools/amicontained v0.4.3 h1:cqq9XiAHfWWY3dk8VU8bSJFu9yh8Il5coEdeTAPq72o=
+github.com/genuinetools/amicontained v0.4.3/go.mod h1:PAMZkg9CcUTa6gNyULQ6tOMTMEb2HTKJufvKeFqDw+o=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/ibm-messaging/mq-golang v2.0.0+incompatible h1:xAufRPYSzoRGaME2+x7LcW5+uvy/G3xL/3Sn3u+G/lY=
+github.com/ibm-messaging/mq-golang v2.0.0+incompatible/go.mod h1:qjsZDb7m1oKnbPeDma2JVJTKgyCA91I4bcJ1qHY+gcA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001 h1:AVd6O+azYjVQYW1l55IqkbL8/JxjrLtO6q4FCmV8N5c=
+software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ=

ha/native-ha.ini.tpl

@@ -0,0 +1,18 @@
+NativeHALocalInstance:
+  Name={{ .Name }}
+  {{ if .CertificateLabel }}
+  CertificateLabel={{ .CertificateLabel }}
+  KeyRepository={{ .KeyRepository }}
+  {{ if .CipherSpec }}
+  CipherSpec={{ .CipherSpec }}
+  {{- end }}
+  {{- end }}
+NativeHAInstance:
+  Name={{ .NativeHAInstance0_Name }}
+  ReplicationAddress={{ .NativeHAInstance0_ReplicationAddress }}
+NativeHAInstance:
+  Name={{ .NativeHAInstance1_Name }}
+  ReplicationAddress={{ .NativeHAInstance1_ReplicationAddress }}
+NativeHAInstance:
+  Name={{ .NativeHAInstance2_Name }}
+  ReplicationAddress={{ .NativeHAInstance2_ReplicationAddress }}

incubating/mqadvanced-server-dev/install-extra-packages.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # -*- mode: sh -*-
-# © Copyright IBM Corporation 2019
+# © Copyright IBM Corporation 2019, 2021
 #
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,17 +25,17 @@ test -f /usr/bin/apt-get && UBUNTU=true || UBUNTU=false
 if ($UBUNTU); then
     export DEBIAN_FRONTEND=noninteractive
     apt-get update
-    apt-get install -y --no-install-recommends sudo
+    apt-get install -y --no-install-recommends libaprutil1
     rm -rf /var/lib/apt/lists/*
 fi
 
 if ($YUM); then
-    yum -y install sudo
+    yum -y install apr-util-openssl
     yum -y clean all
     rm -rf /var/cache/yum/*
 fi
 
 if ($MICRODNF); then
-    microdnf install sudo
+    microdnf --disableplugin=subscription-manager install apr-util-openssl
-    microdnf clean all
+    microdnf --disableplugin=subscription-manager clean all
 fi

incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml

@@ -36,5 +36,9 @@
     </basicRegistry>
     <variable name="httpHost" value="*"/>
     <variable name="managementMode" value="externallyprovisioned"/>
+    <variable name="mqConsoleRemoteSupportEnabled" value="false"/>
+    <variable name="mqConsoleEnableUnsafeInline" value="true"/>
+    <jndiEntry jndiName="mqConsoleDefaultCCDTHostname" value="${env.MQ_CONSOLE_DEFAULT_CCDT_HOSTNAME}"/>
+    <jndiEntry jndiName="mqConsoleDefaultCCDTPort" value="${env.MQ_CONSOLE_DEFAULT_CCDT_PORT}"/>
     <include location="tls.xml"/>
 </server>

install-build-deps.sh

@@ -21,6 +21,8 @@ set -ex
 
 sudo curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-$ARCH
 sudo chmod +x /usr/local/bin/dep
+sudo apt-get update || :
+sudo apt-get install -y jq
 
-go get -u golang.org/x/lint/golint
+go install golang.org/x/lint/golint@latest
 curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b $GOPATH/bin 2.0.0 || echo "Gosec not installed. Platform may not be supported."

install-mq-server-prereqs.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # -*- mode: sh -*-
-# © Copyright IBM Corporation 2015, 2020
+# © Copyright IBM Corporation 2015, 2021
 #
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -66,7 +66,7 @@ if ($RPM); then
   EXTRA_RPMS="bash bc ca-certificates file findutils gawk glibc-common grep ncurses-compat-libs passwd procps-ng sed shadow-utils tar util-linux which"
   # Install additional packages required by MQ, this install process and the runtime scripts
   $YUM && yum -y install --setopt install_weak_deps=false ${EXTRA_RPMS}
-  $MICRODNF && microdnf install ${EXTRA_RPMS}
+  $MICRODNF && microdnf --disableplugin=subscription-manager install ${EXTRA_RPMS}
 fi
 
 # Apply any bug fixes not included in base Ubuntu or MQ image.
@@ -78,4 +78,4 @@ $UBUNTU && apt-get install -y libapparmor1 libsystemd0 systemd systemd-sysv libu
 $UBUNTU && rm -rf /var/lib/apt/lists/*
 $YUM && yum -y clean all
 $YUM && rm -rf /var/cache/yum/*
-$MICRODNF && microdnf clean all
+$MICRODNF && microdnf --disableplugin=subscription-manager clean all

internal/ha/ha.go

@@ -0,0 +1,68 @@
+/*
+© Copyright IBM Corporation 2020, 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package ha contains code for high availability
+package ha
+
+import (
+	"os"
+
+	"github.com/ibm-messaging/mq-container/internal/mqtemplate"
+	"github.com/ibm-messaging/mq-container/internal/tls"
+	"github.com/ibm-messaging/mq-container/pkg/logger"
+)
+
+// ConfigureNativeHA configures native high availability
+func ConfigureNativeHA(log *logger.Logger) error {
+
+	file := "/etc/mqm/native-ha.ini"
+	templateFile := file + ".tpl"
+
+	templateMap := map[string]string{}
+	templateMap["Name"] = os.Getenv("HOSTNAME")
+	templateMap["NativeHAInstance0_Name"] = os.Getenv("MQ_NATIVE_HA_INSTANCE_0_NAME")
+	templateMap["NativeHAInstance1_Name"] = os.Getenv("MQ_NATIVE_HA_INSTANCE_1_NAME")
+	templateMap["NativeHAInstance2_Name"] = os.Getenv("MQ_NATIVE_HA_INSTANCE_2_NAME")
+	templateMap["NativeHAInstance0_ReplicationAddress"] = os.Getenv("MQ_NATIVE_HA_INSTANCE_0_REPLICATION_ADDRESS")
+	templateMap["NativeHAInstance1_ReplicationAddress"] = os.Getenv("MQ_NATIVE_HA_INSTANCE_1_REPLICATION_ADDRESS")
+	templateMap["NativeHAInstance2_ReplicationAddress"] = os.Getenv("MQ_NATIVE_HA_INSTANCE_2_REPLICATION_ADDRESS")
+
+	if os.Getenv("MQ_NATIVE_HA_TLS") == "true" {
+		keyLabel, _, _, err := tls.ConfigureHATLSKeystore()
+		if err != nil {
+			return err
+		}
+		templateMap["CertificateLabel"] = keyLabel
+
+		keyRepository, ok := os.LookupEnv("MQ_NATIVE_HA_KEY_REPOSITORY")
+		if !ok {
+			keyRepository = "/run/runmqserver/ha/tls/key"
+		}
+		templateMap["KeyRepository"] = keyRepository
+
+		cipherSpec, ok := os.LookupEnv("MQ_NATIVE_HA_CIPHERSPEC")
+		if ok {
+			templateMap["CipherSpec"] = cipherSpec
+		}
+	}
+
+	err := mqtemplate.ProcessTemplateFile(templateFile, file, templateMap, log)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/htpasswd/htpasswd.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2020
+© Copyright IBM Corporation 2020, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -110,44 +110,3 @@ func (htpfile mapHtPasswd) updateHtPasswordFile(isTest bool) error {
 	}
 	return ioutil.WriteFile(file, htpfile.GetBytes(), 0660)
 }
-
-// AuthenticateUser verifies if the given user password match with htpasswrd
-func AuthenticateUser(user string, password string, isTest bool) (bool, bool, error) {
-	passwords := mapHtPasswd(map[string]string{})
-
-	if len(strings.TrimSpace(user)) == 0 || len(strings.TrimSpace(password)) == 0 {
-		return false, false, fmt.Errorf("UserId or Password are empty")
-	}
-
-	err := passwords.ReadHtPasswordFile(isTest)
-	if err != nil {
-		return false, false, err
-	}
-
-	ok := false
-	value, found := passwords[user]
-
-	if !found {
-		return found, ok, fmt.Errorf("User not found in the mq.htpasswd file")
-	}
-
-	err = bcrypt.CompareHashAndPassword([]byte(value), []byte(password))
-	return found, err == nil, err
-}
-
-// ValidateUser validates the given user
-func ValidateUser(user string, isTest bool) (bool, error) {
-	passwords := mapHtPasswd(map[string]string{})
-
-	if len(strings.TrimSpace(user)) == 0 {
-		return false, fmt.Errorf("Userid is empty for AuthenticateUser")
-	}
-
-	err := passwords.ReadHtPasswordFile(isTest)
-	if err != nil {
-		return false, err
-	}
-
-	_, found := passwords[strings.TrimSpace(user)]
-	return found, nil
-}

internal/htpasswd/htpasswd_test.go

@@ -1,62 +0,0 @@
-/*
-© Copyright IBM Corporation 2020
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-package htpasswd
-
-import (
-	"testing"
-)
-
-// TestCheckUser verifies Htpassword's use
-func TestCheckUser(t *testing.T) {
-	err := SetPassword("guest", "guestpw", true)
-	if err != nil {
-		t.Fatalf("htpassword test failed due to error:%s\n", err.Error())
-	}
-	found, ok, err := AuthenticateUser("guest", "guestpw", true)
-	if err != nil {
-		t.Fatalf("htpassword test1 failed as user could not be found:%s\n", err.Error())
-	}
-	if found == false || ok == false {
-		t.Fatalf("htpassword test1 failed as user could not be found:%v, ok:%v\n", found, ok)
-	}
-
-	found, ok, err = AuthenticateUser("myguest", "guestpw", true)
-	if err == nil {
-		t.Fatalf("htpassword test2 failed as no error received for non-existing user\n")
-	}
-	if found == true || ok == true {
-		t.Fatalf("htpassword test2 failed for non-existing user found :%v, ok:%v\n", found, ok)
-	}
-
-	found, ok, err = AuthenticateUser("guest", "guest", true)
-	if err == nil {
-		t.Fatalf("htpassword test3 failed as incorrect password of user did not return error\n")
-	}
-
-	if found == false || ok == true {
-		t.Fatalf("htpassword test3 failed for existing user with incorrect passwored found :%v, ok:%v\n", found, ok)
-	}
-
-	found, err = ValidateUser("guest", true)
-	if err != nil || found == false {
-		t.Fatalf("htpassword test4 failed as user could not be found:%v, ok:%v\n", found, ok)
-	}
-
-	found, err = ValidateUser("myguest", true)
-	if err != nil || found == true {
-		t.Fatalf("htpassword test5 failed as non-existing user returned to be found:%v, ok:%v\n", found, ok)
-	}
-}

internal/htpasswd/my.htpasswd

@@ -1 +0,0 @@
-guest:$2y$05$ifFP0nCmFed6.m4iB9CHRuHFps2YeeuwopmOvszWt0GRnN59p8qxW

internal/metrics/metrics.go

@@ -23,9 +23,10 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/ibm-messaging/mq-container/pkg/logger"
 	"github.com/ibm-messaging/mq-container/internal/ready"
+	"github.com/ibm-messaging/mq-container/pkg/logger"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const (
@@ -83,7 +84,7 @@ func startMetricsGathering(qmName string, log *logger.Logger) error {
 	}
 
 	// Setup HTTP server to handle requests from Prometheus
-	http.Handle("/metrics", prometheus.Handler())
+	http.Handle("/metrics", promhttp.Handler())
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(200)
 		// #nosec G104

internal/mqversion/mqversion.go

@@ -0,0 +1,51 @@
+/*
+© Copyright IBM Corporation 2020
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mqversion
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ibm-messaging/mq-container/internal/command"
+)
+
+// Get will return the current MQ version
+func Get() (string, error) {
+	mqVersion, _, err := command.Run("dspmqver", "-b", "-f", "2")
+	if err != nil {
+		return "", fmt.Errorf("Error Getting MQ version: %v", err)
+	}
+	return strings.TrimSpace(mqVersion), nil
+}
+
+// Compare returns an integer comparing two MQ version strings lexicographically. The result will be 0 if currentVersion==checkVersion, -1 if currentVersion < checkVersion, and +1 if currentVersion > checkVersion
+func Compare(checkVersion string) (int, error) {
+	currentVersion, err := Get()
+	if err != nil {
+		return 0, err
+	}
+	// trim any suffix from MQ version x.x.x.x
+	currentVersion = currentVersion[0:7]
+	if currentVersion < checkVersion {
+		return -1, nil
+	} else if currentVersion == checkVersion {
+		return 0, nil
+	} else if currentVersion > checkVersion {
+		return 1, nil
+	}
+	return 0, fmt.Errorf("Failed to compare MQ versions")
+}

internal/mqversion/mqversion_test.go

@@ -0,0 +1,55 @@
+/*
+© Copyright IBM Corporation 2020
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mqversion
+
+import "testing"
+
+func TestCompareLower(t *testing.T) {
+	checkVersion := "9.9.9.9"
+	mqVersionCheck, err := Compare(checkVersion)
+	if err != nil {
+		t.Fatalf("Failed to compare MQ versions: %v", err)
+	}
+	if mqVersionCheck != -1 {
+		t.Errorf("MQ version compare result failed. Expected -1, Got %v", mqVersionCheck)
+	}
+}
+
+func TestCompareHigher(t *testing.T) {
+	checkVersion := "1.1.1.1"
+	mqVersionCheck, err := Compare(checkVersion)
+	if err != nil {
+		t.Fatalf("Failed to compare MQ versions: %v", err)
+	}
+	if mqVersionCheck != 1 {
+		t.Errorf("MQ version compare result failed. Expected 1, Got %v", mqVersionCheck)
+	}
+}
+
+func TestCompareEqual(t *testing.T) {
+	checkVersion, err := Get()
+	if err != nil {
+		t.Fatalf("Failed to get current MQ version: %v", err)
+	}
+	mqVersionCheck, err := Compare(checkVersion)
+	if err != nil {
+		t.Fatalf("Failed to compare MQ versions: %v", err)
+	}
+	if mqVersionCheck != 0 {
+		t.Errorf("MQ version compare result failed. Expected 0, Got %v", mqVersionCheck)
+	}
+}

internal/qmgrauth/pas.go

@@ -1,299 +0,0 @@
-/*
-© Copyright IBM Corporation 2020
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-//This is a developer only configuration and not recommended for production usage.
-package main
-
-/*
-#cgo !windows CFLAGS: -I/opt/mqm/lib64 -D_REENTRANT
-#cgo !windows,!darwin LDFLAGS: -L/opt/mqm/lib64 -lmqm_r -Wl,-rpath,/opt/mqm/lib64 -Wl,-rpath,/usr/lib64
-#cgo darwin   LDFLAGS:         -L/opt/mqm/lib64 -lmqm_r -Wl,-rpath,/opt/mqm/lib64 -Wl,-rpath,/usr/lib64
-#cgo  windows CFLAGS:  -I"C:/Program Files/IBM/MQ/Tools/c/include"
-#cgo windows LDFLAGS: -L "C:/Program Files/IBM/MQ/bin64" -lmqm
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <cmqc.h>
-#include <cmqxc.h>
-#include <cmqzc.h>
-#include <cmqec.h>
-#include <time.h>
-static MQZ_INIT_AUTHORITY           PASStart;
-static MQZ_AUTHENTICATE_USER        OAAuthUser;
-static MQZ_FREE_USER                OAFreeUser;
-static MQZ_TERM_AUTHORITY           OATermAuth;
-extern int Authenticate(char *, char *);
-extern int CheckAuthority(char *);
-static char *OAEnvStr(MQLONG);
-static void FindSize();
-static void PrintDateTime();
-static FILE *fp = NULL;
-static int primary_process = 0;
-
-static void MQENTRY PASStart(
-  MQHCONFIG hc,
-  MQLONG    Options,
-  MQCHAR48   QMgrName,
-  MQLONG    ComponentDataLength,
-  PMQBYTE   ComponentData,
-  PMQLONG   Version,
-  PMQLONG   pCompCode,
-  PMQLONG   pReason) {
-  MQLONG CC       = MQCC_OK;
-  MQLONG Reason   = MQRC_NONE;
-
-  if ((Options & MQZIO_PRIMARY) == MQZIO_PRIMARY)
-    primary_process = 1;
-
-  fp=fopen("/var/mqm/errors/amqpasdev.log","a");
-
-  if (CC == MQCC_OK)
-    hc->MQZEP_Call(hc, MQZID_INIT_AUTHORITY,(PMQFUNC)PASStart,&CC,&Reason);
-
-  if (CC == MQCC_OK)
-    hc->MQZEP_Call(hc,MQZID_TERM_AUTHORITY,(PMQFUNC)OATermAuth,&CC,&Reason);
-
-  if (CC == MQCC_OK)
-    hc->MQZEP_Call(hc,MQZID_AUTHENTICATE_USER,(PMQFUNC)OAAuthUser,&CC,&Reason);
-
-  if (CC == MQCC_OK)
-    hc->MQZEP_Call(hc,MQZID_FREE_USER,(PMQFUNC)OAFreeUser,&CC,&Reason);
-
-  *Version   = MQZAS_VERSION_5;
-  *pCompCode = CC;
-  *pReason   = Reason;
-
-  PrintDateTime();
-  fprintf(fp, "Pluggable OAM Initialized.\n");
-  fprintf(fp, "THIS IS A DEVELOPER ONLY CONFIGURATION AND NOT RECOMMENDED FOR PRODUCTION USAGE");
-  return;
-}
-
-static char *authuserfmt =
-  "\tUser    : \"%12.12s\"\n"\
-  "\tEffUser : \"%12.12s\"\n"\
-  "\tAppName : \"%28.28s\"\n"\
-  "\tApIdDt  : \"%32.32s\"\n"\
-  "\tEnv     : \"%s\"\n"\
-  "\tApp Pid : %d\n"\
-  "\tApp Tid : %d\n"\
-  ;
-static void MQENTRY OAAuthUser (
-     PMQCHAR  pQMgrName,
-     PMQCSP   pSecurityParms,
-     PMQZAC   pApplicationContext,
-     PMQZIC   pIdentityContext,
-     PMQPTR   pCorrelationPtr,
-     PMQBYTE  pComponentData,
-     PMQLONG  pContinuation,
-     PMQLONG  pCompCode,
-     PMQLONG  pReason)
-{
-  char *spuser = NULL;
-  char *sppass = NULL;
-  int gorc = MQRC_NOT_AUTHORIZED;
-
-  if ((pSecurityParms->CSPUserIdLength) > 0) {
-    //Grab the user creds from csp.
-    spuser = malloc(pSecurityParms->CSPUserIdLength+1);
-    strncpy(spuser,pSecurityParms->CSPUserIdPtr,pSecurityParms->CSPUserIdLength);
-    spuser[pSecurityParms->CSPUserIdLength]=0;
-    sppass =  malloc(pSecurityParms->CSPPasswordLength+1);
-    strncpy(sppass,pSecurityParms->CSPPasswordPtr,pSecurityParms->CSPPasswordLength);
-    sppass[pSecurityParms->CSPPasswordLength]=0;
-    gorc = Authenticate(spuser,sppass);
-
-    if (gorc == MQRC_NONE) {
-      *pCompCode = MQCC_OK;
-      *pReason = MQRC_NONE;
-      *pContinuation = MQZCI_CONTINUE;
-      memcpy( pIdentityContext->UserIdentifier
-        , spuser
-        , sizeof(pIdentityContext->UserIdentifier) );
-    } else {
-      *pCompCode = MQCC_WARNING;
-      *pReason = MQRC_NONE;
-      *pContinuation = MQZCI_CONTINUE;
-      //we print to error file only if error'd
-      PrintDateTime();
-      if (fp) {
-        fprintf(fp, authuserfmt,
-          pIdentityContext->UserIdentifier,
-          pApplicationContext->EffectiveUserID,
-          pApplicationContext->ApplName,
-          pIdentityContext->ApplIdentityData,
-          OAEnvStr(pApplicationContext->Environment),
-          pApplicationContext->ProcessId,
-          pApplicationContext->ThreadId);
-
-          fprintf(fp,"\tCSP UserId  : %s\n", spuser);
-          fprintf(fp,"\tCSP Password : %s\n", "****..");
-          fprintf(fp,"\tPAS-Compcode:%d\n",*pCompCode);
-          fprintf(fp,"\tPAS-Reasoncode:%d\n",*pReason);
-      }
-    }
-    free(spuser);
-    free(sppass);
-  } else {
-    //this is only a normal UID authentication.
-    spuser = malloc(sizeof(PMQCHAR12));
-    strncpy(spuser,pApplicationContext->EffectiveUserID,strlen(pApplicationContext->EffectiveUserID));
-    spuser[sizeof(PMQCHAR12)]=0;
-    gorc = CheckAuthority(spuser);
-    if (gorc == MQRC_NONE){
-        *pCompCode = MQCC_OK;
-        *pReason = MQRC_NONE;
-        *pContinuation = MQZCI_CONTINUE;
-        memcpy( pIdentityContext->UserIdentifier
-                , spuser
-                , sizeof(pIdentityContext->UserIdentifier) );
-    } else {
-        *pCompCode = MQCC_WARNING;
-        *pReason = MQRC_NONE;
-        *pContinuation = MQZCI_CONTINUE;
-        //we print only if error'd
-        PrintDateTime();
-        if (fp)
-        {
-          fprintf(fp, authuserfmt,
-	        pIdentityContext->UserIdentifier,
-	        pApplicationContext->EffectiveUserID,
-	        pApplicationContext->ApplName,
-	        pIdentityContext->ApplIdentityData,
-	        OAEnvStr(pApplicationContext->Environment),
-	        pApplicationContext->ProcessId,
-	        pApplicationContext->ThreadId
-	        );
-          fprintf(fp,"\tUID  : %s\n", spuser);
-          fprintf(fp,"\tPAS-Compcode:%d\n",*pCompCode);
-          fprintf(fp,"\tPAS-Reasoncode:%d\n",*pReason);
-        }
-    }
-  }
-  return;
-}
-
-static void MQENTRY OAFreeUser (
-     PMQCHAR  pQMgrName,
-     PMQZFP   pFreeParms,
-     PMQBYTE  pComponentData,
-     PMQLONG  pContinuation,
-
-     PMQLONG  pCompCode,
-     PMQLONG  pReason)
-{
-  *pCompCode = MQCC_WARNING;
-  *pReason   = MQRC_NONE;
-  *pContinuation = MQZCI_CONTINUE;
-  return;
-}
-
-static void MQENTRY OATermAuth(
-  MQHCONFIG  hc,
-  MQLONG     Options,
-  PMQCHAR    pQMgrName,
-  PMQBYTE    pComponentData,
-  PMQLONG    pCompCode,
-  PMQLONG    pReason)
-{
-  if ((primary_process) && ((Options & MQZTO_PRIMARY) == MQZTO_PRIMARY) ||
-	                   ((Options & MQZTO_SECONDARY) == MQZTO_SECONDARY))
-  {
-    if (fp)
-    {
-      fclose(fp);
-      fp = NULL;
-    }
-  }
-  *pCompCode = MQCC_OK;
-  *pReason   = MQRC_NONE;
-}
-
-static void PrintDateTime() {
-  FindSize();
-  struct tm *local;
-    time_t t;
-    t = time(NULL);
-    local = localtime(&t);
-    if (fp) {
-      fprintf(fp, "-------------------------------------------------\n");
-      fprintf(fp, "Local time: %s", asctime(local));
-      local = gmtime(&t);
-      fprintf(fp, "UTC time: %s", asctime(local));
-    }
-    return;
-}
-
-static char *OAEnvStr(MQLONG x)
- {
-   switch (x)
-   {
-   case MQXE_OTHER:          return "Application";
-   case MQXE_MCA:            return "Channel";
-   case MQXE_MCA_SVRCONN:    return "Channel SvrConn";
-   case MQXE_COMMAND_SERVER: return "Command Server";
-   case MQXE_MQSC:           return "MQSC";
-   default:                  return "Invalid Environment";
-   }
- }
-
-static void FindSize()
-{
-  int sz = 0;
-  int prev=ftell(fp);
-  fseek(fp, 0L, SEEK_END);
-  sz=ftell(fp);
-  //if log file size goes over 1mb, rewind it.
-  if (sz > 1000000) {
-    rewind(fp);
-  } else {
-    fseek(fp, prev, SEEK_SET);
-  }
-}
-
-*/
-import "C"
-import "github.com/ibm-messaging/mq-container/internal/htpasswd"
-
-//export MQStart
-func MQStart(hc C.MQHCONFIG, Options C.MQLONG, QMgrName C.PMQCHAR, ComponentDataLength C.MQLONG, ComponentData C.PMQBYTE, Version C.PMQLONG, pCompCode C.PMQLONG, pReason C.PMQLONG) {
-	C.PASStart(hc, Options, QMgrName, ComponentDataLength, ComponentData, Version, pCompCode, pReason)
-}
-
-//export Authenticate
-func Authenticate(x *C.char, y *C.char) C.int {
-	user := C.GoString(x)
-	pwd := C.GoString(y)
-	found, ok, err := htpasswd.AuthenticateUser(user, pwd, false)
-
-	if !found || !ok || err != nil {
-		return C.MQRC_UNKNOWN_OBJECT_NAME
-	}
-	return C.MQRC_NONE
-}
-
-//export CheckAuthority
-func CheckAuthority(x *C.char) C.int {
-	user := C.GoString(x)
-	found, err := htpasswd.ValidateUser(user, false)
-	if !found || err != nil {
-		return C.MQRC_UNKNOWN_OBJECT_NAME
-	}
-	return C.MQRC_NONE
-
-}
-
-func main() {}

internal/ready/ready.go

@@ -76,6 +76,11 @@ func IsRunningAsStandbyQM(name string) (bool, error) {
 	return isRunningQM(name, "(RUNNING AS STANDBY)")
 }
 
+// IsRunningAsReplicaQM returns true if the queue manager is running in replica mode
+func IsRunningAsReplicaQM(name string) (bool, error) {
+	return isRunningQM(name, "(REPLICA)")
+}
+
 func isRunningQM(name string, status string) (bool, error) {
 	out, _, err := command.Run("dspmq", "-n", "-m", name)
 	if err != nil {

internal/tls/tls.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2019
+© Copyright IBM Corporation 2019, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -43,14 +43,20 @@ const cmsKeystoreName = "key.kdb"
 // p12TruststoreName is the name of the PKCS#12 Truststore
 const p12TruststoreName = "trust.p12"
 
-// keystoreDir is the location for the CMS Keystore & PKCS#12 Truststore
+// keystoreDirDefault is the location for the default CMS Keystore & PKCS#12 Truststore
-const keystoreDir = "/run/runmqserver/tls/"
+const keystoreDirDefault = "/run/runmqserver/tls/"
 
-// keyDir is the location of the keys to import
+// keystoreDirHA is the location for the HA CMS Keystore
-const keyDir = "/etc/mqm/pki/keys"
+const keystoreDirHA = "/run/runmqserver/ha/tls/"
 
-// trustDir is the location of the trust certificates to import
+// keyDirDefault is the location of the default keys to import
-const trustDir = "/etc/mqm/pki/trust"
+const keyDirDefault = "/etc/mqm/pki/keys"
+
+// keyDirHA is the location of the HA keys to import
+const keyDirHA = "/etc/mqm/ha/pki/keys"
+
+// trustDirDefault is the location of the trust certificates to import
+const trustDirDefault = "/etc/mqm/pki/trust"
 
 type KeyStoreData struct {
 	Keystore          *keystore.KeyStore
@@ -65,28 +71,43 @@ type P12KeyFiles struct {
 	Password  string
 }
 
-// ConfigureTLSKeystores configures the CMS Keystore & PKCS#12 Truststore
+type TLSStore struct {
-func ConfigureTLSKeystores() (string, KeyStoreData, KeyStoreData, error) {
+	Keystore   KeyStoreData
+	Truststore KeyStoreData
+}
 
-	// Create the CMS Keystore & PKCS#12 Truststore
+func configureTLSKeystores(keystoreDir, keyDir, trustDir string, p12TruststoreRequired bool) (string, KeyStoreData, KeyStoreData, error) {
-	cmsKeystore, p12Truststore, err := generateAllKeystores()
+
+	// Create the CMS Keystore & PKCS#12 Truststore (if required)
+	tlsStore, err := generateAllKeystores(keystoreDir, p12TruststoreRequired)
 	if err != nil {
-		return "", cmsKeystore, p12Truststore, err
+		return "", tlsStore.Keystore, tlsStore.Truststore, err
 	}
 
 	// Process all keys - add them to the CMS KeyStore
-	keyLabel, err := processKeys(&cmsKeystore, &p12Truststore)
+	keyLabel, err := processKeys(&tlsStore, keystoreDir, keyDir)
 	if err != nil {
-		return "", cmsKeystore, p12Truststore, err
+		return "", tlsStore.Keystore, tlsStore.Truststore, err
 	}
 
-	// Process all trust certificates - add them to the CMS KeyStore & PKCS#12 Truststore
+	// Process all trust certificates - add them to the CMS KeyStore & PKCS#12 Truststore (if required)
-	err = processTrustCertificates(&cmsKeystore, &p12Truststore)
+	err = processTrustCertificates(&tlsStore, trustDir)
 	if err != nil {
-		return "", cmsKeystore, p12Truststore, err
+		return "", tlsStore.Keystore, tlsStore.Truststore, err
 	}
 
-	return keyLabel, cmsKeystore, p12Truststore, err
+	return keyLabel, tlsStore.Keystore, tlsStore.Truststore, err
+}
+
+// ConfigureDefaultTLSKeystores configures the CMS Keystore & PKCS#12 Truststore
+func ConfigureDefaultTLSKeystores() (string, KeyStoreData, KeyStoreData, error) {
+	return configureTLSKeystores(keystoreDirDefault, keyDirDefault, trustDirDefault, true)
+}
+
+// ConfigureHATLSKeystore configures the CMS Keystore & PKCS#12 Truststore
+func ConfigureHATLSKeystore() (string, KeyStoreData, KeyStoreData, error) {
+	// *.crt files mounted to the HA TLS dir keyDirHA will be processed as trusted in the CMS keystore
+	return configureTLSKeystores(keystoreDirHA, keyDirHA, keyDirHA, false)
 }
 
 // ConfigureTLS configures TLS for the queue manager
@@ -137,8 +158,9 @@ func configureTLSDev(log *logger.Logger) error {
 	return nil
 }
 
-// generateAllKeystores creates the CMS Keystore & PKCS#12 Truststore
+// generateAllKeystores creates the CMS Keystore & PKCS#12 Truststore (if required)
-func generateAllKeystores() (KeyStoreData, KeyStoreData, error) {
+func generateAllKeystores(keystoreDir string, p12TruststoreRequired bool) (TLSStore, error) {
+
 	var cmsKeystore, p12Truststore KeyStoreData
 
 	// Generate a pasword for use with both the CMS Keystore & PKCS#12 Truststore
@@ -150,28 +172,30 @@ func generateAllKeystores() (KeyStoreData, KeyStoreData, error) {
 	// #nosec G301 - write group permissions are required
 	err := os.MkdirAll(keystoreDir, 0770)
 	if err != nil {
-		return cmsKeystore, p12Truststore, fmt.Errorf("Failed to create Keystore directory: %v", err)
+		return TLSStore{cmsKeystore, p12Truststore}, fmt.Errorf("Failed to create Keystore directory: %v", err)
 	}
 
 	// Create the CMS Keystore
 	cmsKeystore.Keystore = keystore.NewCMSKeyStore(filepath.Join(keystoreDir, cmsKeystoreName), cmsKeystore.Password)
 	err = cmsKeystore.Keystore.Create()
 	if err != nil {
-		return cmsKeystore, p12Truststore, fmt.Errorf("Failed to create CMS Keystore: %v", err)
+		return TLSStore{cmsKeystore, p12Truststore}, fmt.Errorf("Failed to create CMS Keystore: %v", err)
 	}
 
-	// Create the PKCS#12 Truststore
+	// Create the PKCS#12 Truststore (if required)
-	p12Truststore.Keystore = keystore.NewPKCS12KeyStore(filepath.Join(keystoreDir, p12TruststoreName), p12Truststore.Password)
+	if p12TruststoreRequired {
-	err = p12Truststore.Keystore.Create()
+		p12Truststore.Keystore = keystore.NewPKCS12KeyStore(filepath.Join(keystoreDir, p12TruststoreName), p12Truststore.Password)
-	if err != nil {
+		err = p12Truststore.Keystore.Create()
-		return cmsKeystore, p12Truststore, fmt.Errorf("Failed to create PKCS#12 Truststore: %v", err)
+		if err != nil {
+			return TLSStore{cmsKeystore, p12Truststore}, fmt.Errorf("Failed to create PKCS#12 Truststore: %v", err)
+		}
 	}
 
-	return cmsKeystore, p12Truststore, nil
+	return TLSStore{cmsKeystore, p12Truststore}, nil
 }
 
 // processKeys processes all keys - adding them to the CMS KeyStore
-func processKeys(cmsKeystore, p12Truststore *KeyStoreData) (string, error) {
+func processKeys(tlsStore *TLSStore, keystoreDir string, keyDir string) (string, error) {
 
 	// Key label - will be set to the label of the first set of keys
 	keyLabel := ""
@@ -190,7 +214,7 @@ func processKeys(cmsKeystore, p12Truststore *KeyStoreData) (string, error) {
 			}
 
 			// Process private key (*.key)
-			privateKey, keyPrefix, err := processPrivateKey(keySet.Name(), keys)
+			privateKey, keyPrefix, err := processPrivateKey(keyDir, keySet.Name(), keys)
 			if err != nil {
 				return "", err
 			}
@@ -201,13 +225,13 @@ func processKeys(cmsKeystore, p12Truststore *KeyStoreData) (string, error) {
 			}
 
 			// Process certificates (*.crt) - public certificate & optional CA certificate
-			publicCertificate, caCertificate, err := processCertificates(keySet.Name(), keyPrefix, keys, cmsKeystore, p12Truststore)
+			publicCertificate, caCertificate, err := processCertificates(keyDir, keySet.Name(), keyPrefix, keys, &tlsStore.Keystore, &tlsStore.Truststore)
 			if err != nil {
 				return "", err
 			}
 
 			// Create a new PKCS#12 Keystore - containing private key, public certificate & optional CA certificate
-			file, err := pkcs.Encode(rand.Reader, privateKey, publicCertificate, caCertificate, cmsKeystore.Password)
+			file, err := pkcs.Encode(rand.Reader, privateKey, publicCertificate, caCertificate, tlsStore.Keystore.Password)
 			if err != nil {
 				return "", fmt.Errorf("Failed to encode PKCS#12 Keystore %s: %v", keySet.Name()+".p12", err)
 			}
@@ -217,13 +241,13 @@ func processKeys(cmsKeystore, p12Truststore *KeyStoreData) (string, error) {
 			}
 
 			// Import the new PKCS#12 Keystore into the CMS Keystore
-			err = cmsKeystore.Keystore.Import(filepath.Join(keystoreDir, keySet.Name()+".p12"), cmsKeystore.Password)
+			err = tlsStore.Keystore.Keystore.Import(filepath.Join(keystoreDir, keySet.Name()+".p12"), tlsStore.Keystore.Password)
 			if err != nil {
-				return "", fmt.Errorf("Failed tp import keys from %s into CMS Keystore: %v", filepath.Join(keystoreDir, keySet.Name()+".p12"), err)
+				return "", fmt.Errorf("Failed to import keys from %s into CMS Keystore: %v", filepath.Join(keystoreDir, keySet.Name()+".p12"), err)
 			}
 
 			// Relabel the certificate in the CMS Keystore
-			err = relabelCertificate(keySet.Name(), cmsKeystore)
+			err = relabelCertificate(keySet.Name(), &tlsStore.Keystore)
 			if err != nil {
 				return "", err
 			}
@@ -238,8 +262,8 @@ func processKeys(cmsKeystore, p12Truststore *KeyStoreData) (string, error) {
 	return keyLabel, nil
 }
 
-// processTrustCertificates processes all trust certificates - adding them to the CMS KeyStore & PKCS#12 Truststore
+// processTrustCertificates processes all trust certificates - adding them to the CMS KeyStore & PKCS#12 Truststore (if required)
-func processTrustCertificates(cmsKeystore, p12Truststore *KeyStoreData) error {
+func processTrustCertificates(tlsStore *TLSStore, trustDir string) error {
 
 	// Process all trust certiifcates
 	trustList, err := ioutil.ReadDir(trustDir)
@@ -265,15 +289,17 @@ func processTrustCertificates(cmsKeystore, p12Truststore *KeyStoreData) error {
 						}
 
 						// Add to known certificates for the CMS Keystore
-						err = addToKnownCertificates(block, cmsKeystore, true)
+						err = addToKnownCertificates(block, &tlsStore.Keystore, true)
 						if err != nil {
 							return fmt.Errorf("Failed to add to know certificates for CMS Keystore")
 						}
 
-						// Add to known certificates for the PKCS#12 Truststore
+						if tlsStore.Truststore.Keystore != nil {
-						err = addToKnownCertificates(block, p12Truststore, true)
+							// Add to known certificates for the PKCS#12 Truststore
-						if err != nil {
+							err = addToKnownCertificates(block, &tlsStore.Truststore, true)
-							return fmt.Errorf("Failed to add to know certificates for PKCS#12 Truststore")
+							if err != nil {
+								return fmt.Errorf("Failed to add to know certificates for PKCS#12 Truststore")
+							}
 						}
 					}
 				}
@@ -281,17 +307,17 @@ func processTrustCertificates(cmsKeystore, p12Truststore *KeyStoreData) error {
 		}
 	}
 
-	// Add all trust certificates to PKCS#12 Truststore
+	// Add all trust certificates to PKCS#12 Truststore (if required)
-	if len(p12Truststore.TrustedCerts) > 0 {
+	if tlsStore.Truststore.Keystore != nil && len(tlsStore.Truststore.TrustedCerts) > 0 {
-		err = addCertificatesToTruststore(p12Truststore)
+		err = addCertificatesToTruststore(&tlsStore.Truststore)
 		if err != nil {
 			return err
 		}
 	}
 
 	// Add all trust certificates to CMS Keystore
-	if len(cmsKeystore.TrustedCerts) > 0 {
+	if len(tlsStore.Keystore.TrustedCerts) > 0 {
-		err = addCertificatesToCMSKeystore(cmsKeystore)
+		err = addCertificatesToCMSKeystore(&tlsStore.Keystore)
 		if err != nil {
 			return err
 		}
@@ -301,7 +327,7 @@ func processTrustCertificates(cmsKeystore, p12Truststore *KeyStoreData) error {
 }
 
 // processPrivateKey processes the private key (*.key) from a set of keys
-func processPrivateKey(keySetName string, keys []os.FileInfo) (interface{}, string, error) {
+func processPrivateKey(keyDir string, keySetName string, keys []os.FileInfo) (interface{}, string, error) {
 
 	var privateKey interface{}
 	keyPrefix := ""
@@ -336,7 +362,7 @@ func processPrivateKey(keySetName string, keys []os.FileInfo) (interface{}, stri
 }
 
 // processCertificates processes the certificates (*.crt) from a set of keys
-func processCertificates(keySetName, keyPrefix string, keys []os.FileInfo, cmsKeystore, p12Truststore *KeyStoreData) (*x509.Certificate, []*x509.Certificate, error) {
+func processCertificates(keyDir string, keySetName, keyPrefix string, keys []os.FileInfo, cmsKeystore, p12Truststore *KeyStoreData) (*x509.Certificate, []*x509.Certificate, error) {
 
 	var publicCertificate *x509.Certificate
 	var caCertificate []*x509.Certificate
@@ -384,10 +410,12 @@ func processCertificates(keySetName, keyPrefix string, keys []os.FileInfo, cmsKe
 					return nil, nil, fmt.Errorf("Failed to add to know certificates for CMS Keystore")
 				}
 
-				// Add to known certificates for the PKCS#12 Truststore
+				if p12Truststore.Keystore != nil {
-				err = addToKnownCertificates(block, p12Truststore, true)
+					// Add to known certificates for the PKCS#12 Truststore
-				if err != nil {
+					err = addToKnownCertificates(block, p12Truststore, true)
-					return nil, nil, fmt.Errorf("Failed to add to know certificates for PKCS#12 Truststore")
+					if err != nil {
+						return nil, nil, fmt.Errorf("Failed to add to know certificates for PKCS#12 Truststore")
+					}
 				}
 
 				certificate, err := x509.ParseCertificate(block.Bytes)

internal/tls/tls_web.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2019, 2020
+© Copyright IBM Corporation 2019, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -60,7 +60,7 @@ func ConfigureWebKeystore(p12Truststore KeyStoreData, webKeystore string) (strin
 	if webKeystore == "" {
 		webKeystore = webKeystoreDefault
 	}
-	webKeystoreFile := filepath.Join(keystoreDir, webKeystore)
+	webKeystoreFile := filepath.Join(keystoreDirDefault, webKeystore)
 
 	// Check if a new self-signed certificate should be generated
 	genHostName := os.Getenv("MQ_GENERATE_CERTIFICATE_HOSTNAME")

pkg/logger/logger.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2018, 2019
+© Copyright IBM Corporation 2018, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -117,14 +117,22 @@ func (l *Logger) log(level string, msg string) {
 // Debug logs a line as debug
 func (l *Logger) Debug(args ...interface{}) {
 	if l.debug {
-		l.log(debugLevel, fmt.Sprint(args...))
+		if l.json {
+			l.log(debugLevel, fmt.Sprint(args...))
+		} else {
+			l.log(debugLevel, "DEBUG: "+fmt.Sprint(args...))
+		}
 	}
 }
 
 // Debugf logs a line as debug using format specifiers
 func (l *Logger) Debugf(format string, args ...interface{}) {
 	if l.debug {
-		l.log(debugLevel, fmt.Sprintf(format, args...))
+		if l.json {
+			l.log(debugLevel, fmt.Sprintf(format, args...))
+		} else {
+			l.log(debugLevel, fmt.Sprintf("DEBUG: "+format, args...))
+		}
 	}
 }
 

test/docker/Gopkg.lock

@@ -1,114 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  name = "github.com/Azure/go-ansiterm"
-  packages = [
-    ".",
-    "winterm"
-  ]
-  revision = "d6e3b3328b783f23731bc4d058875b0371ff8109"
-
-[[projects]]
-  name = "github.com/Microsoft/go-winio"
-  packages = ["."]
-  revision = "7da180ee92d8bd8bb8c37fc560e673e6557c392f"
-  version = "v0.4.7"
-
-[[projects]]
-  name = "github.com/Sirupsen/logrus"
-  packages = ["."]
-  revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
-  version = "v1.0.5"
-
-[[projects]]
-  name = "github.com/docker/distribution"
-  packages = [
-    "digest",
-    "reference"
-  ]
-  revision = "48294d928ced5dd9b378f7fd7c6f5da3ff3f2c89"
-  version = "v2.6.2"
-
-[[projects]]
-  name = "github.com/docker/docker"
-  packages = [
-    "api/types",
-    "api/types/blkiodev",
-    "api/types/container",
-    "api/types/events",
-    "api/types/filters",
-    "api/types/mount",
-    "api/types/network",
-    "api/types/reference",
-    "api/types/registry",
-    "api/types/strslice",
-    "api/types/swarm",
-    "api/types/time",
-    "api/types/versions",
-    "api/types/volume",
-    "client",
-    "pkg/jsonlog",
-    "pkg/jsonmessage",
-    "pkg/stdcopy",
-    "pkg/term",
-    "pkg/term/windows",
-    "pkg/tlsconfig"
-  ]
-  revision = "f5ec1e2936dcbe7b5001c2b817188b095c700c27"
-  version = "v17.03.2-ce"
-
-[[projects]]
-  name = "github.com/docker/go-connections"
-  packages = [
-    "nat",
-    "sockets",
-    "tlsconfig"
-  ]
-  revision = "3ede32e2033de7505e6500d6c868c2b9ed9f169d"
-  version = "v0.3.0"
-
-[[projects]]
-  name = "github.com/docker/go-units"
-  packages = ["."]
-  revision = "0dadbb0345b35ec7ef35e228dabb8de89a65bf52"
-  version = "v0.3.2"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-  packages = ["ssh/terminal"]
-  revision = "88942b9c40a4c9d203b82b3731787b672d6e809b"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "proxy"
-  ]
-  revision = "6078986fec03a1dcc236c34816c71b0e05018fda"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = [
-    "unix",
-    "windows"
-  ]
-  revision = "13d03a9a82fba647c21a0ef8fba44a795d0f0835"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "c792836365447209421d5dc68a75fa77063408b8a6a2f9325b976581a0d60107"
-  solver-name = "gps-cdcl"
-  solver-version = 1

test/docker/devconfig_test.go

@@ -1,7 +1,7 @@
 // +build mqdev
 
 /*
-© Copyright IBM Corporation 2018, 2019
+© Copyright IBM Corporation 2018, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -43,6 +43,7 @@ func TestDevGoldenPath(t *testing.T) {
 		Env: []string{
 			"LICENSE=accept",
 			"MQ_QMGR_NAME=" + qm,
+			"DEBUG=true",
 		},
 	}
 	id := runContainerWithPorts(t, cli, &containerConfig, []int{9443})
@@ -82,6 +83,8 @@ func TestDevSecure(t *testing.T) {
 			"MQ_QMGR_NAME=" + qm,
 			"MQ_APP_PASSWORD=" + appPassword,
 			"DEBUG=1",
+			"WLP_LOGGING_MESSAGE_FORMAT=JSON",
+			"MQ_ENABLE_EMBEDDED_WEB_SERVER_LOG=true",
 		},
 		Image: imageName(),
 	}

test/docker/devconfig_test_util.go

@@ -1,7 +1,7 @@
 // +build mqdev
 
 /*
-© Copyright IBM Corporation 2018, 2019
+© Copyright IBM Corporation 2018, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -48,8 +48,9 @@ var insecureTLSConfig *tls.Config = &tls.Config{
 }
 
 func waitForWebReady(t *testing.T, cli *client.Client, ID string, tlsConfig *tls.Config) {
+	t.Logf("%s Waiting for web server to be ready", time.Now().Format(time.RFC3339))
 	httpClient := http.Client{
-		Timeout: time.Duration(3 * time.Second),
+		Timeout: time.Duration(10 * time.Second),
 		Transport: &http.Transport{
 			TLSClientConfig: tlsConfig,
 		},
@@ -63,13 +64,13 @@ func waitForWebReady(t *testing.T, cli *client.Client, ID string, tlsConfig *tls
 		case <-time.After(1 * time.Second):
 			req, err := http.NewRequest("GET", url, nil)
 			req.SetBasicAuth("admin", defaultAdminPassword)
-			resp, err := httpClient.Do(req.WithContext(ctx))
+			resp, err := httpClient.Do(req)
 			if err == nil && resp.StatusCode == http.StatusOK {
-				t.Log("MQ web server is ready")
+				t.Logf("%s MQ web server is ready", time.Now().Format(time.RFC3339))
 				return
 			}
 		case <-ctx.Done():
-			t.Fatal("Timed out waiting for web server to become ready")
+			t.Fatalf("%s Timed out waiting for web server to become ready", time.Now().Format(time.RFC3339))
 		}
 	}
 }

test/docker/docker_api_test.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2017, 2019
+© Copyright IBM Corporation 2017, 2020
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -991,7 +991,7 @@ func TestMQIniMergeOnTheSameVolumeButTwoContainers(t *testing.T) {
 		RUN chmod 0660 /etc/mqm/test1.ini
 		USER 1001`, imageName())},
 		{"test1.ini",
-			"Log:\n   LogFilePages=5000"},
+			"Log:\n   LogBufferPages=128"},
 	}
 
 	secondImage := createImage(t, cli, filesSecondContainer)
@@ -1011,7 +1011,7 @@ func TestMQIniMergeOnTheSameVolumeButTwoContainers(t *testing.T) {
 	waitForReady(t, cli, ctr2.ID)
 
 	_, test2 := execContainer(t, cli, ctr2.ID, "", []string{"bash", "-c", catIniFileCommand})
-	changedStanza := strings.Contains(test2, "LogFilePages=5000")
+	changedStanza := strings.Contains(test2, "LogBufferPages=128")
 	//check if stanza that was merged in the first container doesnt exist in this one.
 	firstMergedStanza := strings.Contains(test2, "ApplicationTrace:\n   ApplName=amqsact*\n   Trace=OFF")
 
@@ -1087,6 +1087,7 @@ func TestReadiness(t *testing.T) {
 }
 
 func TestErrorLogRotation(t *testing.T) {
+	t.Skipf("Skipping %v until test defect fixed", t.Name())
 	t.Parallel()
 
 	cli, err := client.NewEnvClient()
@@ -1433,3 +1434,85 @@ func TestTraceStrmqm(t *testing.T) {
 		t.Fatalf("No trace files found in trace directory /var/mqm/trace. RC=%d.", rc)
 	}
 }
+
+// utilTestHealthCheck is used by TestHealthCheck* to run a container with
+// privileges enabled or disabled.  Otherwise the same as the golden path tests.
+func utilTestHealthCheck(t *testing.T, nonewpriv bool) {
+	t.Parallel()
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"},
+	}
+	hostConfig := getDefaultHostConfig(t, cli)
+	hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, fmt.Sprintf("no-new-privileges:%v", nonewpriv))
+	id := runContainerWithHostConfig(t, cli, &containerConfig, hostConfig)
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+	rc, out := execContainer(t, cli, id, "", []string{"chkmqhealthy"})
+	t.Log(out)
+	if rc != 0 {
+		t.Errorf("Expected chkmqhealthy to return with exit code 0; got \"%v\"", rc)
+		t.Logf("Output from chkmqhealthy:\n%v", out)
+	}
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+}
+
+// TestHealthCheckWithNoNewPrivileges tests golden path start/stop plus
+// chkmqhealthy, when running in a container where no new privileges are
+// allowed (i.e. setuid is disabled)
+func TestHealthCheckWithNoNewPrivileges(t *testing.T) {
+	utilTestHealthCheck(t, true)
+}
+
+// TestHealthCheckWithNoNewPrivileges tests golden path start/stop plus
+// chkmqhealthy when running in a container where new privileges are
+// allowed (i.e. setuid is allowed)
+// See https://github.com/ibm-messaging/mq-container/issues/428
+func TestHealthCheckWithNewPrivileges(t *testing.T) {
+	utilTestHealthCheck(t, false)
+}
+
+// utilTestStartedCheck is used by TestStartedCheck* to run a container with
+// privileges enabled or disabled.  Otherwise the same as the golden path tests.
+func utilTestStartedCheck(t *testing.T, nonewpriv bool) {
+	t.Parallel()
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"},
+	}
+	hostConfig := getDefaultHostConfig(t, cli)
+	hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, fmt.Sprintf("no-new-privileges:%v", nonewpriv))
+	id := runContainerWithHostConfig(t, cli, &containerConfig, hostConfig)
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+	rc, out := execContainer(t, cli, id, "", []string{"chkmqstarted"})
+	t.Log(out)
+	if rc != 0 {
+		t.Errorf("Expected chkmqstarted to return with exit code 0; got \"%v\"", rc)
+		t.Logf("Output from chkmqstarted:\n%v", out)
+	}
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+}
+
+// TestStartedCheckWithNoNewPrivileges tests golden path start/stop plus
+// chkmqstarted, when running in a container where no new privileges are
+// allowed (i.e. setuid is disabled)
+func TestStartedCheckWithNoNewPrivileges(t *testing.T) {
+	utilTestStartedCheck(t, true)
+}
+
+// TestStartedCheckWithNoNewPrivileges tests golden path start/stop plus
+// chkmqstarted when running in a container where new privileges are
+// allowed (i.e. setuid is allowed)
+// See https://github.com/ibm-messaging/mq-container/issues/428
+func TestStartedCheckWithNewPrivileges(t *testing.T) {
+	utilTestStartedCheck(t, false)
+}

test/docker/docker_api_test_util.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2017, 2020
+© Copyright IBM Corporation 2017, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -267,20 +267,8 @@ func generateRandomUID() string {
 	return fmt.Sprint(rand.Intn(max-min) + min)
 }
 
-// runContainerWithPorts creates and starts a container, exposing the specified ports on the host.
+// getDefaultHostConfig creates a HostConfig and populates it with the defaults used in testing
-// If no image is specified in the container config, then the image name is retrieved from the TEST_IMAGE
+func getDefaultHostConfig(t *testing.T, cli *client.Client) *container.HostConfig {
-// environment variable.
-func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *container.Config, ports []int) string {
-	if containerConfig.Image == "" {
-		containerConfig.Image = imageName()
-	}
-	// Always run as a random user, unless the test has specified otherwise
-	if containerConfig.User == "" {
-		containerConfig.User = generateRandomUID()
-	}
-	// if coverage
-	containerConfig.Env = append(containerConfig.Env, "COVERAGE_FILE="+t.Name()+".cov")
-	containerConfig.Env = append(containerConfig.Env, "EXIT_CODE_FILE="+getExitCodeFilename(t))
 	hostConfig := container.HostConfig{
 		Binds: []string{
 			coverageBind(t),
@@ -299,6 +287,62 @@ func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *co
 	} else {
 		t.Logf("Detected MQ Advanced image - dropping all capabilities")
 	}
+	return &hostConfig
+}
+
+// runContainerWithHostConfig creates and starts a container, using the supplied HostConfig.
+// Note that a default HostConfig can be created using getDefaultHostConfig.
+func runContainerWithHostConfig(t *testing.T, cli *client.Client, containerConfig *container.Config, hostConfig *container.HostConfig) string {
+	if containerConfig.Image == "" {
+		containerConfig.Image = imageName()
+	}
+	// Always run as a random user, unless the test has specified otherwise
+	if containerConfig.User == "" {
+		containerConfig.User = generateRandomUID()
+	}
+	// if coverage
+	containerConfig.Env = append(containerConfig.Env, "COVERAGE_FILE="+t.Name()+".cov")
+	containerConfig.Env = append(containerConfig.Env, "EXIT_CODE_FILE="+getExitCodeFilename(t))
+	networkingConfig := network.NetworkingConfig{}
+	t.Logf("Running container (%s)", containerConfig.Image)
+	ctr, err := cli.ContainerCreate(context.Background(), containerConfig, hostConfig, &networkingConfig, t.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	startContainer(t, cli, ctr.ID)
+	return ctr.ID
+}
+
+// runContainerWithAllConfig creates and starts a container, using the supplied ContainerConfig, HostConfig,
+// NetworkingConfig, and container name (or the value of t.Name if containerName="").
+func runContainerWithAllConfig(t *testing.T, cli *client.Client, containerConfig *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, containerName string) string {
+	if containerName == "" {
+		containerName = t.Name()
+	}
+	if containerConfig.Image == "" {
+		containerConfig.Image = imageName()
+	}
+	// Always run as a random user, unless the test has specified otherwise
+	if containerConfig.User == "" {
+		containerConfig.User = generateRandomUID()
+	}
+	// if coverage
+	containerConfig.Env = append(containerConfig.Env, "COVERAGE_FILE="+t.Name()+".cov")
+	containerConfig.Env = append(containerConfig.Env, "EXIT_CODE_FILE="+getExitCodeFilename(t))
+	t.Logf("Running container (%s)", containerConfig.Image)
+	ctr, err := cli.ContainerCreate(context.Background(), containerConfig, hostConfig, networkingConfig, containerName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	startContainer(t, cli, ctr.ID)
+	return ctr.ID
+}
+
+// runContainerWithPorts creates and starts a container, exposing the specified ports on the host.
+// If no image is specified in the container config, then the image name is retrieved from the TEST_IMAGE
+// environment variable.
+func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *container.Config, ports []int) string {
+	hostConfig := getDefaultHostConfig(t, cli)
 	for _, p := range ports {
 		port := nat.Port(fmt.Sprintf("%v/tcp", p))
 		hostConfig.PortBindings[port] = []nat.PortBinding{
@@ -307,14 +351,7 @@ func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *co
 			},
 		}
 	}
-	networkingConfig := network.NetworkingConfig{}
+	return runContainerWithHostConfig(t, cli, containerConfig, hostConfig)
-	t.Logf("Running container (%s)", containerConfig.Image)
-	ctr, err := cli.ContainerCreate(context.Background(), containerConfig, &hostConfig, &networkingConfig, t.Name())
-	if err != nil {
-		t.Fatal(err)
-	}
-	startContainer(t, cli, ctr.ID)
-	return ctr.ID
 }
 
 // runContainer creates and starts a container.  If no image is specified in
@@ -526,6 +563,7 @@ func waitForContainer(t *testing.T, cli *client.Client, ID string, timeout time.
 
 // execContainer runs a command in a running container, and returns the exit code and output
 func execContainer(t *testing.T, cli *client.Client, ID string, user string, cmd []string) (int, string) {
+	t.Logf("Running command: %v", cmd)
 	config := types.ExecConfig{
 		User:        user,
 		Privileged:  false,
@@ -609,6 +647,9 @@ func waitForReady(t *testing.T, cli *client.Client, ID string) {
 			} else if rc == 10 {
 				t.Log("MQ Readiness: Queue Manager Running as Standby")
 				return
+			} else if rc == 20 {
+				t.Log("MQ Readiness: Queue Manager Running as Replica")
+				return
 			}
 		case <-ctx.Done():
 			t.Fatal("Timed out waiting for container to become ready")
@@ -785,12 +826,23 @@ func copyFromContainer(t *testing.T, cli *client.Client, id string, file string)
 }
 
 func getPort(t *testing.T, cli *client.Client, ID string, port int) string {
-	i, err := cli.ContainerInspect(context.Background(), ID)
+	var inspectInfo types.ContainerJSON
-	if err != nil {
+	var err error
-		t.Fatal(err)
+	for attemptsRemaining := 3; attemptsRemaining > 0; attemptsRemaining-- {
+		inspectInfo, err = cli.ContainerInspect(context.Background(), ID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		portNat := nat.Port(fmt.Sprintf("%d/tcp", port))
+		if inspectInfo.NetworkSettings.Ports[portNat] == nil || len(inspectInfo.NetworkSettings.Ports[portNat]) == 0 {
+			t.Log("Container port not yet bound")
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		return inspectInfo.NetworkSettings.Ports[portNat][0].HostPort
 	}
-	portNat := nat.Port(fmt.Sprintf("%d/tcp", port))
+	t.Fatal("Failed to get port")
-	return i.NetworkSettings.Ports[portNat][0].HostPort
+	return ""
 }
 
 func countLines(t *testing.T, r io.Reader) int {
@@ -823,3 +875,12 @@ func countTarLines(t *testing.T, b []byte) int {
 	}
 	return total
 }
+
+func getMQVersion(t *testing.T, cli *client.Client) (string, error) {
+	inspect, _, err := cli.ImageInspectWithRaw(context.Background(), imageName())
+	if err != nil {
+		return "", err
+	}
+	version := inspect.ContainerConfig.Labels["version"]
+	return version, nil
+}

test/docker/go.mod

@@ -0,0 +1,23 @@
+module github.com/ibm-messaging/mq-container/test/docker
+
+go 1.15
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/Sirupsen/logrus v1.0.5 // indirect
+	github.com/docker/distribution v2.8.0+incompatible // indirect
+	github.com/docker/docker v1.13.2-0.20170601211448-f5ec1e2936dc
+	github.com/docker/go-connections v0.4.0
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/onsi/ginkgo v1.14.1 // indirect
+	github.com/onsi/gomega v1.10.2 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/stretchr/testify v1.3.0 // indirect
+	golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 // indirect
+	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
+)

test/docker/go.sum

@@ -0,0 +1,109 @@
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
+github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/Sirupsen/logrus v1.0.5 h1:447dy9LxSj+Iaa2uN3yoFHOzU9yJcJYiQPtNz8OXtv0=
+github.com/Sirupsen/logrus v1.0.5/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/distribution v2.8.0+incompatible h1:l9EaZDICImO1ngI+uTifW+ZYvvz7fKISBAKpg+MbWbY=
+github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v1.13.2-0.20170601211448-f5ec1e2936dc h1:y4nIGNQUH6JtUV3pd6HjnzdnHq+96wMDVXhkfZ6jc4E=
+github.com/docker/docker v1.13.2-0.20170601211448-f5ec1e2936dc/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
+github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
+github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

test/docker/mq_multi_instance_test.go

@@ -32,6 +32,7 @@ var miEnv = []string{
 // TestMultiInstanceStartStop creates 2 containers in a multi instance queue manager configuration
 // and starts/stop them checking we always have an active and standby
 func TestMultiInstanceStartStop(t *testing.T) {
+	t.Skipf("Skipping %v until test defect fixed", t.Name())
 	cli, err := client.NewEnvClient()
 	if err != nil {
 		t.Fatal(err)

test/docker/mq_native_ha_test.go

@@ -0,0 +1,219 @@
+/*
+© Copyright IBM Corporation 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package main
+
+import (
+	"testing"
+
+	"github.com/docker/docker/client"
+)
+
+// TestNativeHABasic creates 3 containers in a Native HA queue manager configuration
+// and ensures the queue manger and replicas start as expected
+func TestNativeHABasic(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	version, err := getMQVersion(t, cli)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version < "9.2.2.0" {
+		t.Skipf("Skipping %s as test requires at least MQ 9.2.2.0, but image is version %s", t.Name(), version)
+	}
+
+	containerNames := [3]string{"QM1_1", "QM1_2", "QM1_3"}
+	qmReplicaIDs := [3]string{}
+	qmVolumes := []string{}
+	qmNetwork, err := createBridgeNetwork(cli, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeBridgeNetwork(cli, qmNetwork.ID)
+
+	for i := 0; i <= 2; i++ {
+		vol := createVolume(t, cli, containerNames[i])
+		defer removeVolume(t, cli, vol.Name)
+		qmVolumes = append(qmVolumes, vol.Name)
+
+		containerConfig := getNativeHAContainerConfig(containerNames[i], containerNames, defaultHAPort)
+		hostConfig := getHostConfig(t, 1, "", "", vol.Name)
+		networkingConfig := getNativeHANetworkConfig(qmNetwork.ID)
+
+		ctr := runContainerWithAllConfig(t, cli, &containerConfig, &hostConfig, &networkingConfig, containerNames[i])
+		defer cleanContainer(t, cli, ctr)
+		qmReplicaIDs[i] = ctr
+	}
+
+	waitForReadyHA(t, cli, qmReplicaIDs)
+
+	_, err = getActiveReplicaInstances(t, cli, qmReplicaIDs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
+
+// TestNativeHAFailover creates 3 containers in a Native HA queue manager configuration,
+// stops the active queue manager, checks a replica becomes active, and ensures the stopped
+// queue manager comes back as a replica
+func TestNativeHAFailover(t *testing.T) {
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	version, err := getMQVersion(t, cli)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version < "9.2.2.0" {
+		t.Skipf("Skipping %s as test requires at least MQ 9.2.2.0, but image is version %s", t.Name(), version)
+	}
+
+	containerNames := [3]string{"QM1_1", "QM1_2", "QM1_3"}
+	qmReplicaIDs := [3]string{}
+	qmVolumes := []string{}
+	qmNetwork, err := createBridgeNetwork(cli, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeBridgeNetwork(cli, qmNetwork.ID)
+
+	for i := 0; i <= 2; i++ {
+		vol := createVolume(t, cli, containerNames[i])
+		defer removeVolume(t, cli, vol.Name)
+		qmVolumes = append(qmVolumes, vol.Name)
+
+		containerConfig := getNativeHAContainerConfig(containerNames[i], containerNames, defaultHAPort)
+		hostConfig := getHostConfig(t, 1, "", "", vol.Name)
+		networkingConfig := getNativeHANetworkConfig(qmNetwork.ID)
+
+		ctr := runContainerWithAllConfig(t, cli, &containerConfig, &hostConfig, &networkingConfig, containerNames[i])
+		defer cleanContainer(t, cli, ctr)
+		qmReplicaIDs[i] = ctr
+	}
+
+	waitForReadyHA(t, cli, qmReplicaIDs)
+
+	haStatus, err := getActiveReplicaInstances(t, cli, qmReplicaIDs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stopContainer(t, cli, haStatus.Active)
+	waitForFailoverHA(t, cli, haStatus.Replica)
+	startContainer(t, cli, haStatus.Active)
+	waitForReady(t, cli, haStatus.Active)
+
+	_, err = getActiveReplicaInstances(t, cli, qmReplicaIDs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
+
+// TestNativeHASecure creates 3 containers in a Native HA queue manager configuration
+// with HA TLS enabled, and ensures the queue manger and replicas start as expected
+func TestNativeHASecure(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	version, err := getMQVersion(t, cli)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version < "9.2.2.0" {
+		t.Skipf("Skipping %s as test requires at least MQ 9.2.2.0, but image is version %s", t.Name(), version)
+	}
+
+	containerNames := [3]string{"QM1_1", "QM1_2", "QM1_3"}
+	qmReplicaIDs := [3]string{}
+	qmNetwork, err := createBridgeNetwork(cli, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeBridgeNetwork(cli, qmNetwork.ID)
+
+	for i := 0; i <= 2; i++ {
+		containerConfig := getNativeHAContainerConfig(containerNames[i], containerNames, defaultHAPort)
+		containerConfig.Env = append(containerConfig.Env, "MQ_NATIVE_HA_TLS=true")
+		hostConfig := getNativeHASecureHostConfig(t)
+		networkingConfig := getNativeHANetworkConfig(qmNetwork.ID)
+
+		ctr := runContainerWithAllConfig(t, cli, &containerConfig, &hostConfig, &networkingConfig, containerNames[i])
+		defer cleanContainer(t, cli, ctr)
+		qmReplicaIDs[i] = ctr
+	}
+
+	waitForReadyHA(t, cli, qmReplicaIDs)
+
+	_, err = getActiveReplicaInstances(t, cli, qmReplicaIDs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
+
+// TestNativeHASecure creates 3 containers in a Native HA queue manager configuration
+// with HA TLS enabled, overrides the default CipherSpec, and ensures the queue manger
+// and replicas start as expected
+func TestNativeHASecureCipherSpec(t *testing.T) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	version, err := getMQVersion(t, cli)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version < "9.2.2.0" {
+		t.Skipf("Skipping %s as test requires at least MQ 9.2.2.0, but image is version %s", t.Name(), version)
+	}
+
+	containerNames := [3]string{"QM1_1", "QM1_2", "QM1_3"}
+	qmReplicaIDs := [3]string{}
+	qmNetwork, err := createBridgeNetwork(cli, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeBridgeNetwork(cli, qmNetwork.ID)
+
+	for i := 0; i <= 2; i++ {
+		containerConfig := getNativeHAContainerConfig(containerNames[i], containerNames, defaultHAPort)
+		containerConfig.Env = append(containerConfig.Env, "MQ_NATIVE_HA_TLS=true", "MQ_NATIVE_HA_CIPHERSPEC=TLS_AES_256_GCM_SHA384")
+		hostConfig := getNativeHASecureHostConfig(t)
+		networkingConfig := getNativeHANetworkConfig(qmNetwork.ID)
+
+		ctr := runContainerWithAllConfig(t, cli, &containerConfig, &hostConfig, &networkingConfig, containerNames[i])
+		defer cleanContainer(t, cli, ctr)
+		qmReplicaIDs[i] = ctr
+	}
+
+	waitForReadyHA(t, cli, qmReplicaIDs)
+
+	_, err = getActiveReplicaInstances(t, cli, qmReplicaIDs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}

test/docker/mq_native_ha_test_util.go

@@ -0,0 +1,149 @@
+/*
+© Copyright IBM Corporation 2021
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package main
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/client"
+)
+
+const defaultHAPort = 9414
+
+// HAReplicaStatus represents the Active/Replica/Replica container status of the queue manager
+type HAReplicaStatus struct {
+	Active  string
+	Replica [2]string
+}
+
+func createBridgeNetwork(cli *client.Client, t *testing.T) (types.NetworkCreateResponse, error) {
+	return cli.NetworkCreate(context.Background(), t.Name(), types.NetworkCreate{})
+}
+
+func removeBridgeNetwork(cli *client.Client, networkID string) error {
+	return cli.NetworkRemove(context.Background(), networkID)
+}
+
+func getNativeHAContainerConfig(containerName string, replicaNames [3]string, haPort int) container.Config {
+	return container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=QM1",
+			"AMQ_CLOUD_PAK=true",
+			"MQ_NATIVE_HA=true",
+			fmt.Sprintf("HOSTNAME=%s", containerName),
+			fmt.Sprintf("MQ_NATIVE_HA_INSTANCE_0_NAME=%s", replicaNames[0]),
+			fmt.Sprintf("MQ_NATIVE_HA_INSTANCE_1_NAME=%s", replicaNames[1]),
+			fmt.Sprintf("MQ_NATIVE_HA_INSTANCE_2_NAME=%s", replicaNames[2]),
+			fmt.Sprintf("MQ_NATIVE_HA_INSTANCE_0_REPLICATION_ADDRESS=%s(%d)", replicaNames[0], haPort),
+			fmt.Sprintf("MQ_NATIVE_HA_INSTANCE_1_REPLICATION_ADDRESS=%s(%d)", replicaNames[1], haPort),
+			fmt.Sprintf("MQ_NATIVE_HA_INSTANCE_2_REPLICATION_ADDRESS=%s(%d)", replicaNames[2], haPort),
+		},
+	}
+}
+
+func getNativeHASecureHostConfig(t *testing.T) container.HostConfig {
+	return container.HostConfig{
+		Binds: []string{
+			coverageBind(t),
+			filepath.Join(getCwd(t, true), "../tls") + ":/etc/mqm/ha/pki/keys/ha",
+		},
+	}
+}
+
+func getNativeHANetworkConfig(networkID string) network.NetworkingConfig {
+	return network.NetworkingConfig{
+		EndpointsConfig: map[string]*network.EndpointSettings{
+			networkID: &network.EndpointSettings{},
+		},
+	}
+}
+
+func getActiveReplicaInstances(t *testing.T, cli *client.Client, qmReplicaIDs [3]string) (HAReplicaStatus, error) {
+
+	var actives []string
+	var replicas []string
+
+	for _, id := range qmReplicaIDs {
+		qmReplicaStatus := getQueueManagerStatus(t, cli, id, "QM1")
+		if qmReplicaStatus == "Running" {
+			actives = append(actives, id)
+		} else if qmReplicaStatus == "Replica" {
+			replicas = append(replicas, id)
+		} else {
+			err := fmt.Errorf("Expected status to be Running or Replica, got status: %s", qmReplicaStatus)
+			return HAReplicaStatus{}, err
+		}
+	}
+
+	if len(actives) != 1 || len(replicas) != 2 {
+		err := fmt.Errorf("Expected 1 Active and 2 Replicas, got: %d Active and %d Replica", len(actives), len(replicas))
+		return HAReplicaStatus{}, err
+	}
+
+	return HAReplicaStatus{actives[0], [2]string{replicas[0], replicas[1]}}, nil
+}
+
+func waitForReadyHA(t *testing.T, cli *client.Client, qmReplicaIDs [3]string) {
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	for {
+		select {
+		case <-time.After(1 * time.Second):
+			for _, id := range qmReplicaIDs {
+				rc, _ := execContainer(t, cli, id, "", []string{"chkmqready"})
+				if rc == 0 {
+					t.Log("MQ is ready")
+					rc, _ := execContainer(t, cli, id, "", []string{"chkmqstarted"})
+					if rc == 0 {
+						t.Log("MQ has started")
+						return
+					}
+				}
+			}
+		case <-ctx.Done():
+			t.Fatal("Timed out waiting for HA Queue Manager to become ready")
+		}
+	}
+}
+
+func waitForFailoverHA(t *testing.T, cli *client.Client, replicas [2]string) {
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	for {
+		select {
+		case <-time.After(1 * time.Second):
+			for _, id := range replicas {
+				if status := getQueueManagerStatus(t, cli, id, "QM1"); status == "Running" {
+					return
+				}
+			}
+		case <-ctx.Done():
+			t.Fatal("Timed out waiting for Native HA Queue Manager to failover to an available replica")
+		}
+	}
+}

test/messaging/Dockerfile

@@ -1,4 +1,4 @@
-# © Copyright IBM Corporation 2018, 2019
+# © Copyright IBM Corporation 2018, 2021
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,22 +15,24 @@
 ###############################################################################
 # Application build environment (Maven)
 ###############################################################################
-FROM docker.io/maven:3-ibmjava as builder
+FROM registry.redhat.io/ubi8/openjdk-8 as builder
-COPY pom.xml /usr/src/mymaven/
+COPY pom.xml .
-WORKDIR /usr/src/mymaven
+#WORKDIR /usr/src/mymaven
 # Download dependencies separately, so Docker caches them
 RUN mvn dependency:go-offline install
 # Copy source
-COPY src /usr/src/mymaven/src
+COPY src .
 # Run the main build
 RUN mvn --offline install
 # Print a list of all the files (useful for debugging)
-RUN find /usr/src/mymaven
+RUN find .
 
 ###############################################################################
 # Application runtime (JRE only, no build environment)
 ###############################################################################
-FROM docker.io/ibmjava:8-jre
+# OpenJDK is not technically supported with the MQ client, but is good enough for these tests
-COPY --from=builder /usr/src/mymaven/target/*.jar /opt/app/
+FROM registry.redhat.io/ubi8/openjdk-8-runtime
-COPY --from=builder /usr/src/mymaven/target/lib/*.jar /opt/app/
+COPY --from=builder /home/jboss/target/*.jar /opt/app/
+COPY --from=builder /home/jboss/target/lib/*.jar /opt/app/
+USER 1001
 ENTRYPOINT ["java", "-classpath", "/opt/app/*", "org.junit.platform.console.ConsoleLauncher", "-p", "com.ibm.mqcontainer.test", "--details", "verbose"]

test/messaging/pom.xml

@@ -1,5 +1,5 @@
 <!--
-© Copyright IBM Corporation 2018
+© Copyright IBM Corporation 2018, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ limitations under the License.
     <dependency>
       <groupId>com.ibm.mq</groupId>
       <artifactId>com.ibm.mq.allclient</artifactId>
-      <version>9.1.3.0</version>
+      <version>9.2.0.0</version>
       <scope>compile</scope>
     </dependency>
     <dependency>
@@ -65,5 +65,9 @@ limitations under the License.
         </executions>
       </plugin>
     </plugins>
-</build>
+  </build>
+  <properties>
+      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+      <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+  </properties>
 </project>

test/messaging/src/main/java/com/ibm/mqcontainer/test/JMSTests.java

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2018
+© Copyright IBM Corporation 2018, 2021
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@ limitations under the License.
 */
 package com.ibm.mqcontainer.test;
 
-import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -32,9 +32,12 @@ import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLSocketFactory;
 import javax.net.ssl.TrustManagerFactory;
 
+import com.ibm.mq.MQException;
+import com.ibm.mq.constants.MQConstants;
 import com.ibm.mq.jms.MQConnectionFactory;
 import com.ibm.mq.jms.MQQueue;
 import com.ibm.msg.client.wmq.WMQConstants;
+import com.ibm.msg.client.jms.DetailedJMSSecurityRuntimeException;
 
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
@@ -57,27 +60,18 @@ class JMSTests {
     static SSLSocketFactory createSSLSocketFactory() throws IOException, GeneralSecurityException {
         KeyStore ts=KeyStore.getInstance("jks");
         ts.load(new FileInputStream(TRUSTSTORE), PASSPHRASE.toCharArray());
-        // KeyManagerFactory kmf=KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
         TrustManagerFactory tmf=TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
         tmf.init(ts);
-        // tmf.init();
         SSLContext ctx = SSLContext.getInstance("TLSv1.2");
-        // Security.setProperty("crypto.policy", "unlimited");
         ctx.init(null, tmf.getTrustManagers(), null);
         return ctx.getSocketFactory();
     }
 
-    static JMSContext create(String channel, String addr, String user, String password) throws JMSException, IOException, GeneralSecurityException {
+    static MQConnectionFactory createMQConnectionFactory(String channel, String addr) throws JMSException, IOException, GeneralSecurityException {
-        LOGGER.info(String.format("Connecting to %s/TCP/%s(1414) as %s", channel, addr, user));
         MQConnectionFactory factory = new MQConnectionFactory();
         factory.setTransportType(WMQConstants.WMQ_CM_CLIENT);
         factory.setChannel(channel);
         factory.setConnectionNameList(String.format("%s(1414)", addr));
-        // If a password is set, make sure it gets sent to the queue manager for authentication
-        if (password != null) {
-            factory.setBooleanProperty(WMQConstants.USER_AUTHENTICATION_MQCSP, true);
-        }
-        // factory.setClientReconnectOptions(WMQConstants.WMQ_CLIENT_RECONNECT);
         if (TRUSTSTORE == null) {
             LOGGER.info("Not using TLS");
         }
@@ -94,12 +88,33 @@ class JMSTests {
                  factory.setSSLCipherSuite("TLS_RSA_WITH_AES_128_CBC_SHA256");
             }
         }
-        // Give up if unable to reconnect for 10 minutes
+        return factory;
-        // factory.setClientReconnectTimeout(600);
+    }
-        // LOGGER.info(String.format("user=%s pw=%s", user, password));
+
+    /**
+     * Create a JMSContext with the supplied user and password.
+     */
+    static JMSContext create(String channel, String addr, String user, String password) throws JMSException, IOException, GeneralSecurityException {
+        LOGGER.info(String.format("Connecting to %s/TCP/%s(1414) as %s", channel, addr, user));
+        MQConnectionFactory factory = createMQConnectionFactory(channel, addr);
+        // If a password is set, make sure it gets sent to the queue manager for authentication
+        if (password != null) {
+            factory.setBooleanProperty(WMQConstants.USER_AUTHENTICATION_MQCSP, true);
+        }
+        LOGGER.info(String.format("CSP authentication: %s", factory.getBooleanProperty(WMQConstants.USER_AUTHENTICATION_MQCSP)));
         return factory.createContext(user, password);
     }
 
+    /**
+     * Create a JMSContext with the default user identity (from the OS)
+     */
+    static JMSContext create(String channel, String addr) throws JMSException, IOException, GeneralSecurityException {
+        LOGGER.info(String.format("Connecting to %s/TCP/%s(1414) as OS user '%s'", channel, addr, System.getProperty("user.name")));
+        MQConnectionFactory factory = createMQConnectionFactory(channel, addr);
+        LOGGER.info(String.format("CSP authentication: %s", factory.getBooleanProperty(WMQConstants.USER_AUTHENTICATION_MQCSP)));
+        return factory.createContext();
+    }
+
     @BeforeAll
     private static void waitForQueueManager() {
         for (int i = 0; i < 20; i++) {
@@ -116,28 +131,34 @@ class JMSTests {
         }
     }
 
-    @BeforeEach
-    void connect() throws Exception {
-        context = create(CHANNEL, ADDR, USER, PASSWORD);
-    }
-
     @Test
-    void succeedingTest(TestInfo t) throws JMSException {
+    void putGetTest(TestInfo t) throws Exception {
+        context = create(CHANNEL, ADDR, USER, PASSWORD);
         Queue queue = new MQQueue("DEV.QUEUE.1");
         context.createProducer().send(queue, t.getDisplayName());
         Message m = context.createConsumer(queue).receive();
         assertNotNull(m.getBody(String.class));
     }
 
-    // @Test
-    // void failingTest() {
-    //     fail("a failing test");
-    // }
-
     @Test
-    @Disabled("for demonstration purposes")
+    void defaultIdentityTest(TestInfo t) throws Exception {
-    void skippedTest() {
+        LOGGER.info(String.format("Password='%s'", PASSWORD));
-        // not executed
+        try {
+            // Don't pass a user/password, which should cause the default identity to be used
+            context = create(CHANNEL, ADDR);
+        } catch (DetailedJMSSecurityRuntimeException ex) {
+            Throwable cause = ex.getCause();
+            assertNotNull(cause);
+            assertTrue(cause instanceof MQException);
+            assertEquals(MQConstants.MQRC_NOT_AUTHORIZED, ((MQException)cause).getReason());
+            return;
+        }
+        // The default developer config allows any user to appear as "app", and use a blank password.  This is done with the MCAUSER on the channel.
+        // If this test is run on a queue manager without a password set, then it should be possible to connect without exception.
+        // If this test is run on a queue manager with a password set, then an exception should be thrown, because this test doesn't send a password.
+        if ((PASSWORD != null) && (PASSWORD != "")) {
+            fail("Exception not thrown");
+        }
     }
 
     @AfterEach
@@ -146,9 +167,4 @@ class JMSTests {
             context.close();
         }
     }
-
-    @AfterAll
-    static void tearDownAll() {
-    }
-
 }

test/tls/generate-test-cert.sh

@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/bin/bash -ex
 # -*- mode: sh -*-
-# © Copyright IBM Corporation 2018
+# © Copyright IBM Corporation 2018, 2021
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ PASSWORD=passw0rd
 openssl req \
        -newkey rsa:2048 -nodes -keyout ${KEY} \
        -subj "/CN=localhost" \
+       -addext "subjectAltName = DNS:localhost" \
        -x509 -days 3650 -out ${CERT}
 
 # Add the key and certificate to a PKCS #12 key store, for the server to use

test/tls/server.crt

@@ -1,17 +1,17 @@
 -----BEGIN CERTIFICATE-----
-MIICpDCCAYwCCQC6vpJFnfYO6TANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+MIICwzCCAaugAwIBAgIJAKnwG0VGiDelMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
-b2NhbGhvc3QwHhcNMTkwMzIxMTYxMzUxWhcNMjkwMzE4MTYxMzUxWjAUMRIwEAYD
+BAMMCWxvY2FsaG9zdDAeFw0yMTA5MDYxNTIyMDlaFw0zMTA5MDQxNTIyMDlaMBQx
-VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCu
+EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
-48qtIDwmihFqj2HY3dZjPfROA1MJ+D0c6aEA08ooOczthLB7XdZBQDapj8LFldyt
+ggEBALLmDX3OB/4DlzJzKz/Edc5qVdjdIN/u8pApSQPevT0mAsSK1uw2MObeOo4C
-4ZMbTkqtF5QtPXmJY0wi39foLYlcGXPL1b7y3mypaFou88BcSM3VmfILKXhNeAlt
+DCBmabYeuvGzZ4t3SiejdsHK+qAYMFW51lxTbulv5kUPvTBOY2JCENkVDFjqcK8S
-rXevnuT5kDU7sLVgKGhGwas20T1MU7d0I3bQ5z5c7egL76Hk9fYucjN6RkbwlrJ3
+9ItI/UzTmnBolvZmUKzROHzA/pFb/jkhlzqJO+TqIBXKLF5gdFFTiHHcqfoUyVOV
-TrCXrGIziofn3Zq1t51ygv21c80JD3XJ44YmuCrede4rhOS/4NpwRuZyiwpJ6tlv
+n+49V8z6W2rokz4QIWa5Dlh6VS1B6VXdihJv5P9HV8P8FOtefhA85yaSVKlFS/AC
-0L0QSDGCmt2JT3ty28UAsGznFzC5Qu9KyaR+9Gk4aftiyKxrYWZkgtJmMRU+C1X2
+XRb5FmtmYHBnghLktHS71s/KcPeX27Q1NcKhmZMvHRH95hqEcP25S6SGu69eiCLk
-kFLOHsucGmJswjwubSR7AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAEdlmXVGy86P
+xpbJKqG3fntfooLUDfR2PHQUJ7UCAwEAAaMYMBYwFAYDVR0RBA0wC4IJbG9jYWxo
-XIX5a4ZmHQ5Ns4wm7rY8vzUxlymEQ86En1PN1zAO9gV94tLyNeMptjsFEEo/uJhC
+b3N0MA0GCSqGSIb3DQEBCwUAA4IBAQBamIH23oDh1XxOeMPUEyPLHm5M8LM8FNhT
-Yvg3l5TIr/WCiY2+2XsSHvnbXrlbF3S0fRHa9VaCMRKjzRT68uq2Y891906YGtUE
+GEpf1ICy4TSLipSFhIs3hGzVt22zBBzU59apQ8rXUME5SK+9PLag/t/48rG1SfUA
-m6fCjHqVzX8qaplDf79aVkPydYaYOIZ1a/mCfQcD9XMZ/v5zI9IUDhdoq97bgPhB
+VyYvaeu/cA5NQMWwuyCLqZL1MWn+BLsdAiNtbNHANesnl0i+vUb0GPzSP8soe3PP
-gBOzWLI+hkzyU8jxKAFw1Hwi9lD/P6RXL5arNb/+arOgA3vTW+xGWGevgjVK1Ay9
+N7Fh8SO3Qq6e9zT3iE2tP2OFxzcpg538Xn3qoVPJwmLIfBtvsiK07zqAWdqBWtt6
-81beWiQmn0KbeLZxj+WJ9Nntlf1M4EqPYgsSYs/IlJTYS8W1B0mDJEoovPdFTryY
+cBXyagnmgKvOyv6sKAlTpwP9HqVem3XxZVrhm1KiPHs4Dnks6e79txmB8lqzvWu5
-GyIuQEVcjUE=
+tu4h2ePGJjqUy5JkkoDY0j6hALwEe3ZXBvJ6XUQDi9Hou2k+MaQd
 -----END CERTIFICATE-----

test/tls/server.key

@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCu48qtIDwmihFq
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCy5g19zgf+A5cy
-j2HY3dZjPfROA1MJ+D0c6aEA08ooOczthLB7XdZBQDapj8LFldyt4ZMbTkqtF5Qt
+cys/xHXOalXY3SDf7vKQKUkD3r09JgLEitbsNjDm3jqOAgwgZmm2Hrrxs2eLd0on
-PXmJY0wi39foLYlcGXPL1b7y3mypaFou88BcSM3VmfILKXhNeAltrXevnuT5kDU7
+o3bByvqgGDBVudZcU27pb+ZFD70wTmNiQhDZFQxY6nCvEvSLSP1M05pwaJb2ZlCs
-sLVgKGhGwas20T1MU7d0I3bQ5z5c7egL76Hk9fYucjN6RkbwlrJ3TrCXrGIziofn
+0Th8wP6RW/45IZc6iTvk6iAVyixeYHRRU4hx3Kn6FMlTlZ/uPVfM+ltq6JM+ECFm
-3Zq1t51ygv21c80JD3XJ44YmuCrede4rhOS/4NpwRuZyiwpJ6tlv0L0QSDGCmt2J
+uQ5YelUtQelV3YoSb+T/R1fD/BTrXn4QPOcmklSpRUvwAl0W+RZrZmBwZ4IS5LR0
-T3ty28UAsGznFzC5Qu9KyaR+9Gk4aftiyKxrYWZkgtJmMRU+C1X2kFLOHsucGmJs
+u9bPynD3l9u0NTXCoZmTLx0R/eYahHD9uUukhruvXogi5MaWySqht357X6KC1A30
-wjwubSR7AgMBAAECggEAH9t6teKjUlngJksMBdcTEGzerb9JRw2jBDtCisYJkx5E
+djx0FCe1AgMBAAECggEAagD5A49+mtwjzigB+4H80Def8KVuomIi5psgAaQM+9u3
-SBfdlftX5fbufiCj2B4eXsYyZ8zxKWqcIUmLdA1Udx3TVIXG+bHhOAYtjEwb+xf5
+DiC6ozKlHVeW2KiL6PLmNpzU5v0IINKpZP1uE/yjLxPGKDW6t/BUKww8JLXjw2jf
-JYhdR/IzHG+4eXQKaAIvpXztyl3lU9iC+eaMg4GYzRrGN2wSAG9XgZ5cLF2TLJYU
+aMx+0TKwo0sfRA32S0YPmWNVAsBmm1AbA5vhXcK51QXuiInH406H5+d25ZJrYevF
-jPxp7goz9X6V57aL2G/EFlbFsMaI/6cW7+XoRdo0I4N2Z766gz7GgyxtTVwR5Peq
+liKWSjx9CM/0XO7t20j18mCa8RjBEdsZoHxHsoWNvFJ6DCR25cFShAhR7s4OtkUk
-LjOpqSNS0W57KJxReURfySok9CP1DfyigopsYW8O4jGVDDRLdiN3I8+JhWya2E0j
+yELm1tYYrFOffUM0Q/Fp9uSlCHWMSqPtf/6NEfnszfFEtzDh/N+YqC1Bexv0XPsD
-96hHpN04Oz6HnMm7bdZDVtkZCOiu6xIzLJJxZ4o+kQKBgQDYqOA/hSod7s7w4LBE
+dBPOkUZjWA2Sc8Se1t2GLfrRURzj3GvWH1+GssjsAQKBgQDeIdyzQSqce4Kn0opa
-A6Mp+e0//PYH6/N9SKmSIgQNec9bMGI4yanoblMbg4GM1g7pkvjlC0nTdjnUbLkB
+vdS5moCiv3pyfNd0nYe0awgVos4kHY7/nBq0eyMZAatRHeD3DunVsw3LmvWyEw53
-vIvtVh3XwTIlrZ/4lc7VB23/hmKU+lRc+NJP5fgasAQu0W3+qp2cXo0pnHVwBEku
+app7MTTjVrYaadoBlB6jy2elyF5RcW2jGchZExoNh+0ZQWUiMbYEozPLQTy9ZxMz
-Z7FwDPX0JNDIi/Or2I7dt8JojQKBgQDOpU1AnIXv1/cToYK4nz8BWLxRxwLTxy5A
+t0OcZ1hHPngGgmj5TELZKkwEtQKBgQDOLLh7pdKrdudtim+o4H20jl5yYKl25Iq/
-ucafNKacPlxb5luZRCExiPZwAM8Z3zI9o99rYXOPQmsnknZWJV66Zx0Vo0yTD1CT
+DKVodwUd94cM7xAIOQJrx2XK/YPCfRkKRN1wxzAhYdIVkaaKDVhI8Jeu+H18QOa6
-DWMUj0ugI1wORNMhwZP6YBYWjAeupyU9a7FyU1Geg4sdQt5rMyAEQOoECc8x8foP
+5OlzzZcqJCtACpbVqLaDcmq8pRrAYekiwMIKwC95llvktjilvLfoUnQoXAaX8E8B
-rySHuO/TJwKBgBjMM2ZxymFErQDa5rHSLMGoLmRtgodjlSnYwDfOluIn9/i67/MJ
+yCSUvDh3AQKBgQCxa0h04DLho4Da/D3HdmHHERF3bAqoEPCh0wTF5MsjRNLzY6yI
-+d11iyOSCKji8y/+t2gXw6plVLcgfohZWTaf7ah9H006sx2Tn+m4APoHGo9sm21M
+mq11w/hni77C3mOF0SKRrh7xpcZiQfhHBx12EfpVLjfq5uraYe0LFHanol87G6bf
-uV2Vt7DuRnxJUiqcwo9cLxH9K1/Xzbx299MYWKpJ8G+TvR8FGUz9NE4dAoGAM5gs
+I8Oy6Z/geNW2W1YktqHUGGpRCL0z5nUe1FyrOpv2431Ibbbcj73A6JipFQKBgHdl
-KKSsAE1QwFMEG2qPRZvNMTHaL9w8XSbFQ7zWmI4tazihyCutifujZCWfj9sdZSyE
+vJyWpk73+AQe1JUnFIU4oYd5ZQpeRd9n8m5x5ru4+jPKSi2I3lcOTWvlrqU2Dwc8
-PQBQ5QT1UiUMbMfZ1fqm1V83YERjnsOp6Fk6zZnmgx2GBZiahNn2ydxekqni72nz
+ZEUIhV3/qUsmYxy1p7ft5NnGO912NGhtYqjWmcEk2wsmVr17C99JpniC4OAik4G1
-HRNWfphjZIPsmqFiLg2zIBz+4X6EK+RT35s6LeMCgYEAwF/9jX8kONW5KKZdoNHa
+wWm6bIPsSGFGCb4pcROQlIY+7O6WkxqEDnM4ITcBAoGAHXBKmadFpupUeGSkCwEo
-opkLpa9qkwTGQ9M3AZiRUjM4rtvggYt8FBEP+3BLDLHqfUOkPq82MCRXm+6Cz+sT
+/VjeI4QoKKcWj9K8z8ifCVPz1FiQ1AJ91WMTM7PAmpEDX058Hor9xxJ2bEtQFwUS
-gyPnsPlAh/sr3Pys3olJbUDE9H24k1LU0CI/sSwAFkka0+Q7PVTTe/Dcavitrcrm
+QKvjeU+/Ig0TWjsJBgBPvc0xYLaJptAbjvG4a5nBn7hwbRzLTcKx2OVTmdAkz00H
-+fyiT2oSPZeHSjQE9iIW3OY=
+1lq8cwizfwNgt8ldFFDDRvw=
 -----END PRIVATE KEY-----

travis-build-scripts/artifact-util.sh

@@ -0,0 +1,249 @@
+#!/bin/bash
+
+# © Copyright IBM Corporation 2020
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+usage="
+Usage: artifact-util.sh -c my-registry.com/artifacts/my-project/builds/123 -u me@org.com -p top-secret -f tagcache -l ./.tagcache --upload \"
+
+Where:
+-c - Full artifact destination hostname and path
+-u - The username to access repository
+-p - The password or api-key to access repository
+-f - Name of the file in repository
+-l - The path and name to the file whose contents is to be pushed or retrieved into
+Then one action of either
+--check - Check if the file exists
+--upload - Upload the contents of a file [-l must be specified]
+--get - Get a file and write to a local file [-l must be specified]
+--delete - Delet the remote file from repository
+"
+
+GREEN="\033[32m"
+RED="\033[31m"
+
+END="\033[0m"
+
+
+RIGHTARROW="\xE2\x96\xB6"
+BLUERIGHTARROW=${BLUE}${RIGHTARROW}${END}
+GREENRIGHTARROW=${GREEN}${RIGHTARROW}${END}
+
+ERROR=${RED}
+
+TICK="\xE2\x9C\x94"
+CROSS="\xE2\x9C\x97"
+GREENTICK=${GREEN}${TICK}${END}
+REDCROSS=${RED}${CROSS}${END}
+
+
+SPACER="\n\n"
+
+USER=
+CREDENTIAL=
+FILE_NAME=
+BUILD_ID=
+REGISTRY_HOSTNAME=
+FILE_LOCATION=
+PROPERTY_NAME=
+
+CHECK=false
+UPLOAD=false
+GET=false
+GET_PROPERTY=false
+DELETE=false
+DELETE_NAMESPACE=false
+num_commands_selected=0
+while getopts "f:u:p:c:l:n:-:" flag
+do
+    case "${flag}" in
+        f) FILE_NAME=${OPTARG};;
+        u) USER=${OPTARG};;
+        p) CREDENTIAL=${OPTARG};;
+        c) CACHE_PATH=${OPTARG};;
+        l) FILE_LOCATION=${OPTARG};;
+        n) PROPERTY_NAME=${OPTARG};;
+        -)
+            case "${OPTARG}" in
+                check)
+                    CHECK=true
+                    num_commands_selected=$((num_commands_selected+1))
+                    ;;
+                upload)
+                    UPLOAD=true
+                    num_commands_selected=$((num_commands_selected+1))
+                    ;;
+                get)
+                    GET=true
+                    num_commands_selected=$((num_commands_selected+1))
+                    ;;
+                get-property)
+                    GET_PROPERTY=true
+                    num_commands_selected=$((num_commands_selected+1))
+                    ;;
+                delete) 
+                    DELETE=true
+                    num_commands_selected=$((num_commands_selected+1))
+                    ;;
+                delete-namespace)
+                    DELETE_NAMESPACE=true
+                    num_commands_selected=$((num_commands_selected+1))
+                    ;;
+                *)
+                    if [ "$OPTERR" = 1 ] && [ "${optspec:0:1}" != ":" ]; then
+                        echo "Unknown option --${OPTARG}" >&2
+                    fi
+                    ;;
+            esac;;
+    esac
+done
+
+if [[ $num_commands_selected == 0 || $num_commands_selected -gt 1 ]]; then
+    printf "${REDCROSS} ${ERROR}Too many actions specified. Should be one of ${END}--check${ERROR},${END} --get${ERROR},${END} --upload${ERROR} or${END} --delete${ERROR}!${END}\n"
+    printf $SPACER
+    printf "${ERROR}$usage${END}\n"
+    exit 1
+fi
+
+if [ "$DELETE_NAMESPACE" != "true" ]; then
+    if [[ -z $CACHE_PATH|| -z $USER || -z $CREDENTIAL || -z $FILE_NAME ]] ; then 
+    printf "${REDCROSS} ${ERROR}Missing parameter!${END}\n"
+    printf "Cache Path:"$CACHE_PATH"\n"
+    printf "File name:"$FILE_NAME"\n"
+    printf "User":$USER"\n"
+    printf $SPACER
+    printf "${ERROR}$usage${END}\n"
+    exit 1
+    fi
+fi
+
+REMOTE_PATH="https://${CACHE_PATH}/$TRAVIS_BUILD_ID"
+
+if [ "$CHECK" == "true" ]; then
+    printf "${GREENRIGHTARROW} Checking to see if file ${FILE_NAME} exists in repository ${REMOTE_PATH}\n"
+    FILE_FOUND=`curl -u ${USER}:${CREDENTIAL} -X GET  "${REMOTE_PATH}/${FILE_NAME}" -o /dev/null -w "%{http_code}" -s`
+    if [ "$FILE_FOUND" != "200" ]; then
+        printf "${REDCROSS} File ${FILE_NAME} was not found\n"
+        exit 1
+    else
+        printf "${GREENTICK} File ${FILE_NAME} was found\n"
+    fi
+fi
+
+if [ "$UPLOAD" == "true" ]; then
+    printf "${GREENRIGHTARROW} Attempting to upload the file ${FILE_NAME} to repository ${REMOTE_PATH}\n"
+    if [[ -z $FILE_LOCATION ]]; then
+        printf "${REDCROSS} Location for ${FILE_NAME} was not supplied please do so\n"
+        printf $SPACER
+        printf "${ERROR}$usage${END}\n"
+        exit 1
+    fi
+    if [ ! -f "$FILE_LOCATION" ]; then
+        printf "${REDCROSS} Location supplied ${FILE_LOCATION } for file ${FILE_NAME} did not resolve to a file with contents to upload\n"
+        printf $SPACER
+        printf "${ERROR}$usage${END}\n"
+        exit 1
+    fi
+    curl -u ${USER}:${CREDENTIAL} -X PUT "$REMOTE_PATH/${FILE_NAME}" -T ${FILE_LOCATION}
+fi
+
+if [ "$GET" == "true" ]; then
+    printf "${GREENRIGHTARROW} Attempting to download file ${FILE_NAME} from repository ${REMOTE_PATH} to ${FILE_LOCATION}\n"
+    if [[ -z $FILE_LOCATION ]]; then
+        printf "${REDCROSS} Location for ${FILE_NAME} was not supplied please do so\n"
+        printf $SPACER
+        printf "${ERROR}$usage${END}\n"
+        exit 1
+    fi
+    curl -u ${USER}:${CREDENTIAL} "$REMOTE_PATH/${FILE_NAME}" -o ${FILE_LOCATION} -s
+    if [ $? != 0 ]; then
+        printf "${REDCROSS} Failed download\n"
+    else
+        printf "${GREENTICK} File ${FILE_NAME} was downloaded to ${FILE_LOCATION}\n"
+    fi
+fi
+
+if [ "$GET_PROPERTY" == "true" ]; then
+    if [[ -z $PROPERTY_NAME ]]; then
+        printf "${REDCROSS} Property name to retrieve from '${FILE_NAME}' was not supplied please do so\n"
+        printf $SPACER
+        printf "${ERROR}$usage${END}\n"
+        exit 1
+    fi
+    if [[ -z $FILE_LOCATION ]]; then
+        printf "${REDCROSS} File location to store property value in was not supplied please do so\n"
+        printf $SPACER
+        printf "${ERROR}$usage${END}\n"
+        exit 1
+    fi
+    printf "${GREENRIGHTARROW} Attempting to retrieve ${PROPERTY_NAME} of ${FILE_NAME} from repository ${REMOTE_PATH} and store it in ${FILE_LOCATION}\n"
+
+    query_url="${FILE_NAME}"
+    query_url="${query_url/\/artifactory\//\/artifactory\/api\/storage\//}?properties=${PROPERTY_NAME}"
+    request_result="$(curl -s -u ${USER}:${CREDENTIAL} "${query_url}")"
+
+    if [ $? != 0 ]; then
+        printf "Unable to retrieve properties from ${query_url}"
+        exit 1
+    else
+        printf "${GREENTICK} Properties retrieved from ${query_url}"
+    fi
+
+    jq -r '.properties.snapshot|first' <<<"$request_result" > ${FILE_LOCATION}
+
+    if [ $? != 0 ]; then
+        printf "Unable to write snapshot property to ${FILE_LOCATION}"
+        exit 1
+    else
+        printf "${GREENTICK} Property written to ${FILE_LOCATION}"
+    fi
+
+fi
+
+if [ "$DELETE" == "true" ]; then
+    printf "${GREENRIGHTARROW} Checking to see if file ${FILE_NAME} exists in repository ${REMOTE_PATH} before delete\n"
+    FILE_FOUND=`curl -u ${USER}:${CREDENTIAL} -X GET  "${REMOTE_PATH}/${FILE_NAME}" -o /dev/null -w "%{http_code}" -s`
+    if [ "$FILE_FOUND" != "200" ]; then
+        printf "${REDCROSS} File ${FILE_NAME} was not found to delete\n"
+        exit 1
+    else
+        printf "${GREENTICK} File ${FILE_NAME} was found\n"
+        printf "${GREENRIGHTARROW} Attempting the delete of ${REMOTE_PATH}/${FILE_NAME}"
+        curl -u ${USER}:${CREDENTIAL} -X DELETE  "${REMOTE_PATH}/${FILE_NAME}" -s
+        if [ $? != 0 ]; then
+            printf "${REDCROSS} Failed delete\n"
+        else
+            printf "${GREENTICK} File ${FILE_NAME} was deleted from "${REMOTE_PATH}"\n"
+        fi
+    fi
+fi
+
+if [ "$DELETE_NAMESPACE" == "true" ]; then
+    printf "${GREENRIGHTARROW} Checking to see if repository ${REMOTE_PATH} exists before delete\n"
+    DIR_FOUND=`curl -u ${USER}:${CREDENTIAL} -X GET  "${REMOTE_PATH}" -o /dev/null -w "%{http_code}" -s`
+    if [ "$DIR_FOUND" != "200" ]; then
+        printf "${REDCROSS} Namespace ${REMOTE_PATH} was not found to delete\n"
+        exit 1
+    else
+        printf "${GREENTICK} Namespace ${REMOTE_PATH} was found\n"
+        printf "${GREENRIGHTARROW} Attempting the delete of ${REMOTE_PATH}"
+        curl -u ${USER}:${CREDENTIAL} -X DELETE  "${REMOTE_PATH}" -s
+        if [ $? != 0 ]; then
+            printf "${REDCROSS} Failed delete\n"
+        else
+            printf "${GREENTICK} Namespace ${REMOTE_PATH} deleted \n"
+        fi
+    fi
+fi
+exit 0

travis-build-scripts/build.sh

@@ -16,16 +16,68 @@
 
 set -e
 
-echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r'
+archive_level_cache_dir="$(mktemp -d)"
-make build-devjmstest
+
-echo -en 'travis_fold:end:build-devjmstest\\r'
+get_archive_level() {
-echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r'
+  local level_path
-make build-devserver
+  local archive_variable
-echo -en 'travis_fold:end:build-devserver\\r'
+  archive_variable="$1"
-if [ "$BUILD_ALL" = true ] ; then
+  MQ_ARCHIVE_LEVEL=""
-    if [ "$ARCH" = "amd64" ] ; then
+  level_path="${archive_level_cache_dir}/${archive_variable}.level"
-        echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r'
+
-        make build-advancedserver
+  if [[ ! -f "$level_path" ]]; then
-        echo -en 'travis_fold:end:build-advancedserver\\r'
+    if [[ -z "${REPOSITORY_USER}" || -z "${REPOSITORY_CREDENTIAL}" ]]; then
+      echo 'Skipping level lookup as repository credentials not set'
+      return
     fi
+    if [[ -z "${!archive_variable}" ]]; then
+      echo "Skipping level lookup as '\$${archive_variable}' is not set"
+      return
+    fi
+    ./travis-build-scripts/artifact-util.sh -f "${!archive_variable}" -u "${REPOSITORY_USER}" -p "${REPOSITORY_CREDENTIAL}" -l "$level_path" -n snapshot --get-property
+  fi
+  read -r MQ_ARCHIVE_LEVEL < "$level_path"
+  export MQ_ARCHIVE_LEVEL
+}
+
+if [ "$TRAVIS_BRANCH" = "$MAIN_BRANCH" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
+  echo 'Retrieving global tagcache' && echo -en 'travis_fold:start:tag-cache-retrieve\\r'
+  ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --check
+  ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get
+  echo -en 'travis_fold:end:tag-cache-retrieve\\r'
+fi
+if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then
+  if [ "$LTS" != true ] ; then
+    echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r'
+    make build-devjmstest
+    echo -en 'travis_fold:end:build-devjmstest\\r'
+    echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r'
+    get_archive_level MQ_ARCHIVE_REPOSITORY_DEV
+    make build-devserver
+    echo -en 'travis_fold:end:build-devserver\\r'
+  fi
+  if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then
+      if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" || "$ARCH" = "ppc64le" ]] ; then
+          echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r'
+          get_archive_level MQ_ARCHIVE_REPOSITORY
+          make build-advancedserver
+          echo -en 'travis_fold:end:build-advancedserver\\r'
+      fi
+  fi
+else
+  echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r'
+  make build-devjmstest
+  echo -en 'travis_fold:end:build-devjmstest\\r'
+
+  if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then
+    echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r'
+    get_archive_level MQ_ARCHIVE_REPOSITORY_DEV
+    make build-devserver
+    echo -en 'travis_fold:end:build-devserver\\r'
+  else
+    echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r'
+    get_archive_level MQ_ARCHIVE_REPOSITORY
+    make build-advancedserver
+    echo -en 'travis_fold:end:build-advancedserver\\r'
+  fi
 fi

travis-build-scripts/cleanup-cache.sh

@@ -1,4 +1,6 @@
-# © Copyright IBM Corporation 2018
+#!/bin/bash
+
+# © Copyright IBM Corporation 2020
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,15 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG BASE_IMAGE=mq-golang-build:9.0.5.0-x86_64-ubuntu-16.04
+echo 'Cleaning up remote cache' && echo -en 'travis_fold:start:cleanup\\r'
-
+./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} --delete
-FROM $BASE_IMAGE
+echo -en 'travis_fold:end:cleanup\\r' 
-
-RUN mkdir -p "$GOPATH/src/github.com/ibm-messaging/mq-golang/samples"
-WORKDIR $GOPATH/src/github.com/ibm-messaging/mq-golang/samples
-
-COPY ./samples/clientconn clientconn
-COPY ./samples/mqitest mqitest
-
-RUN go install ./clientconn  \
-  && go install ./mqitest

travis-build-scripts/create-manifest-list.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# © Copyright IBM Corporation 2020
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage="
+Usage: create-image-manifest.sh -r hyc-mq-container-team-docker-local.artifactory.swg-devops.com -n foo -i ibm-mqadvanced-server-dev -t test -d \"sha256:038ad492532b099c324b897ce9da31ae0be312a1d0063f6456f2e3143cc4f4b8 sha256:754f466cf2cfc5183ac705689ce6720f27fecd07c97970ba3ec48769acba067d\"
+
+Where:
+-r - The image registry hostname
+-n - The image registry namespace
+-i - The image name
+-t - The desired top level manifest tag
+-d - A space separated list of sha256 image digests to be included
+"
+
+GREEN="\033[32m"
+RED="\033[31m"
+BLUE="\033[34m"
+PURPLE="\033[35m"
+AQUA="\033[36m"
+
+END="\033[0m"
+
+UNDERLINE="\033[4m"
+BOLD="\033[1m"
+ITALIC="\033[3m"
+TITLE=${BLUE}${BOLD}${UNDERLINE}
+STEPTITLE=${BLUERIGHTARROW}" "${BOLD}${ITALIC}
+SUBSTEPTITLE=${MINIARROW}${MINIARROW}${MINIARROW}" "${ITALIC}
+RIGHTARROW="\xE2\x96\xB6"
+MINIARROW="\xE2\x96\xBB"
+BLUERIGHTARROW=${BLUE}${RIGHTARROW}${END}
+GREENRIGHTARROW=${GREEN}${RIGHTARROW}${END}
+
+ERROR=${RED}
+
+TICK="\xE2\x9C\x94"
+CROSS="\xE2\x9C\x97"
+GREENTICK=${GREEN}${TICK}${END}
+REDCROSS=${RED}${CROSS}${END}
+
+
+SPACER="\n\n"
+
+while getopts r:n:i:t:d:h:u:p: flag
+do
+    case "${flag}" in
+        r) REGISTRY=${OPTARG};;
+        n) NAMESPACE=${OPTARG};;
+        i) IMAGE=${OPTARG};;
+        t) TAG=${OPTARG};;
+        d) DIGESTS=${OPTARG};;
+        u) USER=${OPTARG};;
+        p) CREDENTIAL=${OPTARG};;
+    esac
+done
+
+if [[ -z $REGISTRY || -z $NAMESPACE || -z $IMAGE || -z $TAG || -z $DIGESTS ]] ; then 
+  printf "${REDCROSS} ${ERROR}Missing parameter!${END}\n"
+  printf "${ERROR}$usage${END}\n"
+  exit 1
+fi
+
+# Docker CLI manifest commands require experimental features to be turned on
+export DOCKER_CLI_EXPERIMENTAL=enabled
+
+MANIFESTS=""
+for digest in $DIGESTS ; do \
+  MANIFESTS+=" $REGISTRY/$NAMESPACE/$IMAGE@$digest"
+done
+
+docker login $REGISTRY -u $USER -p $CREDENTIAL
+docker manifest create $REGISTRY/$NAMESPACE/$IMAGE:$TAG $MANIFESTS
+MANIFEST_DIGEST=$(docker manifest push --purge $REGISTRY/$NAMESPACE/$IMAGE:$TAG)
+
+echo $MANIFEST_DIGEST

travis-build-scripts/global-tag.sh

@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/bash
+
+# © Copyright IBM Corporation 2020
 #
-# Copyright 2018 The Prometheus Authors
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -13,17 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-check_license() {
+set -e
-  local file=""
-  for file in $(find . -type f -iname '*.go' ! -path './vendor/*'); do
-    head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" || echo "  ${file}"
-  done
-}
 
-licRes=$(check_license)
+echo 'Cacheing MQ tag...' && echo -en 'travis_fold:start:build-cache-mq-tag\\r'
-
+make cache-mq-tag
-if [ -n "${licRes}" ]; then
+echo -en 'travis_fold:end:cache-mq-tag\\r'
-  echo "license header checking failed:"
+echo 'Caching tagcache for future stages' && echo -en 'travis_fold:start:tag-cache\\r'
-  echo "${licRes}"
+./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --upload
-  exit 255
+echo -en 'travis_fold:end:tag-cache\\r' 
-fi

travis-build-scripts/install-credential-helper.sh

@@ -19,6 +19,9 @@ sudo add-apt-repository "deb [arch=$ARCH] https://download.docker.com/linux/ubun
 sudo apt update
 sudo apt -y install docker-ce pass
 
+echo "default-cache-ttl 1200" > /home/travis/.gnupg/gpg-agent.conf
+gpg-connect-agent reloadagent /bye
+
 mkdir -p $GOPATH/src/github.com/docker
 cd $GOPATH/src/github.com/docker
 git clone https://github.com/docker/docker-credential-helpers
@@ -27,7 +30,7 @@ make pass
 cp bin/docker-credential-pass $GOPATH/bin/docker-credential-pass
 mkdir -p /home/travis/.docker
 echo '{ "credsStore": "pass" }' | tee /home/travis/.docker/config.json
-gpg --batch --gen-key <<-EOF
+gpg2 --batch --gen-key <<-EOF
 %echo generating a standard key
 Key-Type: DSA
 Key-Length: 1024
@@ -36,13 +39,14 @@ Subkey-Length: 1024
 Name-Real: Travis CI
 Name-Email: travis@osism.io
 Expire-Date: 0
+Passphrase: $REGISTRY_PASS
 %commit
 %echo done
 EOF
-key=$(gpg --no-auto-check-trustdb --list-secret-keys | grep ^sec | cut -d/ -f2 | cut -d" " -f1)
+key=$(gpg2 --list-secret-keys | grep uid -B 1 | head -n 1 | sed 's/^ *//g')
-gpg --export-secret-keys | gpg2 --import -
 pass init $key
 pass insert docker-credential-helpers/docker-pass-initialized-check <<-EOF
 pass is initialized
 pass is initialized
 EOF
+gpg2 --passphrase $REGISTRY_PASS --pinentry-mode=loopback --output doc --decrypt ~/.password-store/docker-credential-helpers/docker-pass-initialized-check.gpg

travis-build-scripts/push.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# © Copyright IBM Corporation 2019, 2020
+# © Copyright IBM Corporation 2019, 2021
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,11 +32,9 @@ function push_developer {
 }
 
 function push_production {
-    if [ "$ARCH" = "amd64" ] ; then
+    echo 'Pushing Production image...' && echo -en 'travis_fold:start:push-advancedserver\\r'
-        echo 'Pushing Production image...' && echo -en 'travis_fold:start:push-advancedserver\\r'
+    make push-advancedserver
-        make push-advancedserver
+    echo -en 'travis_fold:end:push-advancedserver\\r'
-        echo -en 'travis_fold:end:push-advancedserver\\r'
-    fi
 }
 
 # call relevant push function

travis-build-scripts/run.sh

@@ -18,6 +18,18 @@ set -e
 
 if [ "$(uname -m)" = "x86_64" ] ; then export ARCH="amd64" ; else export ARCH=$(uname -m) ; fi
 
+# if DOCKER_USER is set, authenticate with docker.io to mitigate rate limit (https://www.docker.com/increase-rate-limits)
+if [ -n "$DOCKER_USER" ] ; then echo 'Authenticating with docker.io...' && docker login -u $DOCKER_USER -p $DOCKER_PASS docker.io ; fi
+
+if [ "$PUSH_MANIFEST_ONLY" = true ] ; then
+  echo 'Retrieving remote tagcache' && echo -en 'travis_fold:start:retrieve-tag-cache\\r'
+  ./travis-build-scripts/artifact-util.sh -c ${CACHE_PATH} -u ${REPOSITORY_USER} -p ${REPOSITORY_CREDENTIAL} -f cache/${TAGCACHE_FILE} -l ./.tagcache --get
+  echo -en 'travis_fold:end:retrieve-tag-cache\\r'
+  make push-manifest
+  ./travis-build-scripts/cleanup-cache.sh
+  exit 0
+fi
+
 echo 'Downgrading Docker (if necessary)...' && echo -en 'travis_fold:start:docker-downgrade\\r'
 eval "$DOCKER_DOWNGRADE"
 echo -en 'travis_fold:end:docker-downgrade\\r'
@@ -29,7 +41,19 @@ echo -en 'travis_fold:end:docker-downgrade\\r'
 ./travis-build-scripts/test.sh
 
 ## Push images
-if [ "$BUILD_ALL" = true ] ; then
+if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then
+  if [ "$BUILD_ALL" = true ] ; then
     ./travis-build-scripts/push.sh developer
     ./travis-build-scripts/push.sh production
+  fi
+else
+  if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then
+    ./travis-build-scripts/push.sh developer
+  else
+    ./travis-build-scripts/push.sh production
+  fi
+fi
+
+if [ "$LTS" = true ] ; then
+    ./travis-build-scripts/push.sh production
 fi

travis-build-scripts/test.sh

@@ -16,15 +16,29 @@
 
 set -e
 
-echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r'
+if [ -z "$BUILD_INTERNAL_LEVEL" ] ; then
-make test-devserver
+  if [ "$LTS" != true ] ; then
-echo -en 'travis_fold:end:test-devserver\\r'
+    echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r'
-if [ "$BUILD_ALL" = true ] ; then
+    make test-devserver
-    if [ "$ARCH" = "amd64" ] ; then
+    echo -en 'travis_fold:end:test-devserver\\r'
-        echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r'
+  fi
-        make test-advancedserver
+  if [ "$BUILD_ALL" = true ] || [ "$LTS" = true ] ; then
-        echo -en 'travis_fold:end:test-advancedserver\\r'
+      if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" || "$ARCH" = "ppc64le" ]] ; then
-    fi
+          echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r'
+          make test-advancedserver
+          echo -en 'travis_fold:end:test-advancedserver\\r'
+      fi
+  fi
+else
+  if [[ "$BUILD_INTERNAL_LEVEL" == *".DE"* ]]; then
+    echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r'
+    make test-devserver
+    echo -en 'travis_fold:end:test-devserver\\r'
+  else
+    echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r'
+    make test-advancedserver
+    echo -en 'travis_fold:end:test-advancedserver\\r'
+  fi
 fi
 echo 'Running gosec scan...' && echo -en 'travis_fold:start:gosec-scan\\r'
 if [ "$ARCH" = "amd64" ] ; then

vendor/github.com/beorn7/perks/.gitignore

@@ -1,2 +0,0 @@
-*.test
-*.prof

vendor/github.com/beorn7/perks/README.md

@@ -1,31 +0,0 @@
-# Perks for Go (golang.org)
-
-Perks contains the Go package quantile that computes approximate quantiles over
-an unbounded data stream within low memory and CPU bounds.
-
-For more information and examples, see:
-http://godoc.org/github.com/bmizerany/perks
-
-A very special thank you and shout out to Graham Cormode (Rutgers University),
-Flip Korn (AT&T Labs–Research), S. Muthukrishnan (Rutgers University), and
-Divesh Srivastava (AT&T Labs–Research) for their research and publication of
-[Effective Computation of Biased Quantiles over Data Streams](http://www.cs.rutgers.edu/~muthu/bquant.pdf)
-
-Thank you, also:
-* Armon Dadgar (@armon)
-* Andrew Gerrand (@nf)
-* Brad Fitzpatrick (@bradfitz)
-* Keith Rarick (@kr)
-
-FAQ:
-
-Q: Why not move the quantile package into the project root?
-A: I want to add more packages to perks later.
-
-Copyright (C) 2013 Blake Mizerany
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/beorn7/perks/histogram/bench_test.go

@@ -1,26 +0,0 @@
-package histogram
-
-import (
-	"math/rand"
-	"testing"
-)
-
-func BenchmarkInsert10Bins(b *testing.B) {
-	b.StopTimer()
-	h := New(10)
-	b.StartTimer()
-	for i := 0; i < b.N; i++ {
-		f := rand.ExpFloat64()
-		h.Insert(f)
-	}
-}
-
-func BenchmarkInsert100Bins(b *testing.B) {
-	b.StopTimer()
-	h := New(100)
-	b.StartTimer()
-	for i := 0; i < b.N; i++ {
-		f := rand.ExpFloat64()
-		h.Insert(f)
-	}
-}

vendor/github.com/beorn7/perks/histogram/histogram.go

@@ -1,108 +0,0 @@
-// Package histogram provides a Go implementation of BigML's histogram package
-// for Clojure/Java. It is currently experimental.
-package histogram
-
-import (
-	"container/heap"
-	"math"
-	"sort"
-)
-
-type Bin struct {
-	Count int
-	Sum   float64
-}
-
-func (b *Bin) Update(x *Bin) {
-	b.Count += x.Count
-	b.Sum += x.Sum
-}
-
-func (b *Bin) Mean() float64 {
-	return b.Sum / float64(b.Count)
-}
-
-type Bins []*Bin
-
-func (bs Bins) Len() int           { return len(bs) }
-func (bs Bins) Less(i, j int) bool { return bs[i].Mean() < bs[j].Mean() }
-func (bs Bins) Swap(i, j int)      { bs[i], bs[j] = bs[j], bs[i] }
-
-func (bs *Bins) Push(x interface{}) {
-	*bs = append(*bs, x.(*Bin))
-}
-
-func (bs *Bins) Pop() interface{} {
-	return bs.remove(len(*bs) - 1)
-}
-
-func (bs *Bins) remove(n int) *Bin {
-	if n < 0 || len(*bs) < n {
-		return nil
-	}
-	x := (*bs)[n]
-	*bs = append((*bs)[:n], (*bs)[n+1:]...)
-	return x
-}
-
-type Histogram struct {
-	res *reservoir
-}
-
-func New(maxBins int) *Histogram {
-	return &Histogram{res: newReservoir(maxBins)}
-}
-
-func (h *Histogram) Insert(f float64) {
-	h.res.insert(&Bin{1, f})
-	h.res.compress()
-}
-
-func (h *Histogram) Bins() Bins {
-	return h.res.bins
-}
-
-type reservoir struct {
-	n       int
-	maxBins int
-	bins    Bins
-}
-
-func newReservoir(maxBins int) *reservoir {
-	return &reservoir{maxBins: maxBins}
-}
-
-func (r *reservoir) insert(bin *Bin) {
-	r.n += bin.Count
-	i := sort.Search(len(r.bins), func(i int) bool {
-		return r.bins[i].Mean() >= bin.Mean()
-	})
-	if i < 0 || i == r.bins.Len() {
-		// TODO(blake): Maybe use an .insert(i, bin) instead of
-		// performing the extra work of a heap.Push.
-		heap.Push(&r.bins, bin)
-		return
-	}
-	r.bins[i].Update(bin)
-}
-
-func (r *reservoir) compress() {
-	for r.bins.Len() > r.maxBins {
-		minGapIndex := -1
-		minGap := math.MaxFloat64
-		for i := 0; i < r.bins.Len()-1; i++ {
-			gap := gapWeight(r.bins[i], r.bins[i+1])
-			if minGap > gap {
-				minGap = gap
-				minGapIndex = i
-			}
-		}
-		prev := r.bins[minGapIndex]
-		next := r.bins.remove(minGapIndex + 1)
-		prev.Update(next)
-	}
-}
-
-func gapWeight(prev, next *Bin) float64 {
-	return next.Mean() - prev.Mean()
-}

vendor/github.com/beorn7/perks/histogram/histogram_test.go

@@ -1,38 +0,0 @@
-package histogram
-
-import (
-	"math/rand"
-	"testing"
-)
-
-func TestHistogram(t *testing.T) {
-	const numPoints = 1e6
-	const maxBins = 3
-
-	h := New(maxBins)
-	for i := 0; i < numPoints; i++ {
-		f := rand.ExpFloat64()
-		h.Insert(f)
-	}
-
-	bins := h.Bins()
-	if g := len(bins); g > maxBins {
-		t.Fatalf("got %d bins, wanted <= %d", g, maxBins)
-	}
-
-	for _, b := range bins {
-		t.Logf("%+v", b)
-	}
-
-	if g := count(h.Bins()); g != numPoints {
-		t.Fatalf("binned %d points, wanted %d", g, numPoints)
-	}
-}
-
-func count(bins Bins) int {
-	binCounts := 0
-	for _, b := range bins {
-		binCounts += b.Count
-	}
-	return binCounts
-}

vendor/github.com/beorn7/perks/quantile/bench_test.go

@@ -1,63 +0,0 @@
-package quantile
-
-import (
-	"testing"
-)
-
-func BenchmarkInsertTargeted(b *testing.B) {
-	b.ReportAllocs()
-
-	s := NewTargeted(Targets)
-	b.ResetTimer()
-	for i := float64(0); i < float64(b.N); i++ {
-		s.Insert(i)
-	}
-}
-
-func BenchmarkInsertTargetedSmallEpsilon(b *testing.B) {
-	s := NewTargeted(TargetsSmallEpsilon)
-	b.ResetTimer()
-	for i := float64(0); i < float64(b.N); i++ {
-		s.Insert(i)
-	}
-}
-
-func BenchmarkInsertBiased(b *testing.B) {
-	s := NewLowBiased(0.01)
-	b.ResetTimer()
-	for i := float64(0); i < float64(b.N); i++ {
-		s.Insert(i)
-	}
-}
-
-func BenchmarkInsertBiasedSmallEpsilon(b *testing.B) {
-	s := NewLowBiased(0.0001)
-	b.ResetTimer()
-	for i := float64(0); i < float64(b.N); i++ {
-		s.Insert(i)
-	}
-}
-
-func BenchmarkQuery(b *testing.B) {
-	s := NewTargeted(Targets)
-	for i := float64(0); i < 1e6; i++ {
-		s.Insert(i)
-	}
-	b.ResetTimer()
-	n := float64(b.N)
-	for i := float64(0); i < n; i++ {
-		s.Query(i / n)
-	}
-}
-
-func BenchmarkQuerySmallEpsilon(b *testing.B) {
-	s := NewTargeted(TargetsSmallEpsilon)
-	for i := float64(0); i < 1e6; i++ {
-		s.Insert(i)
-	}
-	b.ResetTimer()
-	n := float64(b.N)
-	for i := float64(0); i < n; i++ {
-		s.Query(i / n)
-	}
-}

vendor/github.com/beorn7/perks/quantile/example_test.go

@@ -1,121 +0,0 @@
-// +build go1.1
-
-package quantile_test
-
-import (
-	"bufio"
-	"fmt"
-	"log"
-	"os"
-	"strconv"
-	"time"
-
-	"github.com/beorn7/perks/quantile"
-)
-
-func Example_simple() {
-	ch := make(chan float64)
-	go sendFloats(ch)
-
-	// Compute the 50th, 90th, and 99th percentile.
-	q := quantile.NewTargeted(map[float64]float64{
-		0.50: 0.005,
-		0.90: 0.001,
-		0.99: 0.0001,
-	})
-	for v := range ch {
-		q.Insert(v)
-	}
-
-	fmt.Println("perc50:", q.Query(0.50))
-	fmt.Println("perc90:", q.Query(0.90))
-	fmt.Println("perc99:", q.Query(0.99))
-	fmt.Println("count:", q.Count())
-	// Output:
-	// perc50: 5
-	// perc90: 16
-	// perc99: 223
-	// count: 2388
-}
-
-func Example_mergeMultipleStreams() {
-	// Scenario:
-	// We have multiple database shards. On each shard, there is a process
-	// collecting query response times from the database logs and inserting
-	// them into a Stream (created via NewTargeted(0.90)), much like the
-	// Simple example. These processes expose a network interface for us to
-	// ask them to serialize and send us the results of their
-	// Stream.Samples so we may Merge and Query them.
-	//
-	// NOTES:
-	// * These sample sets are small, allowing us to get them
-	// across the network much faster than sending the entire list of data
-	// points.
-	//
-	// * For this to work correctly, we must supply the same quantiles
-	// a priori the process collecting the samples supplied to NewTargeted,
-	// even if we do not plan to query them all here.
-	ch := make(chan quantile.Samples)
-	getDBQuerySamples(ch)
-	q := quantile.NewTargeted(map[float64]float64{0.90: 0.001})
-	for samples := range ch {
-		q.Merge(samples)
-	}
-	fmt.Println("perc90:", q.Query(0.90))
-}
-
-func Example_window() {
-	// Scenario: We want the 90th, 95th, and 99th percentiles for each
-	// minute.
-
-	ch := make(chan float64)
-	go sendStreamValues(ch)
-
-	tick := time.NewTicker(1 * time.Minute)
-	q := quantile.NewTargeted(map[float64]float64{
-		0.90: 0.001,
-		0.95: 0.0005,
-		0.99: 0.0001,
-	})
-	for {
-		select {
-		case t := <-tick.C:
-			flushToDB(t, q.Samples())
-			q.Reset()
-		case v := <-ch:
-			q.Insert(v)
-		}
-	}
-}
-
-func sendStreamValues(ch chan float64) {
-	// Use your imagination
-}
-
-func flushToDB(t time.Time, samples quantile.Samples) {
-	// Use your imagination
-}
-
-// This is a stub for the above example. In reality this would hit the remote
-// servers via http or something like it.
-func getDBQuerySamples(ch chan quantile.Samples) {}
-
-func sendFloats(ch chan<- float64) {
-	f, err := os.Open("exampledata.txt")
-	if err != nil {
-		log.Fatal(err)
-	}
-	sc := bufio.NewScanner(f)
-	for sc.Scan() {
-		b := sc.Bytes()
-		v, err := strconv.ParseFloat(string(b), 64)
-		if err != nil {
-			log.Fatal(err)
-		}
-		ch <- v
-	}
-	if sc.Err() != nil {
-		log.Fatal(sc.Err())
-	}
-	close(ch)
-}

vendor/github.com/beorn7/perks/quantile/stream_test.go

@@ -1,215 +0,0 @@
-package quantile
-
-import (
-	"math"
-	"math/rand"
-	"sort"
-	"testing"
-)
-
-var (
-	Targets = map[float64]float64{
-		0.01: 0.001,
-		0.10: 0.01,
-		0.50: 0.05,
-		0.90: 0.01,
-		0.99: 0.001,
-	}
-	TargetsSmallEpsilon = map[float64]float64{
-		0.01: 0.0001,
-		0.10: 0.001,
-		0.50: 0.005,
-		0.90: 0.001,
-		0.99: 0.0001,
-	}
-	LowQuantiles  = []float64{0.01, 0.1, 0.5}
-	HighQuantiles = []float64{0.99, 0.9, 0.5}
-)
-
-const RelativeEpsilon = 0.01
-
-func verifyPercsWithAbsoluteEpsilon(t *testing.T, a []float64, s *Stream) {
-	sort.Float64s(a)
-	for quantile, epsilon := range Targets {
-		n := float64(len(a))
-		k := int(quantile * n)
-		if k < 1 {
-			k = 1
-		}
-		lower := int((quantile - epsilon) * n)
-		if lower < 1 {
-			lower = 1
-		}
-		upper := int(math.Ceil((quantile + epsilon) * n))
-		if upper > len(a) {
-			upper = len(a)
-		}
-		w, min, max := a[k-1], a[lower-1], a[upper-1]
-		if g := s.Query(quantile); g < min || g > max {
-			t.Errorf("q=%f: want %v [%f,%f], got %v", quantile, w, min, max, g)
-		}
-	}
-}
-
-func verifyLowPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
-	sort.Float64s(a)
-	for _, qu := range LowQuantiles {
-		n := float64(len(a))
-		k := int(qu * n)
-
-		lowerRank := int((1 - RelativeEpsilon) * qu * n)
-		upperRank := int(math.Ceil((1 + RelativeEpsilon) * qu * n))
-		w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
-		if g := s.Query(qu); g < min || g > max {
-			t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
-		}
-	}
-}
-
-func verifyHighPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
-	sort.Float64s(a)
-	for _, qu := range HighQuantiles {
-		n := float64(len(a))
-		k := int(qu * n)
-
-		lowerRank := int((1 - (1+RelativeEpsilon)*(1-qu)) * n)
-		upperRank := int(math.Ceil((1 - (1-RelativeEpsilon)*(1-qu)) * n))
-		w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
-		if g := s.Query(qu); g < min || g > max {
-			t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
-		}
-	}
-}
-
-func populateStream(s *Stream) []float64 {
-	a := make([]float64, 0, 1e5+100)
-	for i := 0; i < cap(a); i++ {
-		v := rand.NormFloat64()
-		// Add 5% asymmetric outliers.
-		if i%20 == 0 {
-			v = v*v + 1
-		}
-		s.Insert(v)
-		a = append(a, v)
-	}
-	return a
-}
-
-func TestTargetedQuery(t *testing.T) {
-	rand.Seed(42)
-	s := NewTargeted(Targets)
-	a := populateStream(s)
-	verifyPercsWithAbsoluteEpsilon(t, a, s)
-}
-
-func TestTargetedQuerySmallSampleSize(t *testing.T) {
-	rand.Seed(42)
-	s := NewTargeted(TargetsSmallEpsilon)
-	a := []float64{1, 2, 3, 4, 5}
-	for _, v := range a {
-		s.Insert(v)
-	}
-	verifyPercsWithAbsoluteEpsilon(t, a, s)
-	// If not yet flushed, results should be precise:
-	if !s.flushed() {
-		for φ, want := range map[float64]float64{
-			0.01: 1,
-			0.10: 1,
-			0.50: 3,
-			0.90: 5,
-			0.99: 5,
-		} {
-			if got := s.Query(φ); got != want {
-				t.Errorf("want %f for φ=%f, got %f", want, φ, got)
-			}
-		}
-	}
-}
-
-func TestLowBiasedQuery(t *testing.T) {
-	rand.Seed(42)
-	s := NewLowBiased(RelativeEpsilon)
-	a := populateStream(s)
-	verifyLowPercsWithRelativeEpsilon(t, a, s)
-}
-
-func TestHighBiasedQuery(t *testing.T) {
-	rand.Seed(42)
-	s := NewHighBiased(RelativeEpsilon)
-	a := populateStream(s)
-	verifyHighPercsWithRelativeEpsilon(t, a, s)
-}
-
-// BrokenTestTargetedMerge is broken, see Merge doc comment.
-func BrokenTestTargetedMerge(t *testing.T) {
-	rand.Seed(42)
-	s1 := NewTargeted(Targets)
-	s2 := NewTargeted(Targets)
-	a := populateStream(s1)
-	a = append(a, populateStream(s2)...)
-	s1.Merge(s2.Samples())
-	verifyPercsWithAbsoluteEpsilon(t, a, s1)
-}
-
-// BrokenTestLowBiasedMerge is broken, see Merge doc comment.
-func BrokenTestLowBiasedMerge(t *testing.T) {
-	rand.Seed(42)
-	s1 := NewLowBiased(RelativeEpsilon)
-	s2 := NewLowBiased(RelativeEpsilon)
-	a := populateStream(s1)
-	a = append(a, populateStream(s2)...)
-	s1.Merge(s2.Samples())
-	verifyLowPercsWithRelativeEpsilon(t, a, s2)
-}
-
-// BrokenTestHighBiasedMerge is broken, see Merge doc comment.
-func BrokenTestHighBiasedMerge(t *testing.T) {
-	rand.Seed(42)
-	s1 := NewHighBiased(RelativeEpsilon)
-	s2 := NewHighBiased(RelativeEpsilon)
-	a := populateStream(s1)
-	a = append(a, populateStream(s2)...)
-	s1.Merge(s2.Samples())
-	verifyHighPercsWithRelativeEpsilon(t, a, s2)
-}
-
-func TestUncompressed(t *testing.T) {
-	q := NewTargeted(Targets)
-	for i := 100; i > 0; i-- {
-		q.Insert(float64(i))
-	}
-	if g := q.Count(); g != 100 {
-		t.Errorf("want count 100, got %d", g)
-	}
-	// Before compression, Query should have 100% accuracy.
-	for quantile := range Targets {
-		w := quantile * 100
-		if g := q.Query(quantile); g != w {
-			t.Errorf("want %f, got %f", w, g)
-		}
-	}
-}
-
-func TestUncompressedSamples(t *testing.T) {
-	q := NewTargeted(map[float64]float64{0.99: 0.001})
-	for i := 1; i <= 100; i++ {
-		q.Insert(float64(i))
-	}
-	if g := q.Samples().Len(); g != 100 {
-		t.Errorf("want count 100, got %d", g)
-	}
-}
-
-func TestUncompressedOne(t *testing.T) {
-	q := NewTargeted(map[float64]float64{0.99: 0.01})
-	q.Insert(3.14)
-	if g := q.Query(0.90); g != 3.14 {
-		t.Error("want PI, got", g)
-	}
-}
-
-func TestDefaults(t *testing.T) {
-	if g := NewTargeted(map[float64]float64{0.99: 0.001}).Query(0.99); g != 0 {
-		t.Errorf("want 0, got %f", g)
-	}
-}

vendor/github.com/beorn7/perks/topk/topk.go

@@ -1,90 +0,0 @@
-package topk
-
-import (
-	"sort"
-)
-
-// http://www.cs.ucsb.edu/research/tech_reports/reports/2005-23.pdf
-
-type Element struct {
-	Value string
-	Count int
-}
-
-type Samples []*Element
-
-func (sm Samples) Len() int {
-	return len(sm)
-}
-
-func (sm Samples) Less(i, j int) bool {
-	return sm[i].Count < sm[j].Count
-}
-
-func (sm Samples) Swap(i, j int) {
-	sm[i], sm[j] = sm[j], sm[i]
-}
-
-type Stream struct {
-	k   int
-	mon map[string]*Element
-
-	// the minimum Element
-	min *Element
-}
-
-func New(k int) *Stream {
-	s := new(Stream)
-	s.k = k
-	s.mon = make(map[string]*Element)
-	s.min = &Element{}
-
-	// Track k+1 so that less frequenet items contended for that spot,
-	// resulting in k being more accurate.
-	return s
-}
-
-func (s *Stream) Insert(x string) {
-	s.insert(&Element{x, 1})
-}
-
-func (s *Stream) Merge(sm Samples) {
-	for _, e := range sm {
-		s.insert(e)
-	}
-}
-
-func (s *Stream) insert(in *Element) {
-	e := s.mon[in.Value]
-	if e != nil {
-		e.Count++
-	} else {
-		if len(s.mon) < s.k+1 {
-			e = &Element{in.Value, in.Count}
-			s.mon[in.Value] = e
-		} else {
-			e = s.min
-			delete(s.mon, e.Value)
-			e.Value = in.Value
-			e.Count += in.Count
-			s.min = e
-		}
-	}
-	if e.Count < s.min.Count {
-		s.min = e
-	}
-}
-
-func (s *Stream) Query() Samples {
-	var sm Samples
-	for _, e := range s.mon {
-		sm = append(sm, e)
-	}
-	sort.Sort(sort.Reverse(sm))
-
-	if len(sm) < s.k {
-		return sm
-	}
-
-	return sm[:s.k]
-}

vendor/github.com/beorn7/perks/topk/topk_test.go

@@ -1,57 +0,0 @@
-package topk
-
-import (
-	"fmt"
-	"math/rand"
-	"sort"
-	"testing"
-)
-
-func TestTopK(t *testing.T) {
-	stream := New(10)
-	ss := []*Stream{New(10), New(10), New(10)}
-	m := make(map[string]int)
-	for _, s := range ss {
-		for i := 0; i < 1e6; i++ {
-			v := fmt.Sprintf("%x", int8(rand.ExpFloat64()))
-			s.Insert(v)
-			m[v]++
-		}
-		stream.Merge(s.Query())
-	}
-
-	var sm Samples
-	for x, s := range m {
-		sm = append(sm, &Element{x, s})
-	}
-	sort.Sort(sort.Reverse(sm))
-
-	g := stream.Query()
-	if len(g) != 10 {
-		t.Fatalf("got %d, want 10", len(g))
-	}
-	for i, e := range g {
-		if sm[i].Value != e.Value {
-			t.Errorf("at %d: want %q, got %q", i, sm[i].Value, e.Value)
-		}
-	}
-}
-
-func TestQuery(t *testing.T) {
-	queryTests := []struct {
-		value string
-		expected int
-	}{
-		{"a", 1},
-		{"b", 2},
-		{"c", 2},
-	}
-
-	stream := New(2)
-	for _, tt := range queryTests {
-		stream.Insert(tt.value)
-		if n := len(stream.Query()); n != tt.expected {
-			t.Errorf("want %d, got %d", tt.expected, n)
-		}
-	}
-}

vendor/github.com/cespare/xxhash/v2/.travis.yml

@@ -0,0 +1,8 @@
+language: go
+go:
+  - "1.x"
+  - master
+env:
+  - TAGS=""
+  - TAGS="-tags purego"
+script: go test $TAGS -v ./...

vendor/github.com/cespare/xxhash/v2/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2016 Caleb Spare
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/cespare/xxhash/v2/README.md

@@ -0,0 +1,67 @@
+# xxhash
+
+[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash)
+[![Build Status](https://travis-ci.org/cespare/xxhash.svg?branch=master)](https://travis-ci.org/cespare/xxhash)
+
+xxhash is a Go implementation of the 64-bit
+[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
+high-quality hashing algorithm that is much faster than anything in the Go
+standard library.
+
+This package provides a straightforward API:
+
+```
+func Sum64(b []byte) uint64
+func Sum64String(s string) uint64
+type Digest struct{ ... }
+    func New() *Digest
+```
+
+The `Digest` type implements hash.Hash64. Its key methods are:
+
+```
+func (*Digest) Write([]byte) (int, error)
+func (*Digest) WriteString(string) (int, error)
+func (*Digest) Sum64() uint64
+```
+
+This implementation provides a fast pure-Go implementation and an even faster
+assembly implementation for amd64.
+
+## Compatibility
+
+This package is in a module and the latest code is in version 2 of the module.
+You need a version of Go with at least "minimal module compatibility" to use
+github.com/cespare/xxhash/v2:
+
+* 1.9.7+ for Go 1.9
+* 1.10.3+ for Go 1.10
+* Go 1.11 or later
+
+I recommend using the latest release of Go.
+
+## Benchmarks
+
+Here are some quick benchmarks comparing the pure-Go and assembly
+implementations of Sum64.
+
+| input size | purego | asm |
+| --- | --- | --- |
+| 5 B   |  979.66 MB/s |  1291.17 MB/s  |
+| 100 B | 7475.26 MB/s | 7973.40 MB/s  |
+| 4 KB  | 17573.46 MB/s | 17602.65 MB/s |
+| 10 MB | 17131.46 MB/s | 17142.16 MB/s |
+
+These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using
+the following commands under Go 1.11.2:
+
+```
+$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes'
+$ go test -benchtime 10s -bench '/xxhash,direct,bytes'
+```
+
+## Projects using this package
+
+- [InfluxDB](https://github.com/influxdata/influxdb)
+- [Prometheus](https://github.com/prometheus/prometheus)
+- [FreeCache](https://github.com/coocood/freecache)

vendor/github.com/cespare/xxhash/v2/go.mod

@@ -0,0 +1,3 @@
+module github.com/cespare/xxhash/v2
+
+go 1.11

vendor/github.com/cespare/xxhash/v2/xxhash.go

@@ -0,0 +1,236 @@
+// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
+// at http://cyan4973.github.io/xxHash/.
+package xxhash
+
+import (
+	"encoding/binary"
+	"errors"
+	"math/bits"
+)
+
+const (
+	prime1 uint64 = 11400714785074694791
+	prime2 uint64 = 14029467366897019727
+	prime3 uint64 = 1609587929392839161
+	prime4 uint64 = 9650029242287828579
+	prime5 uint64 = 2870177450012600261
+)
+
+// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
+// possible in the Go code is worth a small (but measurable) performance boost
+// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
+// convenience in the Go code in a few places where we need to intentionally
+// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
+// result overflows a uint64).
+var (
+	prime1v = prime1
+	prime2v = prime2
+	prime3v = prime3
+	prime4v = prime4
+	prime5v = prime5
+)
+
+// Digest implements hash.Hash64.
+type Digest struct {
+	v1    uint64
+	v2    uint64
+	v3    uint64
+	v4    uint64
+	total uint64
+	mem   [32]byte
+	n     int // how much of mem is used
+}
+
+// New creates a new Digest that computes the 64-bit xxHash algorithm.
+func New() *Digest {
+	var d Digest
+	d.Reset()
+	return &d
+}
+
+// Reset clears the Digest's state so that it can be reused.
+func (d *Digest) Reset() {
+	d.v1 = prime1v + prime2
+	d.v2 = prime2
+	d.v3 = 0
+	d.v4 = -prime1v
+	d.total = 0
+	d.n = 0
+}
+
+// Size always returns 8 bytes.
+func (d *Digest) Size() int { return 8 }
+
+// BlockSize always returns 32 bytes.
+func (d *Digest) BlockSize() int { return 32 }
+
+// Write adds more data to d. It always returns len(b), nil.
+func (d *Digest) Write(b []byte) (n int, err error) {
+	n = len(b)
+	d.total += uint64(n)
+
+	if d.n+n < 32 {
+		// This new data doesn't even fill the current block.
+		copy(d.mem[d.n:], b)
+		d.n += n
+		return
+	}
+
+	if d.n > 0 {
+		// Finish off the partial block.
+		copy(d.mem[d.n:], b)
+		d.v1 = round(d.v1, u64(d.mem[0:8]))
+		d.v2 = round(d.v2, u64(d.mem[8:16]))
+		d.v3 = round(d.v3, u64(d.mem[16:24]))
+		d.v4 = round(d.v4, u64(d.mem[24:32]))
+		b = b[32-d.n:]
+		d.n = 0
+	}
+
+	if len(b) >= 32 {
+		// One or more full blocks left.
+		nw := writeBlocks(d, b)
+		b = b[nw:]
+	}
+
+	// Store any remaining partial block.
+	copy(d.mem[:], b)
+	d.n = len(b)
+
+	return
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+func (d *Digest) Sum(b []byte) []byte {
+	s := d.Sum64()
+	return append(
+		b,
+		byte(s>>56),
+		byte(s>>48),
+		byte(s>>40),
+		byte(s>>32),
+		byte(s>>24),
+		byte(s>>16),
+		byte(s>>8),
+		byte(s),
+	)
+}
+
+// Sum64 returns the current hash.
+func (d *Digest) Sum64() uint64 {
+	var h uint64
+
+	if d.total >= 32 {
+		v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4
+		h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
+		h = mergeRound(h, v1)
+		h = mergeRound(h, v2)
+		h = mergeRound(h, v3)
+		h = mergeRound(h, v4)
+	} else {
+		h = d.v3 + prime5
+	}
+
+	h += d.total
+
+	i, end := 0, d.n
+	for ; i+8 <= end; i += 8 {
+		k1 := round(0, u64(d.mem[i:i+8]))
+		h ^= k1
+		h = rol27(h)*prime1 + prime4
+	}
+	if i+4 <= end {
+		h ^= uint64(u32(d.mem[i:i+4])) * prime1
+		h = rol23(h)*prime2 + prime3
+		i += 4
+	}
+	for i < end {
+		h ^= uint64(d.mem[i]) * prime5
+		h = rol11(h) * prime1
+		i++
+	}
+
+	h ^= h >> 33
+	h *= prime2
+	h ^= h >> 29
+	h *= prime3
+	h ^= h >> 32
+
+	return h
+}
+
+const (
+	magic         = "xxh\x06"
+	marshaledSize = len(magic) + 8*5 + 32
+)
+
+// MarshalBinary implements the encoding.BinaryMarshaler interface.
+func (d *Digest) MarshalBinary() ([]byte, error) {
+	b := make([]byte, 0, marshaledSize)
+	b = append(b, magic...)
+	b = appendUint64(b, d.v1)
+	b = appendUint64(b, d.v2)
+	b = appendUint64(b, d.v3)
+	b = appendUint64(b, d.v4)
+	b = appendUint64(b, d.total)
+	b = append(b, d.mem[:d.n]...)
+	b = b[:len(b)+len(d.mem)-d.n]
+	return b, nil
+}
+
+// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
+func (d *Digest) UnmarshalBinary(b []byte) error {
+	if len(b) < len(magic) || string(b[:len(magic)]) != magic {
+		return errors.New("xxhash: invalid hash state identifier")
+	}
+	if len(b) != marshaledSize {
+		return errors.New("xxhash: invalid hash state size")
+	}
+	b = b[len(magic):]
+	b, d.v1 = consumeUint64(b)
+	b, d.v2 = consumeUint64(b)
+	b, d.v3 = consumeUint64(b)
+	b, d.v4 = consumeUint64(b)
+	b, d.total = consumeUint64(b)
+	copy(d.mem[:], b)
+	b = b[len(d.mem):]
+	d.n = int(d.total % uint64(len(d.mem)))
+	return nil
+}
+
+func appendUint64(b []byte, x uint64) []byte {
+	var a [8]byte
+	binary.LittleEndian.PutUint64(a[:], x)
+	return append(b, a[:]...)
+}
+
+func consumeUint64(b []byte) ([]byte, uint64) {
+	x := u64(b)
+	return b[8:], x
+}
+
+func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) }
+func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) }
+
+func round(acc, input uint64) uint64 {
+	acc += input * prime2
+	acc = rol31(acc)
+	acc *= prime1
+	return acc
+}
+
+func mergeRound(acc, val uint64) uint64 {
+	val = round(0, val)
+	acc ^= val
+	acc = acc*prime1 + prime4
+	return acc
+}
+
+func rol1(x uint64) uint64  { return bits.RotateLeft64(x, 1) }
+func rol7(x uint64) uint64  { return bits.RotateLeft64(x, 7) }
+func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) }
+func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) }
+func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) }
+func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) }
+func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) }
+func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) }

vendor/github.com/cespare/xxhash/v2/xxhash_amd64.go

@@ -0,0 +1,13 @@
+// +build !appengine
+// +build gc
+// +build !purego
+
+package xxhash
+
+// Sum64 computes the 64-bit xxHash digest of b.
+//
+//go:noescape
+func Sum64(b []byte) uint64
+
+//go:noescape
+func writeBlocks(d *Digest, b []byte) int

vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s

@@ -0,0 +1,215 @@
+// +build !appengine
+// +build gc
+// +build !purego
+
+#include "textflag.h"
+
+// Register allocation:
+// AX	h
+// CX	pointer to advance through b
+// DX	n
+// BX	loop end
+// R8	v1, k1
+// R9	v2
+// R10	v3
+// R11	v4
+// R12	tmp
+// R13	prime1v
+// R14	prime2v
+// R15	prime4v
+
+// round reads from and advances the buffer pointer in CX.
+// It assumes that R13 has prime1v and R14 has prime2v.
+#define round(r) \
+	MOVQ  (CX), R12 \
+	ADDQ  $8, CX    \
+	IMULQ R14, R12  \
+	ADDQ  R12, r    \
+	ROLQ  $31, r    \
+	IMULQ R13, r
+
+// mergeRound applies a merge round on the two registers acc and val.
+// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v.
+#define mergeRound(acc, val) \
+	IMULQ R14, val \
+	ROLQ  $31, val \
+	IMULQ R13, val \
+	XORQ  val, acc \
+	IMULQ R13, acc \
+	ADDQ  R15, acc
+
+// func Sum64(b []byte) uint64
+TEXT ·Sum64(SB), NOSPLIT, $0-32
+	// Load fixed primes.
+	MOVQ ·prime1v(SB), R13
+	MOVQ ·prime2v(SB), R14
+	MOVQ ·prime4v(SB), R15
+
+	// Load slice.
+	MOVQ b_base+0(FP), CX
+	MOVQ b_len+8(FP), DX
+	LEAQ (CX)(DX*1), BX
+
+	// The first loop limit will be len(b)-32.
+	SUBQ $32, BX
+
+	// Check whether we have at least one block.
+	CMPQ DX, $32
+	JLT  noBlocks
+
+	// Set up initial state (v1, v2, v3, v4).
+	MOVQ R13, R8
+	ADDQ R14, R8
+	MOVQ R14, R9
+	XORQ R10, R10
+	XORQ R11, R11
+	SUBQ R13, R11
+
+	// Loop until CX > BX.
+blockLoop:
+	round(R8)
+	round(R9)
+	round(R10)
+	round(R11)
+
+	CMPQ CX, BX
+	JLE  blockLoop
+
+	MOVQ R8, AX
+	ROLQ $1, AX
+	MOVQ R9, R12
+	ROLQ $7, R12
+	ADDQ R12, AX
+	MOVQ R10, R12
+	ROLQ $12, R12
+	ADDQ R12, AX
+	MOVQ R11, R12
+	ROLQ $18, R12
+	ADDQ R12, AX
+
+	mergeRound(AX, R8)
+	mergeRound(AX, R9)
+	mergeRound(AX, R10)
+	mergeRound(AX, R11)
+
+	JMP afterBlocks
+
+noBlocks:
+	MOVQ ·prime5v(SB), AX
+
+afterBlocks:
+	ADDQ DX, AX
+
+	// Right now BX has len(b)-32, and we want to loop until CX > len(b)-8.
+	ADDQ $24, BX
+
+	CMPQ CX, BX
+	JG   fourByte
+
+wordLoop:
+	// Calculate k1.
+	MOVQ  (CX), R8
+	ADDQ  $8, CX
+	IMULQ R14, R8
+	ROLQ  $31, R8
+	IMULQ R13, R8
+
+	XORQ  R8, AX
+	ROLQ  $27, AX
+	IMULQ R13, AX
+	ADDQ  R15, AX
+
+	CMPQ CX, BX
+	JLE  wordLoop
+
+fourByte:
+	ADDQ $4, BX
+	CMPQ CX, BX
+	JG   singles
+
+	MOVL  (CX), R8
+	ADDQ  $4, CX
+	IMULQ R13, R8
+	XORQ  R8, AX
+
+	ROLQ  $23, AX
+	IMULQ R14, AX
+	ADDQ  ·prime3v(SB), AX
+
+singles:
+	ADDQ $4, BX
+	CMPQ CX, BX
+	JGE  finalize
+
+singlesLoop:
+	MOVBQZX (CX), R12
+	ADDQ    $1, CX
+	IMULQ   ·prime5v(SB), R12
+	XORQ    R12, AX
+
+	ROLQ  $11, AX
+	IMULQ R13, AX
+
+	CMPQ CX, BX
+	JL   singlesLoop
+
+finalize:
+	MOVQ  AX, R12
+	SHRQ  $33, R12
+	XORQ  R12, AX
+	IMULQ R14, AX
+	MOVQ  AX, R12
+	SHRQ  $29, R12
+	XORQ  R12, AX
+	IMULQ ·prime3v(SB), AX
+	MOVQ  AX, R12
+	SHRQ  $32, R12
+	XORQ  R12, AX
+
+	MOVQ AX, ret+24(FP)
+	RET
+
+// writeBlocks uses the same registers as above except that it uses AX to store
+// the d pointer.
+
+// func writeBlocks(d *Digest, b []byte) int
+TEXT ·writeBlocks(SB), NOSPLIT, $0-40
+	// Load fixed primes needed for round.
+	MOVQ ·prime1v(SB), R13
+	MOVQ ·prime2v(SB), R14
+
+	// Load slice.
+	MOVQ b_base+8(FP), CX
+	MOVQ b_len+16(FP), DX
+	LEAQ (CX)(DX*1), BX
+	SUBQ $32, BX
+
+	// Load vN from d.
+	MOVQ d+0(FP), AX
+	MOVQ 0(AX), R8   // v1
+	MOVQ 8(AX), R9   // v2
+	MOVQ 16(AX), R10 // v3
+	MOVQ 24(AX), R11 // v4
+
+	// We don't need to check the loop condition here; this function is
+	// always called with at least one block of data to process.
+blockLoop:
+	round(R8)
+	round(R9)
+	round(R10)
+	round(R11)
+
+	CMPQ CX, BX
+	JLE  blockLoop
+
+	// Copy vN back to d.
+	MOVQ R8, 0(AX)
+	MOVQ R9, 8(AX)
+	MOVQ R10, 16(AX)
+	MOVQ R11, 24(AX)
+
+	// The number of bytes written is CX minus the old base pointer.
+	SUBQ b_base+8(FP), CX
+	MOVQ CX, ret+32(FP)
+
+	RET

vendor/github.com/cespare/xxhash/v2/xxhash_other.go

@@ -0,0 +1,76 @@
+// +build !amd64 appengine !gc purego
+
+package xxhash
+
+// Sum64 computes the 64-bit xxHash digest of b.
+func Sum64(b []byte) uint64 {
+	// A simpler version would be
+	//   d := New()
+	//   d.Write(b)
+	//   return d.Sum64()
+	// but this is faster, particularly for small inputs.
+
+	n := len(b)
+	var h uint64
+
+	if n >= 32 {
+		v1 := prime1v + prime2
+		v2 := prime2
+		v3 := uint64(0)
+		v4 := -prime1v
+		for len(b) >= 32 {
+			v1 = round(v1, u64(b[0:8:len(b)]))
+			v2 = round(v2, u64(b[8:16:len(b)]))
+			v3 = round(v3, u64(b[16:24:len(b)]))
+			v4 = round(v4, u64(b[24:32:len(b)]))
+			b = b[32:len(b):len(b)]
+		}
+		h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
+		h = mergeRound(h, v1)
+		h = mergeRound(h, v2)
+		h = mergeRound(h, v3)
+		h = mergeRound(h, v4)
+	} else {
+		h = prime5
+	}
+
+	h += uint64(n)
+
+	i, end := 0, len(b)
+	for ; i+8 <= end; i += 8 {
+		k1 := round(0, u64(b[i:i+8:len(b)]))
+		h ^= k1
+		h = rol27(h)*prime1 + prime4
+	}
+	if i+4 <= end {
+		h ^= uint64(u32(b[i:i+4:len(b)])) * prime1
+		h = rol23(h)*prime2 + prime3
+		i += 4
+	}
+	for ; i < end; i++ {
+		h ^= uint64(b[i]) * prime5
+		h = rol11(h) * prime1
+	}
+
+	h ^= h >> 33
+	h *= prime2
+	h ^= h >> 29
+	h *= prime3
+	h ^= h >> 32
+
+	return h
+}
+
+func writeBlocks(d *Digest, b []byte) int {
+	v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4
+	n := len(b)
+	for len(b) >= 32 {
+		v1 = round(v1, u64(b[0:8:len(b)]))
+		v2 = round(v2, u64(b[8:16:len(b)]))
+		v3 = round(v3, u64(b[16:24:len(b)]))
+		v4 = round(v4, u64(b[24:32:len(b)]))
+		b = b[32:len(b):len(b)]
+	}
+	d.v1, d.v2, d.v3, d.v4 = v1, v2, v3, v4
+	return n - len(b)
+}

vendor/github.com/cespare/xxhash/v2/xxhash_safe.go

@@ -0,0 +1,15 @@
+// +build appengine
+
+// This file contains the safe implementations of otherwise unsafe-using code.
+
+package xxhash
+
+// Sum64String computes the 64-bit xxHash digest of s.
+func Sum64String(s string) uint64 {
+	return Sum64([]byte(s))
+}
+
+// WriteString adds more data to d. It always returns len(s), nil.
+func (d *Digest) WriteString(s string) (n int, err error) {
+	return d.Write([]byte(s))
+}

vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go

@@ -0,0 +1,46 @@
+// +build !appengine
+
+// This file encapsulates usage of unsafe.
+// xxhash_safe.go contains the safe implementations.
+
+package xxhash
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+// Notes:
+//
+// See https://groups.google.com/d/msg/golang-nuts/dcjzJy-bSpw/tcZYBzQqAQAJ
+// for some discussion about these unsafe conversions.
+//
+// In the future it's possible that compiler optimizations will make these
+// unsafe operations unnecessary: https://golang.org/issue/2205.
+//
+// Both of these wrapper functions still incur function call overhead since they
+// will not be inlined. We could write Go/asm copies of Sum64 and Digest.Write
+// for strings to squeeze out a bit more speed. Mid-stack inlining should
+// eventually fix this.
+
+// Sum64String computes the 64-bit xxHash digest of s.
+// It may be faster than Sum64([]byte(s)) by avoiding a copy.
+func Sum64String(s string) uint64 {
+	var b []byte
+	bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
+	bh.Len = len(s)
+	bh.Cap = len(s)
+	return Sum64(b)
+}
+
+// WriteString adds more data to d. It always returns len(s), nil.
+// It may be faster than Write([]byte(s)) by avoiding a copy.
+func (d *Digest) WriteString(s string) (n int, err error) {
+	var b []byte
+	bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
+	bh.Len = len(s)
+	bh.Cap = len(s)
+	return d.Write(b)
+}