Merge pull request #273 from arthurbarr/make

Fixes for RHEL build

Makefile-RHEL

@@ -23,7 +23,7 @@ MQ_VERSION ?= 9.1.1.0
 # MQ_ARCHIVE is the name of the file, under the downloads directory, from which MQ Advanced can
 # be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture
 # Does not apply to MQ Advanced for Developers.
-MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION)_LINUX_$(MQ_ARCHIVE_ARCH).tar.gz
+MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION_VRM)_LINUX_$(MQ_ARCHIVE_ARCH).tar.gz
 # MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced
 # for Developers can be installed
 MQ_ARCHIVE_DEV ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION))
@@ -56,6 +56,10 @@ DEV_JMS_IMAGE=mq-dev-jms-test:latest
 IMAGE_REVISION=$(shell git rev-parse HEAD)
 IMAGE_SOURCE=$(shell git config --get remote.origin.url)
 MQDEV=
+EMPTY:=
+SPACE:= $(EMPTY) $(EMPTY)
+# MQ_VERSION_VRM is MQ_VERSION with only the Version, Release and Modifier fields (no Fix field).  e.g. 9.1.1 instead of 9.1.1.0
+MQ_VERSION_VRM=$(subst $(SPACE),.,$(wordlist 1,3,$(subst .,$(SPACE),$(MQ_VERSION))))
 
 
 ifneq (,$(findstring Microsoft,$(shell uname -r)))
@@ -155,7 +159,7 @@ test-devserver: check-test-prereqs test/docker/vendor
 
 
 .PHONY: build-advancedserver
-build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs-ex
+build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs
 	$(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END)))
 	sudo mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_ADVANCEDSERVER)" "$(MQ_VERSION)" "$(MQDEV)"
 
@@ -163,25 +167,19 @@ build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs-ex
 .PHONY: build-devserver
 build-devserver: MQDEV=TRUE
 build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm
-build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs-ex
+build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs
 	$(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER)"$(END)))
 	sudo mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE_DEV)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_VERSION)" "$(MQDEV)"
 	sudo mq-advanced-server-rhel/mqdev-buildah.sh "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_IMAGE_DEVSERVER)" "$(MQ_VERSION)"
 
 
 .PHONY: build-mqgolang-sdk
-build-mqgolang-sdk: check-prereqs downloads/$(MQ_SDK_ARCHIVE) build-mqgolang-sdk-ex
+build-mqgolang-sdk: check-prereqs downloads/$(MQ_SDK_ARCHIVE)
-
-.PHONY: build-mqgolang-sdk-ex
-build-mqgolang-sdk-ex: 
 	$(info $(SPACER)$(shell printf $(TITLE)"Build mq-golang SDK"$(END)))
 	sudo mq-advanced-server-rhel/mq-golang-sdk-buildah.sh "$(MQ_SDK_ARCHIVE)" "$(MQ_IMAGE_GOLANG_SDK)"
 
 .PHONY: build-go-programs
-build-go-programs: check-prereqs downloads/$(MQ_SDK_ARCHIVE) build-go-programs-ex
+build-go-programs: check-prereqs downloads/$(MQ_SDK_ARCHIVE) build-mqgolang-sdk
-
-.PHONY: build-go-programs-ex
-build-go-programs-ex: build-mqgolang-sdk-ex
 	$(info $(SPACER)$(shell printf $(TITLE)"Build go programs"$(END)))
 	IMAGE_REVISION=$(IMAGE_REVISION) IMAGE_SOURCE=$(IMAGE_SOURCE) sudo mq-advanced-server-rhel/go-buildah.sh "$(MQ_IMAGE_GOLANG_SDK)" "$(MQDEV)"
 
@@ -192,4 +190,15 @@ build-devjmstest: check-test-prereqs
 	sudo buildah push $(DEV_JMS_IMAGE) docker-daemon:$(DEV_JMS_IMAGE)
 	docker tag docker.io/$(DEV_JMS_IMAGE) $(DEV_JMS_IMAGE)
 
+.PHONY: debug-vars
+debug-vars:
+	@echo MQ_VERSION=$(MQ_VERSION)
+	@echo MQ_VERSION_VRM=$(MQ_VERSION_VRM)
+	@echo MQ_ARCHIVE=$(MQ_ARCHIVE)
+	@echo MQ_SDK_ARCHIVE=$(MQ_SDK_ARCHIVE)
+	@echo MQ_IMAGE_GOLANG_SDK=$(MQ_IMAGE_GOLANG_SDK)
+	@echo MQ_IMAGE_DEVSERVER_BASE=$(MQ_IMAGE_DEVSERVER_BASE)
+	@echo MQ_IMAGE_DEVSERVER=$(MQ_IMAGE_DEVSERVER)
+	@echo MQ_IMAGE_ADVANCEDSERVER=$(MQ_IMAGE_ADVANCEDSERVER)
+
 include formatting.mk

Makefile-UBUNTU

@@ -1,4 +1,4 @@
-# © Copyright IBM Corporation 2017, 2018
+# © Copyright IBM Corporation 2017, 2019
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@ MQ_VERSION ?= 9.1.1.0
 # MQ_ARCHIVE is the name of the file, under the downloads directory, from which MQ Advanced can
 # be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture
 # Does not apply to MQ Advanced for Developers.
-MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION)_$(MQ_ARCHIVE_TYPE)_$(MQ_ARCHIVE_ARCH).tar.gz
+MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION_VRM)_$(MQ_ARCHIVE_TYPE)_$(MQ_ARCHIVE_ARCH).tar.gz
 # MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced
 # for Developers can be installed
 MQ_ARCHIVE_DEV ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION))
@@ -62,6 +62,10 @@ DEV_JMS_IMAGE=mq-dev-jms-test
 # Variables for versioning
 IMAGE_REVISION=$(shell git rev-parse HEAD)
 IMAGE_SOURCE=$(shell git config --get remote.origin.url)
+EMPTY:=
+SPACE:= $(EMPTY) $(EMPTY)
+# MQ_VERSION_VRM is MQ_VERSION with only the Version, Release and Modifier fields (no Fix field).  e.g. 9.1.1 instead of 9.1.1.0
+MQ_VERSION_VRM=$(subst $(SPACE),.,$(wordlist 1,3,$(subst .,$(SPACE),$(MQ_VERSION))))
 
 ifneq (,$(findstring Microsoft,$(shell uname -r)))
 	DOWNLOADS_DIR=$(patsubst /mnt/c%,C:%,$(realpath ./downloads/))

mq-advanced-server-rhel/go-buildah.sh

@@ -34,12 +34,15 @@ readonly dev=$2
 IMAGE_REVISION=${IMAGE_REVISION:="Not Applicable"}
 IMAGE_SOURCE=${IMAGE_SOURCE:="Not Applicable"}
 
+# Run the build in a container
+# Note the ":Z" on the volume is to allow the container to access the files when SELinux is enabled
 podman run \
-  --volume ${PWD}:/opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/ \
+  --volume ${PWD}:/opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/:Z \
   --env IMAGE_REVISION="$IMAGE_REVISION" \
   --env IMAGE_SOURCE="$IMAGE_SOURCE" \
   --env MQDEV=${dev} \
   --user $(id -u) \
   --rm \
+  --network podman \
   ${tag} \
   bash -c "cd /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/ && ./mq-advanced-server-rhel/go-build.sh"

mq-advanced-server-rhel/install-mq-rhel.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # -*- mode: sh -*-
-# © Copyright IBM Corporation 2018
+# © Copyright IBM Corporation 2018, 2019
 #
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -46,9 +46,10 @@ fi
 
 
 # Accept the MQ license
-buildah run --volume ${dir_extract}:/mnt/mq-download $ctr_mq -- /mnt/mq-download/MQServer/mqlicense.sh -text_only -accept
+buildah run --user root --volume ${dir_extract}:/mnt/mq-download:Z $ctr_mq -- /mnt/mq-download/MQServer/mqlicense.sh -text_only -accept
 
-buildah run --volume ${dir_extract}:/mnt/mq-download $ctr_mq -- bash -c "cd /mnt/mq-download/MQServer && rpm -ivh $mq_packages"
+# Install MQ
+buildah run --user root --volume ${dir_extract}:/mnt/mq-download:Z $ctr_mq -- bash -c "cd /mnt/mq-download/MQServer && rpm -ivh $mq_packages"
 
 rm -rf ${dir_extract}/MQServer
 
@@ -71,7 +72,7 @@ rm -rf $mnt_mq/var/mqm
 mkdir -p $mnt_mq/mnt/mqm
 
 # Create a symlink for /var/mqm -> /mnt/mqm/data
-buildah run $ctr_mq -- ln -s /mnt/mqm/data /var/mqm
+buildah run --user root $ctr_mq -- ln -s /mnt/mqm/data /var/mqm
 
 # Optional: Set these values for the IBM Cloud Vulnerability Report
 sed -i 's/PASS_MAX_DAYS\t99999/PASS_MAX_DAYS\t90/' $mnt_mq/etc/login.defs

mq-advanced-server-rhel/mq-buildah.sh

@@ -16,8 +16,6 @@
 # limitations under the License.
 
 # Build a RHEL image, using the buildah tool
-# Usage
-# mq-buildah.sh ARCHIVE-NAME PACKAGES
 
 set -x
 set -e
@@ -56,15 +54,24 @@ readonly packages=$2
 readonly tag=$3
 readonly version=$4
 readonly mqdev=$5
+readonly mqm_uid=888
+readonly mqm_gid=888
 
 ###############################################################################
 # Install MQ server
 ###############################################################################
 
-# Use the Yum repositories configured on the host
+microdnf_opts="--nodocs"
-cp /etc/yum.repos.d/* ${mnt_mq}/etc/yum.repos.d/
+# Check whether the host is registered with Red Hat
-# Install the packages required by MQ
+if subscription-manager status ; then
-yum install -y --installroot=${mnt_mq} --setopt install_weak_deps=false --setopt=tsflags=nodocs --setopt=override_install_langs=en_US.utf8 \
+  # Host is subscribed, but the minimal image has no enabled repos
+  # Note that the "bc" package is the only one in "extras"
+  microdnf_opts="${microdnf_opts} --enablerepo=rhel-7-server-rpms --enablerepo=rhel-7-server-extras-rpms"
+else
+  # Use the Yum repositories configured on the host
+  cp -R /etc/yum.repos.d/* ${mnt_mq}/etc/yum.repos.d/
+fi
+buildah run ${ctr_mq} -- microdnf ${microdnf_opts} install \
   bash \
   bc \
   coreutils \
@@ -81,27 +88,26 @@ yum install -y --installroot=${mnt_mq} --setopt install_weak_deps=false --setopt
   util-linux \
   which
 
-groupadd --root ${mnt_mq} --system --gid 888 mqm
-useradd --root ${mnt_mq} --system --uid 888 --gid mqm mqm
-usermod --root ${mnt_mq} -aG root mqm
-usermod --root ${mnt_mq} -aG mqm root
-
 # Clean up cached files
-yum clean --installroot=${mnt_mq} all
+buildah run ${ctr_mq} -- microdnf ${microdnf_opts} clean all
-rm -rf ${mnt_mq}/var/cache/yum/*
 rm -rf ${mnt_mq}/etc/yum.repos.d/*
 
+buildah run --user root $ctr_mq -- groupadd --system --gid ${mqm_gid} mqm
+buildah run --user root $ctr_mq -- useradd --system --uid ${mqm_uid} --gid mqm mqm
+buildah run --user root $ctr_mq -- usermod -aG root mqm
+buildah run --user root $ctr_mq -- usermod -aG mqm root
+
 # Install MQ server packages into the MQ builder image
 ./mq-advanced-server-rhel/install-mq-rhel.sh ${ctr_mq} "${mnt_mq}" "${archive}" "${packages}"
 
 # Create the directory for MQ configuration files
 mkdir -p ${mnt_mq}/etc/mqm
-chown 888:888 ${mnt_mq}/etc/mqm
+chown ${mqm_uid}:${mqm_gid} ${mnt_mq}/etc/mqm
 
 # Install the Go binaries into the image
-install --mode 0750 --owner 888 --group 888 ./build/runmqserver ${mnt_mq}/usr/local/bin/
+install --mode 0750 --owner ${mqm_uid} --group 0 ./build/runmqserver ${mnt_mq}/usr/local/bin/
-install --mode 6750 --owner 888 --group 888 ./build/chk* ${mnt_mq}/usr/local/bin/
+install --mode 6750 --owner ${mqm_uid} --group 0 ./build/chk* ${mnt_mq}/usr/local/bin/
-install --mode 0750 --owner 888 --group 888 ./NOTICES.txt ${mnt_mq}/opt/mqm/licenses/notices-container.txt
+install --mode 0750 --owner ${mqm_uid} --group 0 ./NOTICES.txt ${mnt_mq}/opt/mqm/licenses/notices-container.txt
 
 ###############################################################################
 # Final Buildah commands

mq-advanced-server-rhel/mqdev-buildah.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # -*- mode: sh -*-
-# © Copyright IBM Corporation 2018
+# © Copyright IBM Corporation 2018, 2019
 #
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,9 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Build a RHEL image, using the buildah tool
+# Build a RHEL image of MQ Advanced for Developers, using the buildah tool
-# Usage
-# mq-buildah.sh ARCHIVEFILE PACKAGES
 
 set -x
 set -e
@@ -57,12 +55,11 @@ readonly tag=$2
 readonly version=$3
 
 
-useradd --root $mnt_mq --gid mqm admin
+# Run these commands inside the container so that the SELinux context is handled correctly
-groupadd --root $mnt_mq --system mqclient
+buildah run --user root $ctr_mq -- useradd --gid mqm admin
-useradd --root $mnt_mq --gid mqclient app
+buildah run --user root $ctr_mq -- groupadd --system mqclient
-
+buildah run --user root $ctr_mq -- useradd --gid mqclient app
-buildah run $ctr_mq -- id admin
+buildah run --user root $ctr_mq -- bash -c "echo admin:passw0rd | chpasswd"
-buildah run $ctr_mq -- sh -c "echo admin:passw0rd | chpasswd"
 
 mkdir -p $mnt_mq/run/runmqdevserver
 chown 888:888 $mnt_mq/run/runmqdevserver