From c6a921efee225231b9bdcd6ff5e390a0d94c756e Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 1 Oct 2018 10:14:35 +0100 Subject: [PATCH 01/60] Tidy up Travis output (#213) * Ignore generated Prometheus files * Tidy up Travis output * Use relative path for script * Futher tidying of Travis output --- .gitignore | 4 ++++ .travis.yml | 19 +++++++++++-------- install-build-deps-ubuntu.sh | 30 ++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) create mode 100755 install-build-deps-ubuntu.sh diff --git a/.gitignore b/.gitignore index 8d8334b..8ca3f5a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ build coverage downloads incubating/mqipt/ms81* +vendor/github.com/prometheus/client_model/bin/ +vendor/github.com/prometheus/client_model/.classpath +vendor/github.com/prometheus/client_model/.project +vendor/github.com/prometheus/client_model/.settings* diff --git a/.travis.yml b/.travis.yml index a13091c..b223b54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,24 +46,27 @@ jobs: - env: DOCKER_DOWNGRADE="echo nothing to be done" before_install: - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - sudo apt-get update - - sudo apt-get -y install docker-ce - - curl https://glide.sh/get | sh - - sudo curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 - - sudo chmod +x /usr/local/bin/dep + - ./install-build-deps-ubuntu.sh install: - echo nothing script: + - echo 'Downloading Go dependencies...' && echo -en 'travis_fold:start:deps\\r' - make deps + - echo -en 'travis_fold:end:deps\\r' + - echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserver\\r' - make build-devserver + - echo -en 'travis_fold:end:build-devserver\\r' + - echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r' - make build-devjmstest + - echo -en 'travis_fold:end:build-devjmstest\\r' + - echo 'Downgrading Docker (if necessary)...' && echo -en 'travis_fold:start:docker-downgrade\\r' - eval "$DOCKER_DOWNGRADE" + - echo -en 'travis_fold:end:docker-downgrade\\r' + - echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r' - make test-devserver + - echo -en 'travis_fold:end:test-devserver\\r' after_success: - - go get golang.org/x/lint/golint - make lint diff --git a/install-build-deps-ubuntu.sh b/install-build-deps-ubuntu.sh new file mode 100755 index 0000000..bcb1bea --- /dev/null +++ b/install-build-deps-ubuntu.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# -*- mode: sh -*- +# © Copyright IBM Corporation 2015, 2018 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install Docker and dep, required by build (assumes Ubuntu host, as used by Travis build) + +set -ex + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +sudo apt-get update +sudo apt-get -y install docker-ce +curl https://glide.sh/get | sh +sudo curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 +sudo chmod +x /usr/local/bin/dep + +go get golang.org/x/lint/golint From 31f604cc47fd56d0504bf7f2fc3ad4734d8aedbc Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 1 Oct 2018 15:06:51 +0100 Subject: [PATCH 02/60] Document build prereqs for RHEL --- docs/building.md | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/docs/building.md b/docs/building.md index 33dda86..3a3dd7b 100644 --- a/docs/building.md +++ b/docs/building.md @@ -1,18 +1,38 @@ -# Building a Docker image +# Building a container image ## Prerequisites + +### Prerequisites for building an Ubuntu image +If you want to build a container image with Ubuntu Linux as the base OS, then you need to have the following tools installed: + You need to ensure you have the following tools installed: * [Docker](https://www.docker.com/) V17.06.1 or later * [GNU make](https://www.gnu.org/software/make/) If you are working in the Windows Subsystem for Linux, follow [this guide by Microsoft to set up Docker](https://blogs.msdn.microsoft.com/commandline/2017/12/08/cross-post-wsl-interoperability-with-docker/) first. +### Prerequisites for building a Red Hat Enterprise Linux image +If you want to build a container image with Red Hat Enterprise Linux as the base OS, then you need to use a host server with Red Hat Enterprise Linux. You must also have the following tools installed: + +* `[buildah](https://buildah.io)` (available in `rhel-7-server-extras`) +* `[podman](https://podman.io)` (available in `rhel-7-server-extras`) + +In addition, you need the following commonly installed tools: + +* `bash` +* `coreutils` +* `findutils` +* `make` +* `sed` +* `shadow-utils` +* `tar` + ## Building a production image This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures. 1. Create a `downloads` directory in the root of this repository 2. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `IBM_MQ_9.1.0.0_UBUNTU_X86-64.tar.gz` for MQ V9.1.0 for Ubuntu on x86_64 architecture) in the `downloads` directory -2. Run `make build-advancedserver` +2. Run `make build-advancedserver-ubuntu` or `make build-advancedserver-rhel` > **Warning**: Note that MQ offers two different sets of packaging on Linux: one is called "MQ for Linux" and contains RPM files for installing on Red Hat Enterprise Linux and SUSE Linux Enterprise Server. The other package is called "MQ for Ubuntu", and contains DEB files for installing on Ubuntu. @@ -29,22 +49,10 @@ MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 make build-advancedserver ``` ## Building a developer image -Run `make build-devserver`, which will download the latest version of MQ Advanced for Developers from IBM developerWorks. This is currently only available on the `x86_64` architecture. +Run `make build-devserver-ubuntu` or `make build-devserver-rhel`, which will download the latest version of MQ Advanced for Developers from IBM developerWorks. This is currently only available on the `x86_64` architecture. You can use the environment variable `MQ_ARCHIVE_DEV` to specify an alternative local file to install from (which must be in the `downloads` directory). -## Building on a different base image -By default, the MQ images use Ubuntu as the base layer. You can build using a Red Hat Enterprise Linux compatible base layer by setting the `BASE_IMAGE` environment variable. For example: - -``` -BASE_IMAGE=centos:7 make build-advancedserver -``` - -The `make` tool will try and locate the right archive file under the `downloads` directory, based on your platform architecture and your `MQ_VERSION` environment variable, for example `IBM_MQ_9.1.0.0_LINUX_X86_64.tar.gz` for MQ V9.1.0.0 on x86_64. You can also set the `MQ_ARCHIVE` environment variable to set the specific file name. - -Note that if you are using Red Hat Enterprise Linux, you will need to create your own base image layer, with your subscription enabled, as described [here](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_atomic_host/7/html/getting_started_with_containers/get_started_with_docker_formatted_container_images). The MQ image build needs to install some additional packages, and a subscription is required to access the Red Hat repositories. - - ## Installed components -This image includes the core MQ server, Java, language packs, and GSKit. This can be configured by setting the `MQ_PACKAGES` argument to `make`, or directly as a [Docker build argument](https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables-build-arg). +This image includes the core MQ server, Java, language packs, and GSKit. This can be configured by setting the `MQ_PACKAGES` argument to `make`. For the Ubuntu-based image, you can also directly set a [Docker build argument](https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables-build-arg). From d0fce28ef3cd834ea9952d54360b83038e94ea76 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 2 Oct 2018 13:33:20 +0100 Subject: [PATCH 03/60] build Ubuntu image on non-RHEL hosts --- Makefile | 48 ++++++++++++++++++------------------------------ docs/building.md | 13 +++++++------ 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index cd0e45f..10302df 100644 --- a/Makefile +++ b/Makefile @@ -17,66 +17,54 @@ ############################################################################### GO_PKG_DIRS = ./cmd ./internal ./test -BASE_OS = $(shell cat /etc/*-release | grep ID=) -ifeq "$(findstring ubuntu,$(BASE_OS))" "ubuntu" - BASE_OS=UBUNTU -else ifeq "$(findstring rhel,$(BASE_OS))" "rhel" - BASE_OS=RHEL -else - BASE_OS=UNKNOWN +# Set variable if running on a Red Hat Enterprise Linux host +ifneq ($(wildcard /etc/redhat-release),) +REDHAT_RELEASE = $(shell cat /etc/redhat-release) +ifeq "$(findstring Red Hat,$(REDHAT_RELEASE))" "Red Hat" + RHEL_HOST = "true" +endif endif - - ############################################################################### # Build targets ############################################################################### -# default to building UBUNTU as this was the default for the previous Makefile +# Targets default to a RHEL image on a RHEL host, or an Ubuntu image everywhere else + .PHONY: build-devserver -ifeq ($(BASE_OS),UBUNTU) -build-devserver: build-devserver-ubuntu -else ifeq ($(BASE_OS),RHEL) +ifdef RHEL_HOST build-devserver: build-devserver-rhel else -build-devserver: unknownos +build-devserver: build-devserver-ubuntu endif .PHONY: build-advancedserver -ifeq ($(BASE_OS),UBUNTU) -build-advancedserver: build-advancedserver-ubuntu -else ifeq ($(BASE_OS),RHEL) +ifdef RHEL_HOST build-advancedserver: build-advancedserver-rhel else -build-advancedserver: unknownos +build-advancedserver: build-advancedserver-ubuntu endif .PHONY: test-devserver -ifeq ($(BASE_OS),UBUNTU) -test-devserver: test-devserver-ubuntu -else ifeq ($(BASE_OS),RHEL) +ifdef RHEL_HOST test-devserver: test-devserver-rhel else -test-devserver: unknownos +test-devserver: test-devserver-ubuntu endif .PHONY: test-advancedserver -ifeq ($(BASE_OS),UBUNTU) -test-advancedserver: test-advancedserver-ubuntu -else ifeq ($(BASE_OS),RHEL) +ifdef RHEL_HOST test-advancedserver: test-advancedserver-rhel else -test-advancedserver: unknownos +test-advancedserver: test-advancedserver-ubuntu endif .PHONY: build-devjmstest -ifeq ($(BASE_OS),UBUNTU) -build-devjmstest: build-devjmstest-ubuntu -else ifeq ($(BASE_OS),RHEL) +ifdef RHEL_HOST build-devjmstest: build-devjmstest-rhel else -build-devjmstest: unknownos +build-devjmstest: build-devjmstest-ubuntu endif # UBUNTU building targets diff --git a/docs/building.md b/docs/building.md index 3a3dd7b..48f198d 100644 --- a/docs/building.md +++ b/docs/building.md @@ -5,7 +5,6 @@ ### Prerequisites for building an Ubuntu image If you want to build a container image with Ubuntu Linux as the base OS, then you need to have the following tools installed: -You need to ensure you have the following tools installed: * [Docker](https://www.docker.com/) V17.06.1 or later * [GNU make](https://www.gnu.org/software/make/) @@ -14,8 +13,8 @@ If you are working in the Windows Subsystem for Linux, follow [this guide by Mic ### Prerequisites for building a Red Hat Enterprise Linux image If you want to build a container image with Red Hat Enterprise Linux as the base OS, then you need to use a host server with Red Hat Enterprise Linux. You must also have the following tools installed: -* `[buildah](https://buildah.io)` (available in `rhel-7-server-extras`) -* `[podman](https://podman.io)` (available in `rhel-7-server-extras`) +* [`buildah`](https://buildah.io) (available in `rhel-7-server-extras`) +* [`podman`](https://podman.io) (available in `rhel-7-server-extras`) In addition, you need the following commonly installed tools: @@ -31,11 +30,13 @@ In addition, you need the following commonly installed tools: This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures. 1. Create a `downloads` directory in the root of this repository -2. Download MQ from IBM Passport Advantage, and place the downloaded file (for example, `IBM_MQ_9.1.0.0_UBUNTU_X86-64.tar.gz` for MQ V9.1.0 for Ubuntu on x86_64 architecture) in the `downloads` directory -2. Run `make build-advancedserver-ubuntu` or `make build-advancedserver-rhel` +2. Download MQ from [IBM Passport Advantage](https://www.ibm.com/software/passportadvantage/) or [IBM Fix Central](https://www.ibm.com/support/fixcentral), and place the downloaded file (for example, `IBM_MQ_9.1_UBUNTU_X86-64.tar.gz` for MQ V9.1.0 for Ubuntu on x86_64 architecture) in the `downloads` directory +3. Run `make build-advancedserver` > **Warning**: Note that MQ offers two different sets of packaging on Linux: one is called "MQ for Linux" and contains RPM files for installing on Red Hat Enterprise Linux and SUSE Linux Enterprise Server. The other package is called "MQ for Ubuntu", and contains DEB files for installing on Ubuntu. +On a Red Hat Enterprise Linux host, the command `make build-advancedserver` will build a container image using Red Hat Enterprise Linux as the base. On all other hosts, the base image will be Ubuntu. + You can build a different version of MQ by setting the `MQ_VERSION` environment variable, for example: ```bash @@ -49,7 +50,7 @@ MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 make build-advancedserver ``` ## Building a developer image -Run `make build-devserver-ubuntu` or `make build-devserver-rhel`, which will download the latest version of MQ Advanced for Developers from IBM developerWorks. This is currently only available on the `x86_64` architecture. +Run `make build-devserver`, which will download the latest version of MQ Advanced for Developers from IBM developerWorks. This is currently only available on the `x86_64` architecture. On a Red Hat Enterprise Linux host, this command will build a container image using Red Hat Enterprise Linux as the base. On all other hosts, the base image will be Ubuntu. You can use the environment variable `MQ_ARCHIVE_DEV` to specify an alternative local file to install from (which must be in the `downloads` directory). From 6d11b0d8ae460ee878297942de3641365a5cf435 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 8 Oct 2018 16:57:07 +0100 Subject: [PATCH 04/60] update apparmor --- install-mq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-mq.sh b/install-mq.sh index 6273628..d72b4f0 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -139,7 +139,7 @@ rm -rf ${DIR_EXTRACT} # Apply any bug fixes not included in base Ubuntu or MQ image. # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run -$UBUNTU && apt-get install -y gcc-5-base gnupg gpgv libgcrypt20 libstdc++6 perl-base --only-upgrade +$UBUNTU && apt-get install -y libapparmor1 --only-upgrade # End of bug fixes # Clean up cached files From 78ce84b3a1c69f5e4f460571a333ae2e7592fba1 Mon Sep 17 00:00:00 2001 From: Rob Parker Date: Thu, 11 Oct 2018 15:39:22 +0100 Subject: [PATCH 05/60] Implement GOSec for security scanning Fix vulnerabilities (#227) * Implement GOSec for security scanning Fix vulnerabilities * Fix lint failure * address PR comments and fix build break * Fix test break in mqsc --- .gitignore | 1 + Makefile | 26 ++++++++++++++++++++++++++ cmd/chkmqhealthy/main.go | 1 + cmd/chkmqready/main.go | 5 ++++- cmd/runmqdevserver/keystore.go | 24 ++++++++++++++++++++---- cmd/runmqdevserver/main.go | 12 ++++++++++-- cmd/runmqdevserver/mqsc.go | 9 ++++++++- cmd/runmqdevserver/template.go | 7 ++++++- cmd/runmqdevserver/tls.go | 1 + cmd/runmqserver/crtmqvol.go | 1 + cmd/runmqserver/license.go | 1 + cmd/runmqserver/logging.go | 17 ++++++++++++++--- cmd/runmqserver/main.go | 12 ++++++++++-- cmd/runmqserver/mirror.go | 10 ++++++++-- cmd/runmqserver/mqconfig.go | 30 +++++++++++++++--------------- cmd/runmqserver/qmgr.go | 14 +++++++++++--- cmd/runmqserver/signals.go | 3 ++- internal/command/command.go | 2 ++ internal/metrics/metrics.go | 12 ++++++++---- internal/metrics/update.go | 1 + 20 files changed, 150 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 8ca3f5a..da814fc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ vendor/github.com/prometheus/client_model/bin/ vendor/github.com/prometheus/client_model/.classpath vendor/github.com/prometheus/client_model/.project vendor/github.com/prometheus/client_model/.settings* +gosec_results.json diff --git a/Makefile b/Makefile index 10302df..5afa075 100644 --- a/Makefile +++ b/Makefile @@ -141,6 +141,32 @@ lint: $(addsuffix /$(wildcard *.go), $(GO_PKG_DIRS)) @# As of 11/04/2018 there is an open issue to fix it: https://github.com/golang/lint/issues/320 golint -set_exit_status $(sort $(dir $(wildcard $(addsuffix /*/*.go, $(GO_PKG_DIRS))))) +.PHONY: gosec +gosec: $(info $(SPACER)$(shell printf "Running gosec test"$(END))) + @gosec -fmt=json -out=gosec_results.json cmd/... internal/... 2> /dev/null ;\ + cat "gosec_results.json" ;\ + cat gosec_results.json | grep HIGH | grep severity > /dev/null ;\ + if [ $$? -eq 0 ]; then \ + printf "\nFAILURE: gosec found files containing HIGH severity issues - see results.json\n" ;\ + exit 1 ;\ + else \ + printf "\ngosec found no HIGH severity issues\n" ;\ + fi ;\ + cat gosec_results.json | grep MEDIUM | grep severity > /dev/null ;\ + if [ $$? -eq 0 ]; then \ + printf "\nFAILURE: gosec found files containing MEDIUM severity issues - see results.json\n" ;\ + exit 1 ;\ + else \ + printf "\ngosec found no MEDIUM severity issues\n" ;\ + fi ;\ + cat gosec_results.json | grep LOW | grep severity > /dev/null;\ + if [ $$? -eq 0 ]; then \ + printf "\nFAILURE: gosec found files containing LOW severity issues - see results.json\n" ;\ + exit 1;\ + else \ + printf "\ngosec found no LOW severity issues\n" ;\ +fi ;\ + .PHONY: unknownos unknownos: $(info $(SPACER)$(shell printf "ERROR: Unknown OS ("$(BASE_OS)") please run specific make targets"$(END))) diff --git a/cmd/chkmqhealthy/main.go b/cmd/chkmqhealthy/main.go index b26bea8..1420805 100644 --- a/cmd/chkmqhealthy/main.go +++ b/cmd/chkmqhealthy/main.go @@ -32,6 +32,7 @@ func queueManagerHealthy() (bool, error) { return false, err } // Specify the queue manager name, just in case someone's created a second queue manager + // #nosec G204 cmd := exec.Command("dspmq", "-n", "-m", name) // Run the command and wait for completion out, err := cmd.CombinedOutput() diff --git a/cmd/chkmqready/main.go b/cmd/chkmqready/main.go index 4bb0c66..3f978d1 100644 --- a/cmd/chkmqready/main.go +++ b/cmd/chkmqready/main.go @@ -37,5 +37,8 @@ func main() { fmt.Println(err) os.Exit(1) } - conn.Close() + err = conn.Close() + if err != nil { + fmt.Println(err) + } } diff --git a/cmd/runmqdevserver/keystore.go b/cmd/runmqdevserver/keystore.go index 87eac87..5027ead 100644 --- a/cmd/runmqdevserver/keystore.go +++ b/cmd/runmqdevserver/keystore.go @@ -65,11 +65,27 @@ func (ks *KeyStore) Create() error { stashFile := ks.Filename[0:len(ks.Filename)-len(extension)] + ".sth" rdbFile := ks.Filename[0:len(ks.Filename)-len(extension)] + ".rdb" crlFile := ks.Filename[0:len(ks.Filename)-len(extension)] + ".crl" - os.Remove(stashFile) - os.Remove(rdbFile) - os.Remove(crlFile) + err = os.Remove(stashFile) + if err != nil { + log.Errorf("Error removing %s: %v", stashFile, err) + return err + } + err = os.Remove(rdbFile) + if err != nil { + log.Errorf("Error removing %s: %v", rdbFile, err) + return err + } + err = os.Remove(crlFile) + if err != nil { + log.Errorf("Error removing %s: %v", crlFile, err) + return err + } + } + err = os.Remove(ks.Filename) + if err != nil { + log.Errorf("Error removing %s: %v", ks.Filename, err) + return err } - os.Remove(ks.Filename) } else if !os.IsNotExist(err) { // If the keystore exists but cannot be accessed then return the error return err diff --git a/cmd/runmqdevserver/main.go b/cmd/runmqdevserver/main.go index 8546bc8..c6e9112 100644 --- a/cmd/runmqdevserver/main.go +++ b/cmd/runmqdevserver/main.go @@ -30,13 +30,17 @@ import ( var log *logger.Logger func setPassword(user string, password string) error { + // #nosec G204 cmd := exec.Command("chpasswd") stdin, err := cmd.StdinPipe() if err != nil { return err } fmt.Fprintf(stdin, "%s:%s", user, password) - stdin.Close() + err = stdin.Close() + if err != nil { + log.Errorf("Error closing password stdin: %v", err) + } _, _, err = command.RunCmd(cmd) if err != nil { return err @@ -165,6 +169,10 @@ func main() { osExit(1) } else { // Replace this process with runmqserver - syscall.Exec("/usr/local/bin/runmqserver", []string{"runmqserver"}, os.Environ()) + // #nosec G204 + err = syscall.Exec("/usr/local/bin/runmqserver", []string{"runmqserver"}, os.Environ()) + if err != nil { + log.Errorf("Error replacing this process with runmqserver: %v", err) + } } } diff --git a/cmd/runmqdevserver/mqsc.go b/cmd/runmqdevserver/mqsc.go index 77e3dfa..93d58ea 100644 --- a/cmd/runmqdevserver/mqsc.go +++ b/cmd/runmqdevserver/mqsc.go @@ -35,7 +35,14 @@ func updateMQSC(appPasswordRequired bool) error { return err } } else { - os.Remove(mqsc) + _, err := os.Stat(mqsc) + if !os.IsNotExist(err) { + err = os.Remove(mqsc) + if err != nil { + log.Errorf("Error removing file %s: %v", mqsc, err) + return err + } + } } return nil } diff --git a/cmd/runmqdevserver/template.go b/cmd/runmqdevserver/template.go index 7941633..28f8aa8 100644 --- a/cmd/runmqdevserver/template.go +++ b/cmd/runmqdevserver/template.go @@ -36,7 +36,11 @@ func processTemplateFile(templateFile, destFile string, data interface{}) error _, err = os.Stat(dir) if err != nil { if os.IsNotExist(err) { - os.MkdirAll(dir, 0660) + err = os.MkdirAll(dir, 0660) + if err != nil { + log.Error(err) + return err + } mqmUID, mqmGID, err := command.LookupMQM() if err != nil { log.Error(err) @@ -51,6 +55,7 @@ func processTemplateFile(templateFile, destFile string, data interface{}) error return err } } + // #nosec G302 f, err := os.OpenFile(destFile, os.O_CREATE|os.O_WRONLY, 0660) defer f.Close() err = t.Execute(f, data) diff --git a/cmd/runmqdevserver/tls.go b/cmd/runmqdevserver/tls.go index 7e6ed2c..8e21ed6 100644 --- a/cmd/runmqdevserver/tls.go +++ b/cmd/runmqdevserver/tls.go @@ -85,6 +85,7 @@ func configureTLS(qmName string, inputFile string, passPhrase string) error { _, err = os.Stat(dir) if err != nil { if os.IsNotExist(err) { + // #nosec G301 err = os.MkdirAll(dir, 0770) if err != nil { return err diff --git a/cmd/runmqserver/crtmqvol.go b/cmd/runmqserver/crtmqvol.go index 3ff017f..cadd833 100644 --- a/cmd/runmqserver/crtmqvol.go +++ b/cmd/runmqserver/crtmqvol.go @@ -29,6 +29,7 @@ func createVolume(path string) error { fi, err := os.Stat(dataPath) if err != nil { if os.IsNotExist(err) { + // #nosec G301 err = os.MkdirAll(dataPath, 0755) if err != nil { return err diff --git a/cmd/runmqserver/license.go b/cmd/runmqserver/license.go index 309a74b..19f199e 100644 --- a/cmd/runmqserver/license.go +++ b/cmd/runmqserver/license.go @@ -78,6 +78,7 @@ func checkLicense() (bool, error) { return true, nil case ok && lic == "view": file := filepath.Join("/opt/mqm/licenses", resolveLicenseFile()) + // #nosec G304 buf, err := ioutil.ReadFile(file) if err != nil { log.Println(err) diff --git a/cmd/runmqserver/logging.go b/cmd/runmqserver/logging.go index d46ac97..70bee1d 100644 --- a/cmd/runmqserver/logging.go +++ b/cmd/runmqserver/logging.go @@ -33,7 +33,7 @@ import ( // var debug = false var log *logger.Logger -var collectDiagOnFail bool = false +var collectDiagOnFail = false func logTerminationf(format string, args ...interface{}) { logTermination(fmt.Sprintf(format, args)) @@ -108,8 +108,12 @@ func configureLogger(name string) (mirrorFunc, error) { return func(msg string) { // Parse the JSON message, and print a simplified version var obj map[string]interface{} - json.Unmarshal([]byte(msg), &obj) - fmt.Printf(formatSimple(obj["ibm_datetime"].(string), obj["message"].(string))) + err := json.Unmarshal([]byte(msg), &obj) + if err != nil { + fmt.Printf("Failed to Unmarshall JSON - %v", err) + } else { + fmt.Printf(formatSimple(obj["ibm_datetime"].(string), obj["message"].(string))) + } }, nil default: log, err = logger.NewLogger(os.Stdout, d, false, name) @@ -124,20 +128,27 @@ func logDiagnostics() { log.Debug("--- Start Diagnostics ---") // show the directory ownership/permissions + // #nosec G104 out, _, _ := command.Run("ls", "-l", "/mnt/") log.Debugf("/mnt/:\n%s", out) + // #nosec G104 out, _, _ = command.Run("ls", "-l", "/mnt/mqm") log.Debugf("/mnt/mqm:\n%s", out) + // #nosec G104 out, _, _ = command.Run("ls", "-l", "/mnt/mqm/data") log.Debugf("/mnt/mqm/data:\n%s", out) + // #nosec G104 out, _, _ = command.Run("ls", "-l", "/var/mqm") log.Debugf("/var/mqm:\n%s", out) + // #nosec G104 out, _, _ = command.Run("ls", "-l", "/var/mqm/errors") log.Debugf("/var/mqm/errors:\n%s", out) // Print out summary of any FDCs + // #nosec G204 cmd := exec.Command("/opt/mqm/bin/ffstsummary") cmd.Dir = "/var/mqm/errors" + // #nosec G104 outB, _ := cmd.CombinedOutput() log.Debugf("ffstsummary:\n%s", string(outB)) diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 102c81e..4df645b 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -129,7 +129,11 @@ func doMain() error { logTermination(err) return err } - configureQueueManager() + err = configureQueueManager() + if err != nil { + logTermination(err) + return err + } enableMetrics := os.Getenv("MQ_ENABLE_METRICS") if enableMetrics == "true" || enableMetrics == "1" { @@ -145,7 +149,11 @@ func doMain() error { // Reap zombies now, just in case we've already got some signalControl <- reapNow // Write a file to indicate that chkmqready should now work as normal - ready.Set() + err = ready.Set() + if err != nil { + logTermination(err) + return err + } // Wait for terminate signal <-signalControl return nil diff --git a/cmd/runmqserver/mirror.go b/cmd/runmqserver/mirror.go index 14f67c3..19b44e7 100644 --- a/cmd/runmqserver/mirror.go +++ b/cmd/runmqserver/mirror.go @@ -139,7 +139,10 @@ func mirrorLog(ctx context.Context, wg *sync.WaitGroup, path string, fromStart b // Always start at the beginning if we've been told to go from the start if offset != 0 && !fromStart { log.Debugf("Seeking offset %v in file %v", offset, path) - f.Seek(offset, 0) + _, err = f.Seek(offset, 0) + if err != nil { + log.Errorf("Unable to return to offset %v: %v", offset, err) + } } closing := false for { @@ -159,7 +162,10 @@ func mirrorLog(ctx context.Context, wg *sync.WaitGroup, path string, fromStart b // could skip all those messages. This could happen with a very small // MQ error log size. mirrorAvailableMessages(f, mf) - f.Close() + err = f.Close() + if err != nil { + log.Errorf("Unable to close mirror file handle: %v", err) + } // Re-open file log.Debugf("Re-opening error log file %v", path) f, err = os.OpenFile(path, os.O_RDONLY, 0) diff --git a/cmd/runmqserver/mqconfig.go b/cmd/runmqserver/mqconfig.go index da8ed2d..143415f 100644 --- a/cmd/runmqserver/mqconfig.go +++ b/cmd/runmqserver/mqconfig.go @@ -24,19 +24,20 @@ import ( "github.com/genuinetools/amicontained/container" ) -func logContainerRuntime() error { +func logContainerRuntime() { r, err := container.DetectRuntime() if err != nil { - return err + log.Printf("Failed to get container runtime: %v", err) + return } log.Printf("Container runtime: %v", r) - return nil } -func logBaseImage() error { +func logBaseImage() { buf, err := ioutil.ReadFile("/etc/os-release") if err != nil { - return err + log.Printf("Failed to read /etc/os-release: %v", err) + return } lines := strings.Split(string(buf), "\n") for _, l := range lines { @@ -44,41 +45,40 @@ func logBaseImage() error { words := strings.Split(l, "\"") if len(words) >= 2 { log.Printf("Base image: %v", words[1]) - return nil + return } } } - return nil } // logCapabilities logs the Linux capabilities (e.g. setuid, setgid). See https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities -func logCapabilities() error { +func logCapabilities() { caps, err := container.Capabilities() if err != nil { - return err + log.Printf("Failed to get container capabilities: %v", err) + return } for k, v := range caps { if len(v) > 0 { log.Printf("Capabilities (%s set): %v", strings.ToLower(k), strings.Join(v, ",")) } } - return nil } // logSeccomp logs the seccomp enforcing mode, which affects which kernel calls can be made -func logSeccomp() error { +func logSeccomp() { s, err := container.SeccompEnforcingMode() if err != nil { - return err + log.Printf("Failed to get container SeccompEnforcingMode: %v", err) + return } log.Printf("seccomp enforcing mode: %v", s) - return nil } // logSecurityAttributes logs the security attributes of the current process. // The security attributes indicate whether AppArmor or SELinux are being used, // and what the level of confinement is. -func logSecurityAttributes() error { +func logSecurityAttributes() { a, err := readProc("/proc/self/attr/current") // On some systems, if AppArmor or SELinux are not installed, you get an // error when you try and read `/proc/self/attr/current`, even though the @@ -87,10 +87,10 @@ func logSecurityAttributes() error { a = "none" } log.Printf("Process security attributes: %v", a) - return nil } func readProc(filename string) (value string, err error) { + // #nosec G304 buf, err := ioutil.ReadFile(filename) if err != nil { return "", err diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index ec9138d..ffabdc9 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -90,6 +90,7 @@ func configureQueueManager() error { for _, file := range files { if strings.HasSuffix(file.Name(), ".mqsc") { abs := filepath.Join(configDir, file.Name()) + // #nosec G204 cmd := exec.Command("runmqsc") stdin, err := cmd.StdinPipe() if err != nil { @@ -97,6 +98,7 @@ func configureQueueManager() error { return err } // Open the MQSC file for reading + // #nosec G304 f, err := os.Open(abs) if err != nil { log.Printf("Error opening %v: %v", abs, err) @@ -104,10 +106,16 @@ func configureQueueManager() error { // Copy the contents to stdin of the runmqsc process _, err = io.Copy(stdin, f) if err != nil { - log.Printf("Error reading %v: %v", abs, err) + log.Errorf("Error reading %v: %v", abs, err) + } + err = f.Close() + if err != nil { + log.Errorf("Failed to close MQSC file handle: %v", err) + } + err = stdin.Close() + if err != nil { + log.Errorf("Failed to close MQSC stdin: %v", err) } - f.Close() - stdin.Close() // Run the command and wait for completion out, err := cmd.CombinedOutput() if err != nil { diff --git a/cmd/runmqserver/signals.go b/cmd/runmqserver/signals.go index 47c3dfb..003565d 100644 --- a/cmd/runmqserver/signals.go +++ b/cmd/runmqserver/signals.go @@ -43,7 +43,8 @@ func signalHandler(qmgr string) chan int { log.Printf("Signal received: %v", sig) signal.Stop(reapSignals) signal.Stop(stopSignals) - metrics.StopMetricsGathering() + metrics.StopMetricsGathering(log) + // #nosec G104 stopQueueManager(qmgr) // One final reap reapZombies() diff --git a/internal/command/command.go b/internal/command/command.go index 024b877..7790c55 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -53,11 +53,13 @@ func RunCmd(cmd *exec.Cmd) (string, int, error) { // Do not use this function to run shell built-ins (like "cd"), because // the error handling works differently func Run(name string, arg ...string) (string, int, error) { + // #nosec G204 return RunCmd(exec.Command(name, arg...)) } // RunAsMQM runs the specified command as the mqm user func RunAsMQM(name string, arg ...string) (string, int, error) { + // #nosec G204 cmd := exec.Command(name, arg...) cmd.SysProcAttr = &syscall.SysProcAttr{} uid, gid, err := LookupMQM() diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 7c0aaae..0947465 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -44,7 +44,7 @@ func GatherMetrics(qmName string, log *logger.Logger) { err := startMetricsGathering(qmName, log) if err != nil { log.Errorf("Metrics Error: %s", err.Error()) - StopMetricsGathering() + StopMetricsGathering(log) } } @@ -76,6 +76,7 @@ func startMetricsGathering(qmName string, log *logger.Logger) error { http.Handle("/metrics", prometheus.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) + // #nosec G104 w.Write([]byte("Status: METRICS ACTIVE")) }) @@ -83,7 +84,7 @@ func startMetricsGathering(qmName string, log *logger.Logger) error { err = metricsServer.ListenAndServe() if err != nil && err != http.ErrServerClosed { log.Errorf("Metrics Error: Failed to handle metrics request: %v", err) - StopMetricsGathering() + StopMetricsGathering(log) } }() @@ -91,7 +92,7 @@ func startMetricsGathering(qmName string, log *logger.Logger) error { } // StopMetricsGathering stops gathering metrics for the queue manager -func StopMetricsGathering() { +func StopMetricsGathering(log *logger.Logger) { if metricsEnabled { @@ -101,6 +102,9 @@ func StopMetricsGathering() { // Shutdown HTTP server timeout, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - metricsServer.Shutdown(timeout) + err := metricsServer.Shutdown(timeout) + if err != nil { + log.Errorf("Failed to shutdown metrics server: %v", err) + } } } diff --git a/internal/metrics/update.go b/internal/metrics/update.go index 9aceceb..f2f470a 100644 --- a/internal/metrics/update.go +++ b/internal/metrics/update.go @@ -62,6 +62,7 @@ func processMetrics(log *logger.Logger, qmName string) { firstConnect = false startChannel <- true } + // #nosec G104 metrics, _ = initialiseMetrics(log) } From 9a34e9b15cd8b1a6e0f8960479bf0219db47b4d5 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Fri, 12 Oct 2018 11:26:24 +0100 Subject: [PATCH 06/60] Clarify docs and fix links (#226) --- docs/usage.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 007a9a2..cc1ee90 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -5,7 +5,7 @@ In order to use the image, it is necessary to accept the terms of the IBM MQ lic ## Running with the default configuration You can run a queue manager with the default configuration and a listener on port 1414 using the following command. For example, the following command creates and starts a queue manager called `QM1`, and maps port 1414 on the host to the MQ listener on port 1414 inside the container, as well as port 9443 on the host to the web console on port 9443 inside the container: -``` +```sh docker run \ --env LICENSE=accept \ --env MQ_QMGR_NAME=QM1 \ @@ -16,15 +16,15 @@ docker run \ ``` ## Running with the default configuration and a volume -The above example will not persist any configuration data or messages across container runs. In order to do this, you need to use a [volume](https://docs.docker.com/engine/admin/volumes/volumes/). For example, you can create a volume with the following command: +The above example will not persist any configuration data or messages across container runs. In order to do this, you need to use a [volume](https://docs.docker.com/storage/volumes/). For example, you can create a volume with the following command: -``` +```sh docker volume create qm1data ``` You can then run a queue manager using this volume as follows: -``` +```sh docker run \ --env LICENSE=accept \ --env MQ_QMGR_NAME=QM1 \ @@ -40,7 +40,7 @@ The Docker image always uses `/mnt/mqm` for MQ data, which is correctly linked f ## Running with the default configuration and Prometheus metrics enabled You can run a queue manager with [Prometheus](https://prometheus.io) metrics enabled. The following command will generate Prometheus metrics for your queue manager on `/metrics` port `9157`: -``` +```sh docker run \ --env LICENSE=accept \ --env MQ_QMGR_NAME=QM1 \ @@ -58,11 +58,11 @@ You can customize the configuration in several ways: 1. For getting started, you can use the [default developer configuration](developer-config.md), which is available out-of-the-box for the MQ Advanced for Developers image 2. By creating your own image and adding your own MQSC file into the `/etc/mqm` directory on the image. This file will be run when your queue manager is created. -3. By using [remote MQ administration](http://www-01.ibm.com/support/knowledgecenter/SSFKSJ_9.0.0/com.ibm.mq.adm.doc/q021090_.htm), via an MQ command server, the MQ HTTP APIs, or using a tool such as the MQ web console or MQ Explorer. +3. By using [remote MQ administration](https://www.ibm.com/support/knowledgecenter/SSFKSJ_9.1.0/com.ibm.mq.adm.doc/q021090_.htm), via an MQ command server, the MQ HTTP APIs, or using a tool such as the MQ web console or MQ Explorer. Note that a listener is always created on port 1414 inside the container. This port can be mapped to any port on the Docker host. -The following is an *example* `Dockerfile` for creating your own pre-configured image, which adds a custom `config.mqsc` and an administrative user `alice`. Note that it is not normally recommended to include passwords in this way: +The following is an *example* `Dockerfile` for creating your own pre-configured image, which adds a custom MQ configuration file, and an administrative user `alice`. Note that it is not normally recommended to include passwords in this way: ```dockerfile FROM ibmcom/mq @@ -71,9 +71,9 @@ RUN useradd alice -G mqm && \ COPY 20-config.mqsc /etc/mqm/ ``` -Here is an example corresponding `20-config.mqsc` script from the [mqdev blog](https://www.ibm.com/developerworks/community/blogs/messaging/entry/getting_going_without_turning_off_mq_security?lang=en), which allows users with passwords to connect on the `PASSWORD.SVRCONN` channel: +Here is an example corresponding `20-config.mqsc` script from the [mqdev blog](https://developer.ibm.com/messaging/2018/10/01/archives-getting-going-without-turning-off-ibm-mq-security/), which allows users with passwords to connect on the `PASSWORD.SVRCONN` channel: -``` +```mqsc DEFINE CHANNEL(PASSWORD.SVRCONN) CHLTYPE(SVRCONN) REPLACE SET CHLAUTH(PASSWORD.SVRCONN) TYPE(BLOCKUSER) USERLIST('nobody') DESCR('Allow privileged users on this channel') SET CHLAUTH('*') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(NOACCESS) DESCR('BackStop rule') @@ -82,10 +82,12 @@ ALTER AUTHINFO(SYSTEM.DEFAULT.AUTHINFO.IDPWOS) AUTHTYPE(IDPWOS) ADOPTCTX(YES) REFRESH SECURITY TYPE(CONNAUTH) ``` +The file `20-config.mqsc` should be saved into the same directory as the `Dockerfile`. + ## Running MQ commands It is recommended that you configure MQ in your own custom image. However, you may need to run MQ commands directly inside the process space of the container. To run a command against a running queue manager, you can use `docker exec`, for example: -``` +```sh docker exec \ --tty \ --interactive \ From fe8a87b39fbde2d809be5855d716652b216f9c8c Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 8 Oct 2018 15:57:29 +0100 Subject: [PATCH 07/60] Fix build failure in RHEL makefile --- Makefile-RHEL | 70 ++++++++++++++----- mq-advanced-server-rhel/go-buildah.sh | 10 +++ mq-advanced-server-rhel/install-mq-rhel.sh | 10 +++ mq-advanced-server-rhel/mq-buildah.sh | 27 +++++++ .../mq-golang-sdk-buildah.sh | 22 ++++++ mq-advanced-server-rhel/mqdev-buildah.sh | 25 +++++++ 6 files changed, 146 insertions(+), 18 deletions(-) diff --git a/Makefile-RHEL b/Makefile-RHEL index 3a8f0de..b8f793e 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -82,18 +82,47 @@ MQ_ARCHIVE_DEV_9.1.0.0=mqadv_dev910_linux_$(MQ_DEV_ARCH).tar.gz ############################################################################### # Build targets ############################################################################### +.PHONY: vars +vars: +#ifeq "$(findstring ubuntu,$(BASE_IMAGE))","ubuntu" + @echo $(MQ_ARCHIVE_ARCH) + @echo $(MQ_ARCHIVE_TYPE) + @echo $(MQ_ARCHIVE) -# Vendor Go dependencies for the Docker tests -test/docker/vendor: - cd test/docker && dep ensure -vendor-only +.PHONY: default +default: build-devserver test-devserver + +# Build all components (except incubating ones) +.PHONY: all +all: build-devserver build-advancedserver + +.PHONY: test-all +test-all: build-devjmstest test-devserver test-advancedserver + +.PHONY: devserver +devserver: build-devserver build-devjmstest test-devserver + +# Build incubating components +.PHONY: incubating +incubating: build-explorer downloads/$(MQ_ARCHIVE_DEV): $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END))) mkdir -p downloads cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_ARCHIVE_DEV) +downloads/$(MQ_SDK_ARCHIVE): + $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END))) + mkdir -p downloads + cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_SDK_ARCHIVE) + .PHONY: downloads -downloads: downloads/$(MQ_ARCHIVE_DEV) +downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE) + +# Vendor Go dependencies for the Docker tests +test/docker/vendor: + cd test/docker && dep ensure -vendor-only + .PHONY: check-prereqs check-prereqs: @@ -111,7 +140,7 @@ check-prereqs: .PHONY: test-advancedserver test-advancedserver: check-test-prereqs test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER) on $(shell docker --version)"$(END))) - buildah push $(MQ_IMAGE_ADVANCEDSERVER) docker-daemon:$(MQ_IMAGE_ADVANCEDSERVER) + sudo buildah push $(MQ_IMAGE_ADVANCEDSERVER) docker-daemon:$(MQ_IMAGE_ADVANCEDSERVER) docker tag docker.io/$(MQ_IMAGE_ADVANCEDSERVER) $(MQ_IMAGE_ADVANCEDSERVER) cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) EXPECTED_LICENSE=Production go test $(TEST_OPTS_DOCKER) @@ -119,44 +148,49 @@ test-advancedserver: check-test-prereqs test/docker/vendor .PHONY: test-devserver test-devserver: check-test-prereqs test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER) on $(shell docker --version)"$(END))) - buildah push $(MQ_IMAGE_DEVSERVER) docker-daemon:$(MQ_IMAGE_DEVSERVER) + sudo buildah push $(MQ_IMAGE_DEVSERVER) docker-daemon:$(MQ_IMAGE_DEVSERVER) docker tag docker.io/$(MQ_IMAGE_DEVSERVER) $(MQ_IMAGE_DEVSERVER) cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) go test -tags mqdev $(TEST_OPTS_DOCKER) .PHONY: build-advancedserver build-advancedserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE) -build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs +build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs-ex $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END))) - mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_ADVANCEDSERVER)" "$(MQ_VERSION)" "$(MQDEV)" + sudo mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_ADVANCEDSERVER)" "$(MQ_VERSION)" "$(MQDEV)" .PHONY: build-devserver build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV) build-devserver: MQDEV=TRUE build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm -build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs +build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs-ex $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER)"$(END))) - mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE_DEV)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_VERSION)" "$(MQDEV)" - mq-advanced-server-rhel/mqdev-buildah.sh "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_IMAGE_DEVSERVER)" "$(MQ_VERSION)" + sudo mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE_DEV)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_VERSION)" "$(MQDEV)" + sudo mq-advanced-server-rhel/mqdev-buildah.sh "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_IMAGE_DEVSERVER)" "$(MQ_VERSION)" .PHONY: build-mqgolang-sdk -build-mqgolang-sdk: check-prereqs downloads/$(MQ_SDK_ARCHIVE) - $(info $(SPACER)$(shell printf $(TITLE)"Build mq-golang SDK"$(END))) - mq-advanced-server-rhel/mq-golang-sdk-buildah.sh "$(MQ_SDK_ARCHIVE)" "$(MQ_IMAGE_GOLANG_SDK)" +build-mqgolang-sdk: check-prereqs downloads/$(MQ_SDK_ARCHIVE) build-mqgolang-sdk-ex +.PHONY: build-mqgolang-sdk-ex +build-mqgolang-sdk-ex: + $(info $(SPACER)$(shell printf $(TITLE)"Build mq-golang SDK"$(END))) + sudo mq-advanced-server-rhel/mq-golang-sdk-buildah.sh "$(MQ_SDK_ARCHIVE)" "$(MQ_IMAGE_GOLANG_SDK)" .PHONY: build-go-programs -build-go-programs: check-prereqs build-mqgolang-sdk +build-go-programs: check-prereqs downloads/$(MQ_SDK_ARCHIVE) build-go-programs-ex + +.PHONY: build-go-programs-ex +build-go-programs-ex: build-mqgolang-sdk-ex $(info $(SPACER)$(shell printf $(TITLE)"Build go programs"$(END))) - IMAGE_REVISION=$(IMAGE_REVISION) IMAGE_SOURCE=$(IMAGE_SOURCE) mq-advanced-server-rhel/go-buildah.sh "$(MQ_IMAGE_GOLANG_SDK)" "$(MQDEV)" + IMAGE_REVISION=$(IMAGE_REVISION) IMAGE_SOURCE=$(IMAGE_SOURCE) sudo mq-advanced-server-rhel/go-buildah.sh "$(MQ_IMAGE_GOLANG_SDK)" "$(MQDEV)" .PHONY: build-devjmstest build-devjmstest: check-test-prereqs $(info $(SPACER)$(shell printf $(TITLE)"Build JMS tests for developer config"$(END))) - cd test/messaging && ./buildah.sh $(DEV_JMS_IMAGE) - buildah push $(DEV_JMS_IMAGE) docker-daemon:$(DEV_JMS_IMAGE) + cd test/messaging && sudo ./buildah.sh $(DEV_JMS_IMAGE) + sudo buildah push $(DEV_JMS_IMAGE) docker-daemon:$(DEV_JMS_IMAGE) docker tag docker.io/$(DEV_JMS_IMAGE) $(DEV_JMS_IMAGE) include formatting.mk diff --git a/mq-advanced-server-rhel/go-buildah.sh b/mq-advanced-server-rhel/go-buildah.sh index e5d25f8..9e9e63d 100755 --- a/mq-advanced-server-rhel/go-buildah.sh +++ b/mq-advanced-server-rhel/go-buildah.sh @@ -18,6 +18,16 @@ # Run the Go build script inside the Go container, mounting the source # directory in +function usage { + echo "Usage: $0 TAG DevModeFlag" + exit 20 +} + +if [ "$#" -ne 2 ]; then + echo "ERROR: Invalid number of parameters" + usage +fi + readonly tag=$1 readonly dev=$2 diff --git a/mq-advanced-server-rhel/install-mq-rhel.sh b/mq-advanced-server-rhel/install-mq-rhel.sh index 59cfc5f..e880fc0 100755 --- a/mq-advanced-server-rhel/install-mq-rhel.sh +++ b/mq-advanced-server-rhel/install-mq-rhel.sh @@ -19,6 +19,16 @@ set -ex +function usage { + echo "Usage: $0 MQContainer MountLocation ARCHIVENAME PACKAGES" + exit 20 +} + +if [ "$#" -ne 4 ]; then + echo "ERROR: Invalid number of parameters" + usage +fi + readonly ctr_mq=$1 readonly mnt_mq=$2 readonly archive=$3 diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 12c803e..719ca36 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -22,12 +22,34 @@ set -x set -e +function usage { + echo "Usage: $0 ARCHIVENAME PACKAGES TAG VERSION MQDevFlag" + exit 20 +} + +if [ "$#" -ne 5 ]; then + echo "ERROR: Invalid number of parameters" + usage +fi + ############################################################################### # Setup MQ server working container ############################################################################### readonly ctr_mq=$(buildah from rhel7) +if [ -z "$ctr_mq" ] +then + echo "ERROR: ctr_mq is empty. Check above output for errors" + exit 50 +fi + readonly mnt_mq=$(buildah mount $ctr_mq) +if [ -z "$mnt_mq" ] +then + echo "ERROR: mnt_mq is empty. Check above output for errors" + exit 50 +fi + readonly archive=downloads/$1 readonly packages=$2 readonly tag=$3 @@ -82,9 +104,11 @@ install --mode 0750 --owner 888 --group 888 ./NOTICES.txt ${mnt_mq}/opt/mqm/lice if [ "$mqdev" = "TRUE" ]; then OSTAG="mq messaging developer" DISNAME="IBM MQ Advanced Server Developer Edition" + PID="98102d16795c4263ad9ca075190a2d4d" else OSTAG="mq messaging" DISNAME="IBM MQ Advanced Server" + PID="4486e8c4cc9146fd9b3ce1f14a2dfc5b" fi buildah config \ @@ -102,6 +126,9 @@ buildah config \ --label run="docker run -d -e LICENSE=accept --name ibm-mq ${tag%:*}" \ --label summary="$DISNAME" \ --label description="IBM MQ is messaging middleware that simplifies and accelerates the integration of diverse applications and business data across multiple platforms. It uses message queues to facilitate the exchanges of information and offers a single messaging solution for cloud, mobile, Internet of Things (IoT) and on-premises environments." \ + --label IBM_PRODUCT_ID="$PID" \ + --label IBM_PRODUCT_NAME="$DISNAME" \ + --label IBM_PRODUCT_VERSION="$version" \ --env AMQ_ADDITIONAL_JSON_LOG=1 \ --env LANG=en_US.UTF-8 \ --env LOG_FORMAT=basic \ diff --git a/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh b/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh index 60add5a..b73a130 100755 --- a/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh +++ b/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh @@ -18,13 +18,35 @@ # Build a RHEL image for building Go programs which use MQ set -ex + +function usage { + echo "Usage: $0 ARCHIVENAME TAG" + exit 20 +} + +if [ "$#" -ne 2 ]; then + echo "ERROR: Invalid number of parameters" + usage +fi + readonly mq_archive=downloads/$1 readonly tag=$2 # Use plain RHEL 7 container # Note: Red Hat's devtools/go-toolset-7-rhel7 image doesn't allow use of 'root' # user required for installing the MQ SDK readonly ctr_mq=$(buildah from rhel7) +if [ -z "$ctr_mq" ] +then + echo "ERROR: ctr_mq is empty. Check above output for errors" + exit 50 +fi + readonly mnt_mq=$(buildah mount $ctr_mq) +if [ -z "$mnt_mq" ] +then + echo "ERROR: mnt_mq is empty. Check above output for errors" + exit 50 +fi # Add mqm user groupadd --root $mnt_mq --system --gid 888 mqm diff --git a/mq-advanced-server-rhel/mqdev-buildah.sh b/mq-advanced-server-rhel/mqdev-buildah.sh index 4c9f7e8..e32ea68 100755 --- a/mq-advanced-server-rhel/mqdev-buildah.sh +++ b/mq-advanced-server-rhel/mqdev-buildah.sh @@ -22,6 +22,16 @@ set -x set -e +function usage { + echo "Usage: $0 BASETAG TAG VERSION" + exit 20 +} + +if [ "$#" -ne 3 ]; then + echo "ERROR: Invalid number of parameters" + usage +fi + ############################################################################### # Setup MQ server working container ############################################################################### @@ -30,7 +40,19 @@ set -e # Resulting image won't have yum, for example readonly basetag=$1 readonly ctr_mq=$(buildah from $basetag) +if [ -z "$ctr_mq" ] +then + echo "ERROR: ctr_mq is empty. Check above output for errors" + exit 50 +fi + readonly mnt_mq=$(buildah mount $ctr_mq) +if [ -z "$mnt_mq" ] +then + echo "ERROR: mnt_mq is empty. Check above output for errors" + exit 50 +fi + readonly tag=$2 readonly version=$3 @@ -74,6 +96,9 @@ buildah config \ --label run="docker run -d -e LICENSE=accept --name ibm-mq-dev ${tag%:*}" \ --label summary="IBM MQ Advanced Server Developer Edition" \ --label description="IBM MQ is messaging middleware that simplifies and accelerates the integration of diverse applications and business data across multiple platforms. It uses message queues to facilitate the exchanges of information and offers a single messaging solution for cloud, mobile, Internet of Things (IoT) and on-premises environments." \ + --label IBM_PRODUCT_ID="98102d16795c4263ad9ca075190a2d4d" \ + --label IBM_PRODUCT_NAME="IBM MQ Advanced Server Developer Edition" \ + --label IBM_PRODUCT_VERSION="$version" \ --env AMQ_ADDITIONAL_JSON_LOG=1 \ --env LANG=en_US.UTF-8 \ --env LOG_FORMAT=basic \ From 1208a5d08b386d1ad57d552cd1447f21f08873b8 Mon Sep 17 00:00:00 2001 From: Rob Parker Date: Thu, 18 Oct 2018 13:50:20 +0100 Subject: [PATCH 08/60] add RHEL into the docker tag for RHEL builds (#234) --- Makefile-RHEL | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile-RHEL b/Makefile-RHEL index b8f793e..0bd0939 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -32,9 +32,9 @@ MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image -MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-RHEL-$(ARCH) # MQ_IMAGE_DEVSERVER is the name and tag of the built MQ Advanced for Developers image -MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-RHEL-$(ARCH) # MQ_IMAGE_SDK is the name and tag of the built MQ Advanced for Developers SDK image MQ_IMAGE_SDK ?=mq-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) # MQ_IMAGE_GOLANG_SDK is the name and tag of the built MQ Advanced for Developers SDK image, plus Go tools From a3c0af9648cd88b38593c70d1baf06016573677e Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Thu, 18 Oct 2018 15:10:45 +0100 Subject: [PATCH 09/60] Add sudo to groupadd --- mq-advanced-server-rhel/mq-golang-sdk-buildah.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh b/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh index b73a130..7874128 100755 --- a/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh +++ b/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh @@ -49,10 +49,10 @@ then fi # Add mqm user -groupadd --root $mnt_mq --system --gid 888 mqm -useradd --root $mnt_mq --system --uid 888 --gid mqm mqm -usermod --root $mnt_mq -aG root mqm -usermod --root $mnt_mq -aG mqm root +sudo groupadd --root $mnt_mq --system --gid 888 mqm +sudo useradd --root $mnt_mq --system --uid 888 --gid mqm mqm +sudo usermod --root $mnt_mq -aG root mqm +sudo usermod --root $mnt_mq -aG mqm root # Enable Yum repository for "optional" RPMs, which is needed for "golang" buildah run ${ctr_mq} -- yum-config-manager --enable rhel-7-server-optional-rpms From b276e0b4ef32d19d47fee7534e76a4a53dd650e0 Mon Sep 17 00:00:00 2001 From: LPowlett <44359672+LPowlett@users.noreply.github.com> Date: Wed, 24 Oct 2018 09:19:39 +0100 Subject: [PATCH 10/60] move image builds to before_script (#237) --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b223b54..d53347b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,7 @@ before_install: install: - echo nothing -script: +before_script: - echo 'Downloading Go dependencies...' && echo -en 'travis_fold:start:deps\\r' - make deps - echo -en 'travis_fold:end:deps\\r' @@ -61,6 +61,8 @@ script: - echo 'Building Developer JMS test image...' && echo -en 'travis_fold:start:build-devjmstest\\r' - make build-devjmstest - echo -en 'travis_fold:end:build-devjmstest\\r' + +script: - echo 'Downgrading Docker (if necessary)...' && echo -en 'travis_fold:start:docker-downgrade\\r' - eval "$DOCKER_DOWNGRADE" - echo -en 'travis_fold:end:docker-downgrade\\r' From b73ad12011add0b183f1079b15244eb45b1d560f Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 7 Nov 2018 11:36:04 +0000 Subject: [PATCH 11/60] Security fixes, Nov 2018 --- install-mq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-mq.sh b/install-mq.sh index d72b4f0..7358155 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -139,7 +139,7 @@ rm -rf ${DIR_EXTRACT} # Apply any bug fixes not included in base Ubuntu or MQ image. # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run -$UBUNTU && apt-get install -y libapparmor1 --only-upgrade +$UBUNTU && apt-get install -y libapparmor1 libsystemd0 systemd systemd-sysv libudev1 --only-upgrade # End of bug fixes # Clean up cached files From e33710eb00561be4797d34f2f0f1fd2153b16e69 Mon Sep 17 00:00:00 2001 From: Luke Powlett Date: Fri, 9 Nov 2018 16:04:36 +0000 Subject: [PATCH 12/60] remove create qm option from console --- .../web/installations/Installation1/servers/mqweb/mqwebuser.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml index fb1f855..d575484 100644 --- a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml +++ b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml @@ -35,5 +35,6 @@ + From 5f000ff891b2a487aac121f13ad5a1aa0de23a17 Mon Sep 17 00:00:00 2001 From: Rob Parker Date: Fri, 30 Nov 2018 17:26:19 +0000 Subject: [PATCH 13/60] Update to IBM MQ version 9.1.1.0 (#251) * Update to MQ v9.1.1 * update incubating to 911 * check docker version on travis * travis docker version is now high enough --- .travis.yml | 3 +-- CHANGELOG.md | 16 ++++++++++++ Dockerfile-server | 2 +- Makefile-RHEL | 3 ++- Makefile-UBUNTU | 3 ++- README.md | 2 +- docs/building.md | 4 +-- docs/testing.md | 6 ++--- incubating/Dockerfile-sfbridge | 2 +- incubating/mq-explorer/Dockerfile | 2 +- incubating/mq-golang-sdk/Dockerfile | 2 +- incubating/mqadvanced-server-dev/Dockerfile | 4 +-- install-build-deps-ubuntu.sh | 4 --- manifests/dockerhub/manifest-9.1.1.yaml | 29 +++++++++++++++++++++ manifests/dockerhub/manifest-9.yaml | 6 ++--- manifests/dockerhub/manifest-latest.yaml | 6 ++--- manifests/dockerstore/manifest-9.1.1.yaml | 29 +++++++++++++++++++++ 17 files changed, 97 insertions(+), 26 deletions(-) create mode 100644 manifests/dockerhub/manifest-9.1.1.yaml create mode 100644 manifests/dockerstore/manifest-9.1.1.yaml diff --git a/.travis.yml b/.travis.yml index d53347b..0631a76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,7 @@ cache: env: - BASE_IMAGE=ubuntu:16.04 -# Commented out temporarily until Issue 166 is resolved -# - BASE_IMAGE=centos:latest + - BASE_IMAGE=centos:latest jobs: include: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d51d44..042c004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Change log +## 9.1.1.0 (2018-11-30) + +* Updated to MQ version 9.1.1.0 +* Created seperate RedHat Makefile for building images on RedHat machines with buildah +* Enabled REST messaging capability for app user. +* Added support for container suplimentary groups +* Removed IBM MQ version 9.0.5 details. +* Added additional Diagnostics ([#203](https://github.com/ibm-messaging/mq-container/pull/203)) +* Implementted GOSec to perform code scans for security vulnerabilities. (([#227](https://github.com/ibm-messaging/mq-container/pull/227))) +* Removed Queue manager create option from the MQ Console. +* Fixes for the following issues: + * Check explicitly for `/mnt/mqm` ([#175](https://github.com/ibm-messaging/mq-container/pull/175)) + * Force string output in chkmqhealthy ([#174](https://github.com/ibm-messaging/mq-container/pull/174)) + * Use -aG not -G when adding a group for a user + * Security fixes for libsystemd0 systemd systemd-sysv & libudev1 + ## 9.1.0.0 (2018-07-23) * Updated to MQ version 9.1.0.0 diff --git a/Dockerfile-server b/Dockerfile-server index 4324555..ba81c0c 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -13,7 +13,7 @@ # limitations under the License. ARG BASE_IMAGE=ubuntu:16.04 -ARG BUILDER_IMAGE=mq-golang-sdk:9.0.5.0-x86_64-ubuntu-16.04 +ARG BUILDER_IMAGE=mq-golang-sdk:9.1.1.0-x86_64-ubuntu-16.04 ############################################################################### # Build stage to build Go code diff --git a/Makefile-RHEL b/Makefile-RHEL index 0bd0939..11f552c 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -19,7 +19,7 @@ # BASE_IMAGE is the base image to use for MQ, for example "ubuntu" or "rhel" BASE_IMAGE ?= rhel # MQ_VERSION is the fully qualified MQ version number to build -MQ_VERSION ?= 9.1.0.0 +MQ_VERSION ?= 9.1.1.0 # MQ_ARCHIVE is the name of the file, under the downloads directory, from which MQ Advanced can # be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture # Does not apply to MQ Advanced for Developers. @@ -78,6 +78,7 @@ endif # Archive names for IBM MQ Advanced for Developers MQ_ARCHIVE_DEV_9.0.5.0=mqadv_dev905_linux_x86-64.tar.gz MQ_ARCHIVE_DEV_9.1.0.0=mqadv_dev910_linux_$(MQ_DEV_ARCH).tar.gz +MQ_ARCHIVE_DEV_9.1.1.0=mqadv_dev910_linux_$(MQ_DEV_ARCH).tar.gz ############################################################################### # Build targets diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index 4ff4c65..71a76b1 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -19,7 +19,7 @@ # BASE_IMAGE is the base image to use for MQ, for example "ubuntu" or "rhel" BASE_IMAGE ?= ubuntu:16.04 # MQ_VERSION is the fully qualified MQ version number to build -MQ_VERSION ?= 9.1.0.0 +MQ_VERSION ?= 9.1.1.0 # MQ_ARCHIVE is the name of the file, under the downloads directory, from which MQ Advanced can # be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture # Does not apply to MQ Advanced for Developers. @@ -91,6 +91,7 @@ endif # Archive names for IBM MQ Advanced for Developers MQ_ARCHIVE_DEV_9.0.5.0=mqadv_dev905_$(MQ_ARCHIVE_DEV_PLATFORM)_x86-64.tar.gz MQ_ARCHIVE_DEV_9.1.0.0=mqadv_dev910_$(MQ_ARCHIVE_DEV_PLATFORM)_$(MQ_DEV_ARCH).tar.gz +MQ_ARCHIVE_DEV_9.1.1.0=mqadv_dev911_$(MQ_ARCHIVE_DEV_PLATFORM)_$(MQ_DEV_ARCH).tar.gz ############################################################################### # Build targets diff --git a/README.md b/README.md index 95f631e..27d1aec 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ The Dockerfiles and associated code and scripts are licensed under the [Apache L Licenses for the products installed within the images are as follows: - [IBM MQ Advanced for Developers](http://www14.software.ibm.com/cgi-bin/weblap/lap.pl?la_formnum=Z125-3301-14&li_formnum=L-APIG-AVCJ4S) (International License Agreement for Non-Warranted Programs). This license may be viewed from an image using the `LICENSE=view` environment variable as described above or by following the link above. -- [IBM MQ Advanced](http://www14.software.ibm.com/cgi-bin/weblap/lap.pl?la_formnum=Z125-3301-14&li_formnum=L-APIG-AV6GV5) (International Program License Agreement). This license may be viewed from an image using the `LICENSE=view` environment variable as described above or by following the link above. +- [IBM MQ Advanced](http://www14.software.ibm.com/cgi-bin/weblap/lap.pl?la_formnum=Z125-3301-14&li_formnum=L-APIG-AZYF4X) (International Program License Agreement). This license may be viewed from an image using the `LICENSE=view` environment variable as described above or by following the link above. - License information for Ubuntu packages may be found in `/usr/share/doc/${package}/copyright` Note: The IBM MQ Advanced for Developers license does not permit further distribution and the terms restrict usage to a developer machine. diff --git a/docs/building.md b/docs/building.md index 48f198d..96bec9f 100644 --- a/docs/building.md +++ b/docs/building.md @@ -30,7 +30,7 @@ In addition, you need the following commonly installed tools: This procedure works for building the MQ Continuous Delivery release, on `x86_64`, `ppc64le` and `s390x` architectures. 1. Create a `downloads` directory in the root of this repository -2. Download MQ from [IBM Passport Advantage](https://www.ibm.com/software/passportadvantage/) or [IBM Fix Central](https://www.ibm.com/support/fixcentral), and place the downloaded file (for example, `IBM_MQ_9.1_UBUNTU_X86-64.tar.gz` for MQ V9.1.0 for Ubuntu on x86_64 architecture) in the `downloads` directory +2. Download MQ from [IBM Passport Advantage](https://www.ibm.com/software/passportadvantage/) or [IBM Fix Central](https://www.ibm.com/support/fixcentral), and place the downloaded file (for example, `IBM_MQ_9.1.1_UBUNTU_X86-64.tar.gz` for MQ V9.1.1 for Ubuntu on x86_64 architecture) in the `downloads` directory 3. Run `make build-advancedserver` > **Warning**: Note that MQ offers two different sets of packaging on Linux: one is called "MQ for Linux" and contains RPM files for installing on Red Hat Enterprise Linux and SUSE Linux Enterprise Server. The other package is called "MQ for Ubuntu", and contains DEB files for installing on Ubuntu. @@ -40,7 +40,7 @@ On a Red Hat Enterprise Linux host, the command `make build-advancedserver` will You can build a different version of MQ by setting the `MQ_VERSION` environment variable, for example: ```bash -MQ_VERSION=9.0.5.0 make build-advancedserver +MQ_VERSION=9.1.0.0 make build-advancedserver ``` If you have an MQ archive file with a different file name, you can specify a particular file (which must be in the `downloads` directory). You should also specify the MQ version, so that the resulting image is tagged correctly, for example: diff --git a/docs/testing.md b/docs/testing.md index e66d0d8..2d39ef6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -25,7 +25,7 @@ make test-advancedserver You can specify the image to use directly by using the `MQ_IMAGE_ADVANCEDSERVER` or `MQ_IMAGE_DEVSERVER` variables, for example: ``` -MQ_IMAGE_ADVANCEDSERVER=mqadvanced-server:9.1.0.0-x86_64-ubuntu-16.04 make test-advancedserver +MQ_IMAGE_ADVANCEDSERVER=mqadvanced-server:9.1.1.0-x86_64-ubuntu-16.04 make test-advancedserver ``` You can pass parameters to `go test` with an environment variable. For example, to run the "TestGoldenPath" test, run the following command:: @@ -34,10 +34,10 @@ You can pass parameters to `go test` with an environment variable. For example, TEST_OPTS_DOCKER="-run TestGoldenPath" make test-advancedserver ``` -You can also use the same environment variables you specified when [building](./building), for example, the following will try and test an image called `mqadvanced-server:9.0.5.0-x86_64-ubuntu-16.04`: +You can also use the same environment variables you specified when [building](./building), for example, the following will try and test an image called `mqadvanced-server:9.1.0.0-x86_64-ubuntu-16.04`: ``` -MQ_VERSION=9.0.5.0 make test-advancedserver +MQ_VERSION=9.1.0.0 make test-advancedserver ``` ### Running the Docker tests with code coverage diff --git a/incubating/Dockerfile-sfbridge b/incubating/Dockerfile-sfbridge index 134cc4f..0815f43 100644 --- a/incubating/Dockerfile-sfbridge +++ b/incubating/Dockerfile-sfbridge @@ -15,7 +15,7 @@ FROM ubuntu:16.04 # The URL to download the MQ installer from in tar.gz format -ARG MQ_URL=https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev903_ubuntu_x86-64.tar.gz +ARG MQ_URL=https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev911_ubuntu_x86-64.tar.gz # The MQ packages to install ARG MQ_PACKAGES="ibmmq-sfbridge" diff --git a/incubating/mq-explorer/Dockerfile b/incubating/mq-explorer/Dockerfile index 0deffcb..9692099 100644 --- a/incubating/mq-explorer/Dockerfile +++ b/incubating/mq-explorer/Dockerfile @@ -15,7 +15,7 @@ FROM ubuntu:16.04 # The URL to download the MQ installer from in tar.gz format -ARG MQ_URL=https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev903_ubuntu_x86-64.tar.gz +ARG MQ_URL=https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev911_ubuntu_x86-64.tar.gz # The MQ packages to install ARG MQ_PACKAGES="ibmmq-explorer" diff --git a/incubating/mq-golang-sdk/Dockerfile b/incubating/mq-golang-sdk/Dockerfile index 584992d..589aad0 100644 --- a/incubating/mq-golang-sdk/Dockerfile +++ b/incubating/mq-golang-sdk/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG BASE_IMAGE=mq-sdk:9.0.5.0-x86_64-ubuntu-16.04 +ARG BASE_IMAGE=mq-sdk:9.1.1.0-x86_64-ubuntu-16.04 FROM $BASE_IMAGE diff --git a/incubating/mqadvanced-server-dev/Dockerfile b/incubating/mqadvanced-server-dev/Dockerfile index f022258..148635a 100644 --- a/incubating/mqadvanced-server-dev/Dockerfile +++ b/incubating/mqadvanced-server-dev/Dockerfile @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG BASE_IMAGE=mqadvanced-server-dev-base:9.0.5.0-x86_64-ubuntu-16.04 -ARG BUILDER_IMAGE=mq-golang-sdk:9.0.5.0-x86_64-ubuntu-16.04 +ARG BASE_IMAGE=mqadvanced-server-dev-base:9.1.1.0-x86_64-ubuntu-16.04 +ARG BUILDER_IMAGE=mq-golang-sdk:9.1.1.0-x86_64-ubuntu-16.04 ############################################################################### # Build stage to build Go code diff --git a/install-build-deps-ubuntu.sh b/install-build-deps-ubuntu.sh index bcb1bea..027a834 100755 --- a/install-build-deps-ubuntu.sh +++ b/install-build-deps-ubuntu.sh @@ -19,10 +19,6 @@ set -ex -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -sudo apt-get update -sudo apt-get -y install docker-ce curl https://glide.sh/get | sh sudo curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 sudo chmod +x /usr/local/bin/dep diff --git a/manifests/dockerhub/manifest-9.1.1.yaml b/manifests/dockerhub/manifest-9.1.1.yaml new file mode 100644 index 0000000..95c154f --- /dev/null +++ b/manifests/dockerhub/manifest-9.1.1.yaml @@ -0,0 +1,29 @@ +# © Copyright IBM Corporation 2018 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +image: ibmcom/mq:9.1.1.0 +manifests: + - image: ibmcom/mq:9.1.1.0-x86_64 + platform: + architecture: amd64 + os: linux + - image: ibmcom/mq:9.1.1.0-ppc64le + platform: + architecture: ppc64le + os: linux + - image: ibmcom/mq:9.1.1.0-s390x + platform: + architecture: s390x + os: linux + diff --git a/manifests/dockerhub/manifest-9.yaml b/manifests/dockerhub/manifest-9.yaml index 540b10c..a6d1426 100644 --- a/manifests/dockerhub/manifest-9.yaml +++ b/manifests/dockerhub/manifest-9.yaml @@ -14,15 +14,15 @@ image: ibmcom/mq:9 manifests: - - image: ibmcom/mq:9.1.0.0-x86_64 + - image: ibmcom/mq:9.1.1.0-x86_64 platform: architecture: amd64 os: linux - - image: ibmcom/mq:9.1.0.0-ppc64le + - image: ibmcom/mq:9.1.1.0-ppc64le platform: architecture: ppc64le os: linux - - image: ibmcom/mq:9.1.0.0-s390x + - image: ibmcom/mq:9.1.1.0-s390x platform: architecture: s390x os: linux diff --git a/manifests/dockerhub/manifest-latest.yaml b/manifests/dockerhub/manifest-latest.yaml index 051eac8..453184d 100644 --- a/manifests/dockerhub/manifest-latest.yaml +++ b/manifests/dockerhub/manifest-latest.yaml @@ -14,15 +14,15 @@ image: ibmcom/mq:latest manifests: - - image: ibmcom/mq:9.1.0.0-x86_64 + - image: ibmcom/mq:9.1.1.0-x86_64 platform: architecture: amd64 os: linux - - image: ibmcom/mq:9.1.0.0-ppc64le + - image: ibmcom/mq:9.1.1.0-ppc64le platform: architecture: ppc64le os: linux - - image: ibmcom/mq:9.1.0.0-s390x + - image: ibmcom/mq:9.1.1.0-s390x platform: architecture: s390x os: linux diff --git a/manifests/dockerstore/manifest-9.1.1.yaml b/manifests/dockerstore/manifest-9.1.1.yaml new file mode 100644 index 0000000..9c79f70 --- /dev/null +++ b/manifests/dockerstore/manifest-9.1.1.yaml @@ -0,0 +1,29 @@ +# © Copyright IBM Corporation 2018 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +image: ibmcorp/mqadvanced-server-dev:9.1.1.0 +manifests: + - image: ibmcorp/mqadvanced-server-dev:9.1.1.0-x86_64 + platform: + architecture: amd64 + os: linux + - image: ibmcorp/mqadvanced-server-dev:9.1.1.0-ppc64le + platform: + architecture: ppc64le + os: linux + - image: ibmcorp/mqadvanced-server-dev:9.1.1.0-s390x + platform: + architecture: s390x + os: linux + From c063ddd67d31c02b277e094c554257dee11e9548 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 3 Dec 2018 10:33:44 +0000 Subject: [PATCH 14/60] remove deprecated tag manifest --- manifests/dockerhub/manifest-9.yaml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 manifests/dockerhub/manifest-9.yaml diff --git a/manifests/dockerhub/manifest-9.yaml b/manifests/dockerhub/manifest-9.yaml deleted file mode 100644 index a6d1426..0000000 --- a/manifests/dockerhub/manifest-9.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# © Copyright IBM Corporation 2018 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -image: ibmcom/mq:9 -manifests: - - image: ibmcom/mq:9.1.1.0-x86_64 - platform: - architecture: amd64 - os: linux - - image: ibmcom/mq:9.1.1.0-ppc64le - platform: - architecture: ppc64le - os: linux - - image: ibmcom/mq:9.1.1.0-s390x - platform: - architecture: s390x - os: linux - From 4145f077b6210c1856fd834abad1a2f016d5862f Mon Sep 17 00:00:00 2001 From: Rob Parker Date: Wed, 5 Dec 2018 13:01:31 +0000 Subject: [PATCH 15/60] update perl-base to fix security vulnerability (#253) --- install-mq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-mq.sh b/install-mq.sh index 7358155..2acde1b 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -139,7 +139,7 @@ rm -rf ${DIR_EXTRACT} # Apply any bug fixes not included in base Ubuntu or MQ image. # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run -$UBUNTU && apt-get install -y libapparmor1 libsystemd0 systemd systemd-sysv libudev1 --only-upgrade +$UBUNTU && apt-get install -y libapparmor1 libsystemd0 systemd systemd-sysv libudev1 perl-base --only-upgrade # End of bug fixes # Clean up cached files From 9c8b3825be31236264407848b144e097b5935b98 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 12 Dec 2018 13:11:47 +0000 Subject: [PATCH 16/60] Add image tag into info output, add ability to print info output on demand and prevent multiple instances of runmqserver being run --- Dockerfile-server | 3 +- Makefile-UBUNTU | 3 +- cmd/runmqserver/main.go | 20 +++++++ cmd/runmqserver/process.go | 63 +++++++++++++++++++++ cmd/runmqserver/version.go | 7 +++ incubating/mqadvanced-server-dev/Dockerfile | 3 +- 6 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 cmd/runmqserver/process.go diff --git a/Dockerfile-server b/Dockerfile-server index ba81c0c..97d70e9 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -22,10 +22,11 @@ FROM $BUILDER_IMAGE as builder WORKDIR /go/src/github.com/ibm-messaging/mq-container/ ARG IMAGE_REVISION="Not specified" ARG IMAGE_SOURCE="Not specified" +ARG IMAGE_TAG="Not specified" COPY cmd/ ./cmd COPY internal/ ./internal COPY vendor/ ./vendor -RUN go build -ldflags "-X \"main.ImageCreated=$(date --iso-8601=seconds)\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\"" ./cmd/runmqserver/ +RUN go build -ldflags "-X \"main.ImageCreated=$(date --iso-8601=seconds)\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\" -X \"main.ImageTag=$IMAGE_TAG\"" ./cmd/runmqserver/ RUN go build ./cmd/chkmqready/ RUN go build ./cmd/chkmqhealthy/ # Run all unit tests diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index 71a76b1..0ca502e 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -205,6 +205,7 @@ define docker-build-mq --build-arg BUILDER_IMAGE=$(MQ_IMAGE_GOLANG_SDK) \ --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" \ --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" \ + --build-arg IMAGE_TAG="$1" \ --label IBM_PRODUCT_ID=$4 \ --label IBM_PRODUCT_NAME=$5 \ --label IBM_PRODUCT_VERSION=$6 \ @@ -236,7 +237,7 @@ build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV) build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk-ex $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER_BASE)"$(END))) $(call docker-build-mq,$(MQ_IMAGE_DEVSERVER_BASE),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) - $(DOCKER) build --tag $(MQ_IMAGE_DEVSERVER) --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" --build-arg BASE_IMAGE=$(MQ_IMAGE_DEVSERVER_BASE) --build-arg BUILDER_IMAGE=$(MQ_IMAGE_GOLANG_SDK) --file incubating/mqadvanced-server-dev/Dockerfile . + $(DOCKER) build --tag $(MQ_IMAGE_DEVSERVER) --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" --build-arg IMAGE_TAG="$(MQ_IMAGE_DEVSERVER)" --build-arg BASE_IMAGE=$(MQ_IMAGE_DEVSERVER_BASE) --build-arg BUILDER_IMAGE=$(MQ_IMAGE_GOLANG_SDK) --file incubating/mqadvanced-server-dev/Dockerfile . .PHONY: build-advancedserver-cover build-advancedserver-cover: docker-version diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 4df645b..01f00d5 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -20,6 +20,7 @@ package main import ( "context" "errors" + "flag" "os" "sync" @@ -29,12 +30,31 @@ import ( ) func doMain() error { + var infoFlag = flag.Bool("info", false, "Display debug info, then exit") + flag.Parse() + + // Configure the logger so we can output messages name, nameErr := name.GetQueueManagerName() mf, err := configureLogger(name) if err != nil { logTermination(err) return err } + + // Check whether they only want debug info + if *infoFlag { + logVersionInfo() + logConfig() + return nil + } + + err = verifySingleProcess() + if err != nil { + // We don't do the normal termination here as it would create a termination file. + log.Error(err) + return err + } + if nameErr != nil { logTermination(err) return err diff --git a/cmd/runmqserver/process.go b/cmd/runmqserver/process.go new file mode 100644 index 0000000..03d6c56 --- /dev/null +++ b/cmd/runmqserver/process.go @@ -0,0 +1,63 @@ +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/ibm-messaging/mq-container/internal/command" +) + +// Verifies that we are the main or only instance of this program +func verifySingleProcess() error { + programName, err := determineExecutable() + if err != nil { + return fmt.Errorf("Failed to determine name of this program - %v", err) + } + + // Verify that there is only one runmqserver + _, err = verifyOnlyOne(programName) + if err != nil { + return fmt.Errorf("You cannot run more than one instance of this program") + } + + return nil +} + +// Verifies that there is only one instance running of the given program name. +func verifyOnlyOne(programName string) (int, error) { + out, _, _ := command.Run("ps", "-e", "--format", "cmd") + //if this goes wrong then assume we are the only one + numOfProg := strings.Count(out, programName) + if numOfProg != 1 { + return numOfProg, fmt.Errorf("Expected there to be only 1 instance of %s but found %d", programName, numOfProg) + } + return numOfProg, nil +} + +// Determines the name of the currently running executable. +func determineExecutable() (string, error) { + file, err := os.Executable() + if err != nil { + return "", err + } + + _, exec := filepath.Split(file) + return exec, nil +} diff --git a/cmd/runmqserver/version.go b/cmd/runmqserver/version.go index 35af770..0142364 100644 --- a/cmd/runmqserver/version.go +++ b/cmd/runmqserver/version.go @@ -29,6 +29,8 @@ var ( ImageRevision = "Not specified" // ImageSource is the URL to get source code for building the image ImageSource = "Not specified" + // ImageTag is the tag of the image + ImageTag = "Not specified" ) func logDateStamp() { @@ -43,6 +45,10 @@ func logGitCommit() { log.Printf("Image source: %v", ImageSource) } +func logImageTag() { + log.Printf("Image tag: %v", ImageTag) +} + func logMQVersion() { mqVersion, _, err := command.Run("dspmqver", "-b", "-f", "2") if err != nil { @@ -67,5 +73,6 @@ func logVersionInfo() { logDateStamp() logGitRepo() logGitCommit() + logImageTag() logMQVersion() } diff --git a/incubating/mqadvanced-server-dev/Dockerfile b/incubating/mqadvanced-server-dev/Dockerfile index 148635a..2c996d8 100644 --- a/incubating/mqadvanced-server-dev/Dockerfile +++ b/incubating/mqadvanced-server-dev/Dockerfile @@ -21,12 +21,13 @@ ARG BUILDER_IMAGE=mq-golang-sdk:9.1.1.0-x86_64-ubuntu-16.04 FROM $BUILDER_IMAGE as builder ARG IMAGE_REVISION="Not specified" ARG IMAGE_SOURCE="Not specified" +ARG IMAGE_TAG="Not specified" WORKDIR /go/src/github.com/ibm-messaging/mq-container/ COPY cmd/ ./cmd COPY internal/ ./internal COPY vendor/ ./vendor # Re-build runmqserver, with code tagged with 'mqdev' enabled -RUN go build -ldflags "-X \"main.ImageCreated=$(date --iso-8601=seconds)\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\"" --tags 'mqdev' ./cmd/runmqserver +RUN go build -ldflags "-X \"main.ImageCreated=$(date --iso-8601=seconds)\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\" -X \"main.ImageTag=$IMAGE_TAG\"" --tags 'mqdev' ./cmd/runmqserver RUN go build ./cmd/runmqdevserver/ # Run all unit tests RUN go test -v ./cmd/runmqdevserver/... From 3e07814bf648b4e757ee69c589e25fb695fe1b07 Mon Sep 17 00:00:00 2001 From: Rob Parker Date: Thu, 10 Jan 2019 10:21:20 +0000 Subject: [PATCH 17/60] update travis build stages to remove duplicate build and label (#263) * update travis build stages to remove duplicate build and label * update copyright to 2019 --- .travis.yml | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0631a76..b240462 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,22 +27,26 @@ cache: directories: - downloads -env: - - BASE_IMAGE=ubuntu:16.04 - - BASE_IMAGE=centos:latest - jobs: include: - - if: type IN (pull_request) - env: DOCKER_DOWNGRADE="docker save -o images.tar mqadvanced-server-dev mq-dev-jms-test && - sudo apt-get autoremove -y docker-ce && - curl -fsSL \"https://apt.dockerproject.org/gpg\" | sudo apt-key add - && - sudo apt-add-repository \"deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -cs) main\" && - sudo apt-get update && - sudo apt-get install docker-engine=1.12.6-0~ubuntu-$(lsb_release -cs) && - docker load -q -i images.tar && - export DOCKER_API_VERSION=\"1.24\"" - - env: DOCKER_DOWNGRADE="echo nothing to be done" + - stage: build and test + env: + - BASE_IMAGE=ubuntu:16.04 + - DOCKER_DOWNGRADE="echo nothing to be done" + - env: + - BASE_IMAGE=centos:latest + - DOCKER_DOWNGRADE="echo nothing to be done" + - if: type IN (pull_request) OR tag IS present + env: + - BASE_IMAGE=ubuntu:16.04 + - DOCKER_DOWNGRADE="docker save -o images.tar mqadvanced-server-dev mq-dev-jms-test && + sudo apt-get autoremove -y docker-ce && + curl -fsSL \"https://apt.dockerproject.org/gpg\" | sudo apt-key add - && + sudo apt-add-repository \"deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -cs) main\" && + sudo apt-get update && + sudo apt-get install docker-engine=1.12.6-0~ubuntu-$(lsb_release -cs) && + docker load -q -i images.tar && + export DOCKER_API_VERSION=\"1.24\"" before_install: - ./install-build-deps-ubuntu.sh From 525ff82fe799788ad5c716af5f44300c9cdec888 Mon Sep 17 00:00:00 2001 From: LPowlett <44359672+LPowlett@users.noreply.github.com> Date: Thu, 10 Jan 2019 13:41:43 +0000 Subject: [PATCH 18/60] Error checking invalid mqsc commands (#261) * testRepeatingMqsc * testInvalidMqsc * 9.1.1 Makefile * testRepeatingMqsc * testInvalidMqsc * update copyright 2019 * update invalid mqsc error message * update changelog to reflect MQSC changes --- CHANGELOG.md | 7 +++- LICENSE | 2 +- cmd/runmqserver/qmgr.go | 5 +-- .../mqadvanced-server-dev/10-dev.mqsc.tpl | 6 ++-- test/docker/docker_api_test.go | 32 ++++++++++++++++++- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 042c004..bcd410f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,16 @@ # Change log +## vNext + +* [New IGNSTATE parameter for runmqsc START and STOP commands](https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_9.1.0/com.ibm.mq.pro.doc/q132310_.htm#q132310___ignstateparm) - From MQ version 9.1.1.0, any MQSC scripts included in the image should make use of the `IGNSTATE(YES)` parameter on any `START` and `STOP` commands. This allows for consistency when executing scripts multiple times (e.g. when a container is restarted) + + ## 9.1.1.0 (2018-11-30) * Updated to MQ version 9.1.1.0 * Created seperate RedHat Makefile for building images on RedHat machines with buildah * Enabled REST messaging capability for app user. -* Added support for container suplimentary groups +* Added support for container supplementary groups * Removed IBM MQ version 9.0.5 details. * Added additional Diagnostics ([#203](https://github.com/ibm-messaging/mq-container/pull/203)) * Implementted GOSec to perform code scans for security vulnerabilities. (([#227](https://github.com/ibm-messaging/mq-container/pull/227))) diff --git a/LICENSE b/LICENSE index f2249fd..94482e6 100644 --- a/LICENSE +++ b/LICENSE @@ -176,7 +176,7 @@ END OF TERMS AND CONDITIONS - © Copyright IBM Corporation. 2015, 2018 + © Copyright IBM Corporation. 2015, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index ffabdc9..8d6e76c 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -119,7 +119,8 @@ func configureQueueManager() error { // Run the command and wait for completion out, err := cmd.CombinedOutput() if err != nil { - log.Println(err) + log.Errorf("Error running MQSC file %v: %v", file.Name(), err) + return err } // Print the runmqsc output, adding tab characters to make it more readable as part of the log log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1)) diff --git a/incubating/mqadvanced-server-dev/10-dev.mqsc.tpl b/incubating/mqadvanced-server-dev/10-dev.mqsc.tpl index dd46bea..987e8e3 100644 --- a/incubating/mqadvanced-server-dev/10-dev.mqsc.tpl +++ b/incubating/mqadvanced-server-dev/10-dev.mqsc.tpl @@ -1,4 +1,4 @@ -* © Copyright IBM Corporation 2017, 2018 +* © Copyright IBM Corporation 2017, 2019 * * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. -STOP LISTENER('SYSTEM.LISTENER.TCP.1') +STOP LISTENER('SYSTEM.LISTENER.TCP.1') IGNSTATE(YES) * Developer queues DEFINE QLOCAL('DEV.QUEUE.1') REPLACE @@ -50,4 +50,4 @@ SET AUTHREC PROFILE('DEV.**') GROUP('mqclient') OBJTYPE(TOPIC) AUTHADD(PUB,SUB) * Developer listener DEFINE LISTENER('DEV.LISTENER.TCP') TRPTYPE(TCP) PORT(1414) CONTROL(QMGR) REPLACE -START LISTENER('DEV.LISTENER.TCP') +START LISTENER('DEV.LISTENER.TCP') IGNSTATE(YES) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index aaa26e2..7c76aea 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -411,6 +411,36 @@ func TestMQSC(t *testing.T) { } } +// TestInvalidMQSC creates a new image with an MQSC file containing invalid MQSC, +// tries to start a container based on that image, and checks that container terminates +func TestInvalidMQSC(t *testing.T) { + t.Parallel() + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + var files = []struct { + Name, Body string + }{ + {"Dockerfile", fmt.Sprintf("FROM %v\nRUN rm -f /etc/mqm/*.mqsc\nADD mqscTest.mqsc /etc/mqm/", imageName())}, + {"mqscTest.mqsc", "DEFINE INVALIDLISTENER('TEST.LISTENER.TCP') TRPTYPE(TCP) PORT(1414) CONTROL(QMGR) REPLACE"}, + } + tag := createImage(t, cli, files) + defer deleteImage(t, cli, tag) + + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + Image: tag, + } + id := runContainer(t, cli, &containerConfig) + defer cleanContainer(t, cli, id) + rc := waitForContainer(t, cli, id, 5) + if rc != 1 { + t.Errorf("Expected rc=1, got rc=%v", rc) + } + expectTerminationMessage(t) +} + // TestReadiness creates a new image with large amounts of MQSC in, to // ensure that the readiness check doesn't pass until configuration has finished. // WARNING: This test is sensitive to the speed of the machine it's running on. From be11b3cda1883bf09847bed42edd39a40dab1767 Mon Sep 17 00:00:00 2001 From: LPowlett <44359672+LPowlett@users.noreply.github.com> Date: Thu, 10 Jan 2019 14:55:22 +0000 Subject: [PATCH 19/60] Dependancy upgrades (#264) * dependancy upgrades * revert dep to 0.4.1 * revert file changes --- install-build-deps-ubuntu.sh | 2 +- test/docker/Gopkg.toml | 2 +- test/messaging/pom.xml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/install-build-deps-ubuntu.sh b/install-build-deps-ubuntu.sh index 027a834..eab7577 100755 --- a/install-build-deps-ubuntu.sh +++ b/install-build-deps-ubuntu.sh @@ -20,7 +20,7 @@ set -ex curl https://glide.sh/get | sh -sudo curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 +sudo curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64 sudo chmod +x /usr/local/bin/dep go get golang.org/x/lint/golint diff --git a/test/docker/Gopkg.toml b/test/docker/Gopkg.toml index c29f50e..9dbde48 100644 --- a/test/docker/Gopkg.toml +++ b/test/docker/Gopkg.toml @@ -18,7 +18,7 @@ [[constraint]] name = "github.com/docker/go-connections" - version = "0.3.0" + version = "0.4.0" [prune] go-tests = true diff --git a/test/messaging/pom.xml b/test/messaging/pom.xml index 6644de3..90a3a99 100644 --- a/test/messaging/pom.xml +++ b/test/messaging/pom.xml @@ -32,19 +32,19 @@ limitations under the License. org.junit.jupiter junit-jupiter-api - 5.2.0 + 5.3.2 compile org.junit.jupiter junit-jupiter-engine - 5.2.0 + 5.3.2 runtime org.junit.platform junit-platform-console-standalone - 1.2.0 + 1.3.2 runtime From 2bfdd51a013997eb7ea88c83abaa88582b84b259 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Thu, 24 Jan 2019 13:51:10 +0000 Subject: [PATCH 20/60] RHEL build uses redist client and RHEL minimal --- Makefile-RHEL | 16 ++++---- mq-advanced-server-rhel/go-build.sh | 8 ++-- mq-advanced-server-rhel/go-buildah.sh | 8 ++-- mq-advanced-server-rhel/mq-buildah.sh | 30 +++++++++------ .../mq-golang-sdk-buildah.sh | 37 +++++++------------ 5 files changed, 47 insertions(+), 52 deletions(-) diff --git a/Makefile-RHEL b/Makefile-RHEL index 11f552c..af0bd47 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,8 +27,8 @@ MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION)_LINUX_$(MQ_ARCHIVE_ARCH).tar.gz # MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced # for Developers can be installed MQ_ARCHIVE_DEV ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) -# MQ_SDK_ARCHIVE specifies the archive to use for building the golang programs. Defaults vary on developer or advanced. -MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) +# MQ_SDK_ARCHIVE specifies the archive to use for the MQ redistributable client, which is used for building the golang programs. +MQ_SDK_ARCHIVE ?= $(MQ_VERSION)-IBM-MQC-Redist-LinuxX64.tar.gz # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image @@ -78,7 +78,7 @@ endif # Archive names for IBM MQ Advanced for Developers MQ_ARCHIVE_DEV_9.0.5.0=mqadv_dev905_linux_x86-64.tar.gz MQ_ARCHIVE_DEV_9.1.0.0=mqadv_dev910_linux_$(MQ_DEV_ARCH).tar.gz -MQ_ARCHIVE_DEV_9.1.1.0=mqadv_dev910_linux_$(MQ_DEV_ARCH).tar.gz +MQ_ARCHIVE_DEV_9.1.1.0=mqadv_dev911_linux_$(MQ_DEV_ARCH).tar.gz ############################################################################### # Build targets @@ -113,9 +113,9 @@ downloads/$(MQ_ARCHIVE_DEV): cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE): - $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END))) + $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced redistributable client "$(MQ_VERSION)$(END))) mkdir -p downloads - cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_SDK_ARCHIVE) + cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist/$(MQ_SDK_ARCHIVE) .PHONY: downloads downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE) @@ -133,7 +133,7 @@ check-prereqs: yum list | grep yum-utils || (echo "Missing required package yum-utils" && exit 1) .PHONY: check-test-prereqs -check-prereqs: +check-test-prereqs: $(info $(SPACER)$(shell printf $(TITLE)"Checking for prereqs"$(END))) which buildah || (echo "Missing required program buildah" && exit 1) which docker || (echo "Missing required program docker" && exit 1) @@ -155,14 +155,12 @@ test-devserver: check-test-prereqs test/docker/vendor .PHONY: build-advancedserver -build-advancedserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE) build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs-ex $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END))) sudo mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_ADVANCEDSERVER)" "$(MQ_VERSION)" "$(MQDEV)" .PHONY: build-devserver -build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV) build-devserver: MQDEV=TRUE build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs-ex diff --git a/mq-advanced-server-rhel/go-build.sh b/mq-advanced-server-rhel/go-build.sh index 831ba41..eeb17f8 100755 --- a/mq-advanced-server-rhel/go-build.sh +++ b/mq-advanced-server-rhel/go-build.sh @@ -1,6 +1,6 @@ #!/bin/bash # -*- mode: sh -*- -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,9 +17,11 @@ # Builds and tests the golang programs used by the MQ image. -set -e +set -ex -cd $GOPATH/src/github.com/ibm-messaging/mq-container/ +# Handle a GOPATH with multiple entries (just choose the first one) +IFS=':' read -ra DIR <<< "$GOPATH" +cd ${DIR[0]}/src/github.com/ibm-messaging/mq-container/ # Build and test the Go code mkdir -p build diff --git a/mq-advanced-server-rhel/go-buildah.sh b/mq-advanced-server-rhel/go-buildah.sh index 9e9e63d..9a8d6a0 100755 --- a/mq-advanced-server-rhel/go-buildah.sh +++ b/mq-advanced-server-rhel/go-buildah.sh @@ -1,6 +1,6 @@ #!/bin/bash # -*- mode: sh -*- -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,11 +35,11 @@ IMAGE_REVISION=${IMAGE_REVISION:="Not Applicable"} IMAGE_SOURCE=${IMAGE_SOURCE:="Not Applicable"} podman run \ - --volume ${PWD}:/go/src/github.com/ibm-messaging/mq-container/ \ - --env GOPATH=/go \ + --volume ${PWD}:/opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/ \ --env IMAGE_REVISION="$IMAGE_REVISION" \ --env IMAGE_SOURCE="$IMAGE_SOURCE" \ --env MQDEV=${dev} \ + --user $(id -u) \ --rm \ ${tag} \ - bash -c "cd /go/src/github.com/ibm-messaging/mq-container/ && ./mq-advanced-server-rhel/go-build.sh" + bash -c "cd /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/ && ./mq-advanced-server-rhel/go-build.sh" diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 719ca36..0c138eb 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -1,6 +1,6 @@ #!/bin/bash # -*- mode: sh -*- -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,13 +17,13 @@ # Build a RHEL image, using the buildah tool # Usage -# mq-buildah.sh ARCHIVEFILE PACKAGES +# mq-buildah.sh ARCHIVE-NAME PACKAGES set -x set -e function usage { - echo "Usage: $0 ARCHIVENAME PACKAGES TAG VERSION MQDevFlag" + echo "Usage: $0 ARCHIVE-NAME PACKAGES TAG VERSION MQDevFlag" exit 20 } @@ -36,7 +36,8 @@ fi # Setup MQ server working container ############################################################################### -readonly ctr_mq=$(buildah from rhel7) +# Use RHEL 7 minimal container (which doesn't include things like Python or Yum) +readonly ctr_mq=$(buildah from rhel7-minimal) if [ -z "$ctr_mq" ] then echo "ERROR: ctr_mq is empty. Check above output for errors" @@ -60,13 +61,10 @@ readonly mqdev=$5 # Install MQ server ############################################################################### -groupadd --root ${mnt_mq} --system --gid 888 mqm -useradd --root ${mnt_mq} --system --uid 888 --gid mqm mqm -usermod --root ${mnt_mq} -aG root mqm -usermod --root ${mnt_mq} -aG mqm root - +# Use the Yum repositories configured on the host +cp /etc/yum.repos.d/* ${mnt_mq}/etc/yum.repos.d/ # Install the packages required by MQ -buildah run $ctr_mq -- yum install -y --setopt install_weak_deps=false --setopt=tsflags=nodocs --setopt=override_install_langs=en_US.utf8 \ +yum install -y --installroot=${mnt_mq} --setopt install_weak_deps=false --setopt=tsflags=nodocs --setopt=override_install_langs=en_US.utf8 \ bash \ bc \ coreutils \ @@ -78,12 +76,20 @@ buildah run $ctr_mq -- yum install -y --setopt install_weak_deps=false --setopt= passwd \ procps-ng \ sed \ + shadow-utils \ tar \ - util-linux + util-linux \ + which + +groupadd --root ${mnt_mq} --system --gid 888 mqm +useradd --root ${mnt_mq} --system --uid 888 --gid mqm mqm +usermod --root ${mnt_mq} -aG root mqm +usermod --root ${mnt_mq} -aG mqm root # Clean up cached files -buildah run $ctr_mq -- yum clean all +yum clean --installroot=${mnt_mq} all rm -rf ${mnt_mq}/var/cache/yum/* +rm -rf ${mnt_mq}/etc/yum.repos.d/* # Install MQ server packages into the MQ builder image ./mq-advanced-server-rhel/install-mq-rhel.sh ${ctr_mq} "${mnt_mq}" "${archive}" "${packages}" diff --git a/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh b/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh index 7874128..fbca330 100755 --- a/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh +++ b/mq-advanced-server-rhel/mq-golang-sdk-buildah.sh @@ -1,6 +1,6 @@ #!/bin/bash # -*- mode: sh -*- -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ set -ex function usage { - echo "Usage: $0 ARCHIVENAME TAG" + echo "Usage: $0 REDIST-ARCHIVE-NAME TAG" exit 20 } @@ -29,41 +29,30 @@ if [ "$#" -ne 2 ]; then usage fi -readonly mq_archive=downloads/$1 +readonly mq_redist_archive=downloads/$1 readonly tag=$2 -# Use plain RHEL 7 container -# Note: Red Hat's devtools/go-toolset-7-rhel7 image doesn't allow use of 'root' -# user required for installing the MQ SDK -readonly ctr_mq=$(buildah from rhel7) +# Use Red Hat's Go toolset image as the base +readonly ctr_mq=$(buildah from devtools/go-toolset-7-rhel7) if [ -z "$ctr_mq" ] then echo "ERROR: ctr_mq is empty. Check above output for errors" exit 50 fi -readonly mnt_mq=$(buildah mount $ctr_mq) -if [ -z "$mnt_mq" ] +readonly mnt_mq_go=$(buildah mount $ctr_mq) +if [ -z "$mnt_mq_go" ] then - echo "ERROR: mnt_mq is empty. Check above output for errors" + echo "ERROR: mnt_mq_go is empty. Check above output for errors" exit 50 fi -# Add mqm user -sudo groupadd --root $mnt_mq --system --gid 888 mqm -sudo useradd --root $mnt_mq --system --uid 888 --gid mqm mqm -sudo usermod --root $mnt_mq -aG root mqm -sudo usermod --root $mnt_mq -aG mqm root +# Install the MQ redistributable client (including header files) into the Go builder image +mkdir -p ${mnt_mq_go}/opt/mqm +tar -xzf ${mq_redist_archive} -C ${mnt_mq_go}/opt/mqm -# Enable Yum repository for "optional" RPMs, which is needed for "golang" -buildah run ${ctr_mq} -- yum-config-manager --enable rhel-7-server-optional-rpms -# Install Go compiler -buildah run ${ctr_mq} -- yum install -y golang git gcc - -# Install the MQ SDK into the Go builder image -./mq-advanced-server-rhel/install-mq-rhel.sh ${ctr_mq} "${mnt_mq}" "${mq_archive}" "MQSeriesRuntime-*.rpm MQSeriesSDK-*.rpm MQSeriesSamples*.rpm" # Clean up Yum files -buildah run ${ctr_mq} -- yum clean all --releasever 7 -rm -rf ${mnt_mq}/var/cache/yum/* +rm -rf ${mnt_mq_go}/etc/yum.repos.d/* + buildah unmount ${ctr_mq} # Set environment variables for MQ/Go compilation buildah config \ From df6ce917c2253c3ff80e725444d24844c7fc7a1c Mon Sep 17 00:00:00 2001 From: LPowlett <44359672+LPowlett@users.noreply.github.com> Date: Mon, 28 Jan 2019 11:03:54 +0000 Subject: [PATCH 21/60] endmqm -r try to reconnect (#268) --- cmd/runmqserver/qmgr.go | 2 +- test/docker/mqmetric_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index 8d6e76c..0082e14 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -131,7 +131,7 @@ func configureQueueManager() error { func stopQueueManager(name string) error { log.Println("Stopping queue manager") - out, _, err := command.Run("endmqm", "-w", name) + out, _, err := command.Run("endmqm", "-w", "-r", name) if err != nil { log.Printf("Error stopping queue manager: %v", string(out)) return err diff --git a/test/docker/mqmetric_test.go b/test/docker/mqmetric_test.go index cb3d67e..6f08056 100644 --- a/test/docker/mqmetric_test.go +++ b/test/docker/mqmetric_test.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -279,7 +279,7 @@ func TestQMRestart(t *testing.T) { // Restart just the QM (to simulate a lost connection) t.Log("Stopping queue manager\n") - rc, out := execContainer(t, cli, id, "mqm", []string{"endmqm", "-w", defaultMetricQMName}) + rc, out := execContainer(t, cli, id, "mqm", []string{"endmqm", "-w", "-r", defaultMetricQMName}) if rc != 0 { t.Fatalf("Failed to stop the queue manager. rc=%d, err=%s", rc, out) } From 43676049b7696b464851a639d857c828aeac52e0 Mon Sep 17 00:00:00 2001 From: LPowlett <44359672+LPowlett@users.noreply.github.com> Date: Mon, 4 Feb 2019 10:25:55 +0000 Subject: [PATCH 22/60] RHEL Security Vulnerability Test (#270) * check for security vulnerabilities on rhel * import * check host is red hat * filepath join * imports --- docs/testing.md | 6 +++++ test/docker/docker_api_test.go | 43 ++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 2d39ef6..ba398b4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -8,6 +8,12 @@ You need to ensure you have the following tools installed: * [dep](https://github.com/golang/dep) (official Go dependency management tool) - needed to prepare for running the tests * [Helm](https://helm.sh) - only needed for running the Kubernetes tests +### Prerequisites for testing a RedHat image +If you want to test a container image with Red Hat Enterprise Linux as the base OS, then you need to use a host server with Red Hat Enterprise Linux. You must also have the following tools installed: + +* [Yum](http://yum.baseurl.org/) (available in `rhel-7-server-extras`) +* [Buildah](https://buildah.io) (available in `rhel-7-server-extras`) + ## Running the tests There are two main sets of tests: diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 7c76aea..190fbc0 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -34,6 +34,8 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" + + "github.com/ibm-messaging/mq-container/internal/command" ) func TestLicenseNotSet(t *testing.T) { @@ -106,9 +108,9 @@ func goldenPath(t *testing.T, metric bool) { stopContainer(t, cli, id) } -// TestSecurityVulnerabilities checks for any vulnerabilities in the image, as reported +// TestSecurityVulnerabilitiesUbuntu checks for any vulnerabilities in the image, as reported // by Ubuntu -func TestSecurityVulnerabilities(t *testing.T) { +func TestSecurityVulnerabilitiesUbuntu(t *testing.T) { t.Parallel() cli, err := client.NewEnvClient() if err != nil { @@ -135,6 +137,43 @@ func TestSecurityVulnerabilities(t *testing.T) { } } +// TestSecurityVulnerabilitiesRedHat checks for any vulnerabilities in the image, as reported +// by Red Hat +func TestSecurityVulnerabilitiesRedHat(t *testing.T) { + t.Parallel() + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + _, ret, _ := command.Run("bash", "-c", "test -f /etc/redhat-release") + if ret != 0 { + t.Skip("Skipping test because host is not RedHat-based") + } + rc, _ := runContainerOneShot(t, cli, "bash", "-c", "test -f /etc/redhat-release") + if rc != 0 { + t.Skip("Skipping test because container is not RedHat-based") + } + id, _, err := command.Run("buildah", "from", imageName()) + if err != nil { + t.Fatal(err) + } + id = strings.TrimSpace(id) + defer command.Run("buildah", "rm", id) + mnt, _, err := command.Run("buildah", "mount", id) + if err != nil { + t.Fatal(err) + } + mnt = strings.TrimSpace(mnt) + _, _, err = command.Run("bash", "-c", "cp /etc/yum.repos.d/* "+ filepath.Join(mnt, "/etc/yum.repos.d/")) + if err != nil { + t.Fatal(err) + } + out, ret, _ := command.Run("bash", "-c", "yum --installroot="+mnt+" updateinfo list sec | grep /Sec") + if ret != 1{ + t.Errorf("Expected no vulnerabilities, found the following:\n%v", out) + } +} + func utilTestNoQueueManagerName(t *testing.T, hostName string, expectedName string) { search := "QMNAME(" + expectedName + ")" cli, err := client.NewEnvClient() From f3c858184f74f22fd9ef82f18d89a74849b6c847 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Feb 2019 13:51:15 +0000 Subject: [PATCH 23/60] use correct download files in RHEL build process --- Makefile-RHEL | 31 ++++++++++++++++++++----------- Makefile-UBUNTU | 8 ++++++-- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Makefile-RHEL b/Makefile-RHEL index af0bd47..ab1b2bf 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -23,7 +23,7 @@ MQ_VERSION ?= 9.1.1.0 # MQ_ARCHIVE is the name of the file, under the downloads directory, from which MQ Advanced can # be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture # Does not apply to MQ Advanced for Developers. -MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION)_LINUX_$(MQ_ARCHIVE_ARCH).tar.gz +MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION_VRM)_LINUX_$(MQ_ARCHIVE_ARCH).tar.gz # MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced # for Developers can be installed MQ_ARCHIVE_DEV ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) @@ -56,6 +56,10 @@ DEV_JMS_IMAGE=mq-dev-jms-test:latest IMAGE_REVISION=$(shell git rev-parse HEAD) IMAGE_SOURCE=$(shell git config --get remote.origin.url) MQDEV= +EMPTY:= +SPACE:= $(EMPTY) $(EMPTY) +# MQ_VERSION_VRM is MQ_VERSION with only the Version, Release and Modifier fields (no Fix field). e.g. 9.1.1 instead of 9.1.1.0 +MQ_VERSION_VRM=$(subst $(SPACE),.,$(wordlist 1,3,$(subst .,$(SPACE),$(MQ_VERSION)))) ifneq (,$(findstring Microsoft,$(shell uname -r))) @@ -155,7 +159,7 @@ test-devserver: check-test-prereqs test/docker/vendor .PHONY: build-advancedserver -build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs-ex +build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END))) sudo mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_ADVANCEDSERVER)" "$(MQ_VERSION)" "$(MQDEV)" @@ -163,25 +167,19 @@ build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs-ex .PHONY: build-devserver build-devserver: MQDEV=TRUE build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm -build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs-ex +build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER)"$(END))) sudo mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE_DEV)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_VERSION)" "$(MQDEV)" sudo mq-advanced-server-rhel/mqdev-buildah.sh "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_IMAGE_DEVSERVER)" "$(MQ_VERSION)" .PHONY: build-mqgolang-sdk -build-mqgolang-sdk: check-prereqs downloads/$(MQ_SDK_ARCHIVE) build-mqgolang-sdk-ex - -.PHONY: build-mqgolang-sdk-ex -build-mqgolang-sdk-ex: +build-mqgolang-sdk: check-prereqs downloads/$(MQ_SDK_ARCHIVE) $(info $(SPACER)$(shell printf $(TITLE)"Build mq-golang SDK"$(END))) sudo mq-advanced-server-rhel/mq-golang-sdk-buildah.sh "$(MQ_SDK_ARCHIVE)" "$(MQ_IMAGE_GOLANG_SDK)" .PHONY: build-go-programs -build-go-programs: check-prereqs downloads/$(MQ_SDK_ARCHIVE) build-go-programs-ex - -.PHONY: build-go-programs-ex -build-go-programs-ex: build-mqgolang-sdk-ex +build-go-programs: check-prereqs downloads/$(MQ_SDK_ARCHIVE) build-mqgolang-sdk $(info $(SPACER)$(shell printf $(TITLE)"Build go programs"$(END))) IMAGE_REVISION=$(IMAGE_REVISION) IMAGE_SOURCE=$(IMAGE_SOURCE) sudo mq-advanced-server-rhel/go-buildah.sh "$(MQ_IMAGE_GOLANG_SDK)" "$(MQDEV)" @@ -192,4 +190,15 @@ build-devjmstest: check-test-prereqs sudo buildah push $(DEV_JMS_IMAGE) docker-daemon:$(DEV_JMS_IMAGE) docker tag docker.io/$(DEV_JMS_IMAGE) $(DEV_JMS_IMAGE) +.PHONY: debug-vars +debug-vars: + @echo MQ_VERSION=$(MQ_VERSION) + @echo MQ_VERSION_VRM=$(MQ_VERSION_VRM) + @echo MQ_ARCHIVE=$(MQ_ARCHIVE) + @echo MQ_SDK_ARCHIVE=$(MQ_SDK_ARCHIVE) + @echo MQ_IMAGE_GOLANG_SDK=$(MQ_IMAGE_GOLANG_SDK) + @echo MQ_IMAGE_DEVSERVER_BASE=$(MQ_IMAGE_DEVSERVER_BASE) + @echo MQ_IMAGE_DEVSERVER=$(MQ_IMAGE_DEVSERVER) + @echo MQ_IMAGE_ADVANCEDSERVER=$(MQ_IMAGE_ADVANCEDSERVER) + include formatting.mk diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index 0ca502e..d64d152 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2017, 2018 +# © Copyright IBM Corporation 2017, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ MQ_VERSION ?= 9.1.1.0 # MQ_ARCHIVE is the name of the file, under the downloads directory, from which MQ Advanced can # be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture # Does not apply to MQ Advanced for Developers. -MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION)_$(MQ_ARCHIVE_TYPE)_$(MQ_ARCHIVE_ARCH).tar.gz +MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION_VRM)_$(MQ_ARCHIVE_TYPE)_$(MQ_ARCHIVE_ARCH).tar.gz # MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced # for Developers can be installed MQ_ARCHIVE_DEV ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) @@ -62,6 +62,10 @@ DEV_JMS_IMAGE=mq-dev-jms-test # Variables for versioning IMAGE_REVISION=$(shell git rev-parse HEAD) IMAGE_SOURCE=$(shell git config --get remote.origin.url) +EMPTY:= +SPACE:= $(EMPTY) $(EMPTY) +# MQ_VERSION_VRM is MQ_VERSION with only the Version, Release and Modifier fields (no Fix field). e.g. 9.1.1 instead of 9.1.1.0 +MQ_VERSION_VRM=$(subst $(SPACE),.,$(wordlist 1,3,$(subst .,$(SPACE),$(MQ_VERSION)))) ifneq (,$(findstring Microsoft,$(shell uname -r))) DOWNLOADS_DIR=$(patsubst /mnt/c%,C:%,$(realpath ./downloads/)) From 9a8ff9b524632dc2450fd490c5edc19e83a2df43 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Feb 2019 13:53:59 +0000 Subject: [PATCH 24/60] copy sub-directories under /etc/yum.repos.d --- mq-advanced-server-rhel/mq-buildah.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 0c138eb..8df50a0 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -62,7 +62,7 @@ readonly mqdev=$5 ############################################################################### # Use the Yum repositories configured on the host -cp /etc/yum.repos.d/* ${mnt_mq}/etc/yum.repos.d/ +cp -R /etc/yum.repos.d/* ${mnt_mq}/etc/yum.repos.d/ # Install the packages required by MQ yum install -y --installroot=${mnt_mq} --setopt install_weak_deps=false --setopt=tsflags=nodocs --setopt=override_install_langs=en_US.utf8 \ bash \ From 1b8c816f577ecb055b30751b6a5274f975ac4934 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 5 Feb 2019 15:54:31 +0000 Subject: [PATCH 25/60] fix dockerfile reference to build explorer --- Makefile-UBUNTU | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index 0ca502e..7c73fb5 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -245,7 +245,7 @@ build-advancedserver-cover: docker-version .PHONY: build-explorer build-explorer: downloads/$(MQ_ARCHIVE_DEV) docker-pull - $(call docker-build-mq,mq-explorer:latest-$(ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) + $(call docker-build-mq,mq-explorer:latest-$(ARCH),incubating/mq-explorer/Dockerfile,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) .PHONY: build-sdk build-sdk: downloads/$(MQ_SDK_ARCHIVE) build-sdk-ex From 6840a575f94eb42d6da4cced84750a280197a3b3 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 6 Feb 2019 13:29:44 +0000 Subject: [PATCH 26/60] Fix for building on subscribed and unsubscribed hosts --- mq-advanced-server-rhel/go-buildah.sh | 1 + mq-advanced-server-rhel/mq-buildah.sh | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/mq-advanced-server-rhel/go-buildah.sh b/mq-advanced-server-rhel/go-buildah.sh index 9a8d6a0..b010d7b 100755 --- a/mq-advanced-server-rhel/go-buildah.sh +++ b/mq-advanced-server-rhel/go-buildah.sh @@ -41,5 +41,6 @@ podman run \ --env MQDEV=${dev} \ --user $(id -u) \ --rm \ + --network podman \ ${tag} \ bash -c "cd /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/ && ./mq-advanced-server-rhel/go-build.sh" diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 8df50a0..22b5177 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -61,10 +61,18 @@ readonly mqdev=$5 # Install MQ server ############################################################################### -# Use the Yum repositories configured on the host -cp -R /etc/yum.repos.d/* ${mnt_mq}/etc/yum.repos.d/ -# Install the packages required by MQ -yum install -y --installroot=${mnt_mq} --setopt install_weak_deps=false --setopt=tsflags=nodocs --setopt=override_install_langs=en_US.utf8 \ +microdnf_opts="--nodocs" +# Check whether the host is registered with Red Hat +subscription-manager status +if [ "$?" -eq "0" ]; then + # Host is subscribed, but the minimal image has no enabled repos + microdnf_opts="${microdnf_opts} --enablerepo=rhel-7-server-rpms" + # buildah run ${ctr_mq} -- microdnf microdnf --enablerepo=rhel-7-server-rpms --nodocs install ${prereq_packages} +else + # Use the Yum repositories configured on the host + cp -R /etc/yum.repos.d/* ${mnt_mq}/etc/yum.repos.d/ +fi +buildah run ${ctr_mq} -- microdnf ${microdnf_opts} install \ bash \ bc \ coreutils \ @@ -81,16 +89,15 @@ yum install -y --installroot=${mnt_mq} --setopt install_weak_deps=false --setopt util-linux \ which +# Clean up cached files +buildah run ${ctr_mq} -- microdnf ${microdnf_opts} clean all +rm -rf ${mnt_mq}/etc/yum.repos.d/* + groupadd --root ${mnt_mq} --system --gid 888 mqm useradd --root ${mnt_mq} --system --uid 888 --gid mqm mqm usermod --root ${mnt_mq} -aG root mqm usermod --root ${mnt_mq} -aG mqm root -# Clean up cached files -yum clean --installroot=${mnt_mq} all -rm -rf ${mnt_mq}/var/cache/yum/* -rm -rf ${mnt_mq}/etc/yum.repos.d/* - # Install MQ server packages into the MQ builder image ./mq-advanced-server-rhel/install-mq-rhel.sh ${ctr_mq} "${mnt_mq}" "${archive}" "${packages}" From 599f5f4b5385eb9c70e6c0efcfbf959fa7ab947d Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 29 Jan 2019 11:13:40 +0000 Subject: [PATCH 27/60] Build on host with SELinux enabled --- mq-advanced-server-rhel/go-buildah.sh | 4 +++- mq-advanced-server-rhel/install-mq-rhel.sh | 7 ++++--- mq-advanced-server-rhel/mq-buildah.sh | 18 +++++++++--------- mq-advanced-server-rhel/mqdev-buildah.sh | 15 ++++++--------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/mq-advanced-server-rhel/go-buildah.sh b/mq-advanced-server-rhel/go-buildah.sh index b010d7b..c0d411f 100755 --- a/mq-advanced-server-rhel/go-buildah.sh +++ b/mq-advanced-server-rhel/go-buildah.sh @@ -34,8 +34,10 @@ readonly dev=$2 IMAGE_REVISION=${IMAGE_REVISION:="Not Applicable"} IMAGE_SOURCE=${IMAGE_SOURCE:="Not Applicable"} +# Run the build in a container +# Note the ":Z" on the volume is to allow the container to access the files when SELinux is enabled podman run \ - --volume ${PWD}:/opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/ \ + --volume ${PWD}:/opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/:Z \ --env IMAGE_REVISION="$IMAGE_REVISION" \ --env IMAGE_SOURCE="$IMAGE_SOURCE" \ --env MQDEV=${dev} \ diff --git a/mq-advanced-server-rhel/install-mq-rhel.sh b/mq-advanced-server-rhel/install-mq-rhel.sh index e880fc0..9396bbe 100755 --- a/mq-advanced-server-rhel/install-mq-rhel.sh +++ b/mq-advanced-server-rhel/install-mq-rhel.sh @@ -46,9 +46,10 @@ fi # Accept the MQ license -buildah run --volume ${dir_extract}:/mnt/mq-download $ctr_mq -- /mnt/mq-download/MQServer/mqlicense.sh -text_only -accept +buildah run --user root --volume ${dir_extract}:/mnt/mq-download:Z $ctr_mq -- /mnt/mq-download/MQServer/mqlicense.sh -text_only -accept -buildah run --volume ${dir_extract}:/mnt/mq-download $ctr_mq -- bash -c "cd /mnt/mq-download/MQServer && rpm -ivh $mq_packages" +# Install MQ +buildah run --user root --volume ${dir_extract}:/mnt/mq-download:Z $ctr_mq -- bash -c "cd /mnt/mq-download/MQServer && rpm -ivh $mq_packages" rm -rf ${dir_extract}/MQServer @@ -71,7 +72,7 @@ rm -rf $mnt_mq/var/mqm mkdir -p $mnt_mq/mnt/mqm # Create a symlink for /var/mqm -> /mnt/mqm/data -buildah run $ctr_mq -- ln -s /mnt/mqm/data /var/mqm +buildah run --user root $ctr_mq -- ln -s /mnt/mqm/data /var/mqm # Optional: Set these values for the IBM Cloud Vulnerability Report sed -i 's/PASS_MAX_DAYS\t99999/PASS_MAX_DAYS\t90/' $mnt_mq/etc/login.defs diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 22b5177..a1c0176 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -16,8 +16,6 @@ # limitations under the License. # Build a RHEL image, using the buildah tool -# Usage -# mq-buildah.sh ARCHIVE-NAME PACKAGES set -x set -e @@ -56,6 +54,8 @@ readonly packages=$2 readonly tag=$3 readonly version=$4 readonly mqdev=$5 +readonly mqm_uid=888 +readonly mqm_gid=888 ############################################################################### # Install MQ server @@ -93,10 +93,10 @@ buildah run ${ctr_mq} -- microdnf ${microdnf_opts} install \ buildah run ${ctr_mq} -- microdnf ${microdnf_opts} clean all rm -rf ${mnt_mq}/etc/yum.repos.d/* -groupadd --root ${mnt_mq} --system --gid 888 mqm -useradd --root ${mnt_mq} --system --uid 888 --gid mqm mqm -usermod --root ${mnt_mq} -aG root mqm -usermod --root ${mnt_mq} -aG mqm root +buildah run --user root $ctr_mq -- groupadd --system --gid ${mqm_gid} mqm +buildah run --user root $ctr_mq -- useradd --system --uid ${mqm_uid} --gid mqm mqm +buildah run --user root $ctr_mq -- usermod -aG root mqm +buildah run --user root $ctr_mq -- usermod -aG mqm root # Install MQ server packages into the MQ builder image ./mq-advanced-server-rhel/install-mq-rhel.sh ${ctr_mq} "${mnt_mq}" "${archive}" "${packages}" @@ -106,9 +106,9 @@ mkdir -p ${mnt_mq}/etc/mqm chown 888:888 ${mnt_mq}/etc/mqm # Install the Go binaries into the image -install --mode 0750 --owner 888 --group 888 ./build/runmqserver ${mnt_mq}/usr/local/bin/ -install --mode 6750 --owner 888 --group 888 ./build/chk* ${mnt_mq}/usr/local/bin/ -install --mode 0750 --owner 888 --group 888 ./NOTICES.txt ${mnt_mq}/opt/mqm/licenses/notices-container.txt +install --mode 0750 --owner ${mqm_uid} --group 0 ./build/runmqserver ${mnt_mq}/usr/local/bin/ +install --mode 6750 --owner ${mqm_uid} --group 0 ./build/chk* ${mnt_mq}/usr/local/bin/ +install --mode 0750 --owner ${mqm_uid} --group 0 ./NOTICES.txt ${mnt_mq}/opt/mqm/licenses/notices-container.txt ############################################################################### # Final Buildah commands diff --git a/mq-advanced-server-rhel/mqdev-buildah.sh b/mq-advanced-server-rhel/mqdev-buildah.sh index e32ea68..5c92847 100755 --- a/mq-advanced-server-rhel/mqdev-buildah.sh +++ b/mq-advanced-server-rhel/mqdev-buildah.sh @@ -15,9 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Build a RHEL image, using the buildah tool -# Usage -# mq-buildah.sh ARCHIVEFILE PACKAGES +# Build a RHEL image of MQ Advanced for Developers, using the buildah tool set -x set -e @@ -57,12 +55,11 @@ readonly tag=$2 readonly version=$3 -useradd --root $mnt_mq --gid mqm admin -groupadd --root $mnt_mq --system mqclient -useradd --root $mnt_mq --gid mqclient app - -buildah run $ctr_mq -- id admin -buildah run $ctr_mq -- sh -c "echo admin:passw0rd | chpasswd" +# Run these commands inside the container so that the SELinux context is handled correctly +buildah run --user root $ctr_mq -- useradd --gid mqm admin +buildah run --user root $ctr_mq -- groupadd --system mqclient +buildah run --user root $ctr_mq -- useradd --gid mqclient app +buildah run --user root $ctr_mq -- bash -c "echo admin:passw0rd | chpasswd" mkdir -p $mnt_mq/run/runmqdevserver chown 888:888 $mnt_mq/run/runmqdevserver From d9c70c48c53ee2dbb580e6e4ff10d58ffb24da6d Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 6 Feb 2019 14:23:25 +0000 Subject: [PATCH 28/60] Fix subscription checking logic --- mq-advanced-server-rhel/mq-buildah.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index a1c0176..8ca540e 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -63,11 +63,10 @@ readonly mqm_gid=888 microdnf_opts="--nodocs" # Check whether the host is registered with Red Hat -subscription-manager status -if [ "$?" -eq "0" ]; then +if subscription-manager status ; then # Host is subscribed, but the minimal image has no enabled repos - microdnf_opts="${microdnf_opts} --enablerepo=rhel-7-server-rpms" - # buildah run ${ctr_mq} -- microdnf microdnf --enablerepo=rhel-7-server-rpms --nodocs install ${prereq_packages} + # Note that the "bc" package is the only one in "extras" + microdnf_opts="${microdnf_opts} --enablerepo=rhel-7-server-rpms --enablerepo=rhel-7-server-extras-rpms" else # Use the Yum repositories configured on the host cp -R /etc/yum.repos.d/* ${mnt_mq}/etc/yum.repos.d/ From 77319629fef2a5cd8e51870e3ca233dfd38f2c0f Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 6 Feb 2019 16:11:52 +0000 Subject: [PATCH 29/60] Update copyright dates --- mq-advanced-server-rhel/install-mq-rhel.sh | 2 +- mq-advanced-server-rhel/mqdev-buildah.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mq-advanced-server-rhel/install-mq-rhel.sh b/mq-advanced-server-rhel/install-mq-rhel.sh index 9396bbe..a0e795b 100755 --- a/mq-advanced-server-rhel/install-mq-rhel.sh +++ b/mq-advanced-server-rhel/install-mq-rhel.sh @@ -1,6 +1,6 @@ #!/bin/bash # -*- mode: sh -*- -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/mq-advanced-server-rhel/mqdev-buildah.sh b/mq-advanced-server-rhel/mqdev-buildah.sh index 5c92847..8103098 100755 --- a/mq-advanced-server-rhel/mqdev-buildah.sh +++ b/mq-advanced-server-rhel/mqdev-buildah.sh @@ -1,6 +1,6 @@ #!/bin/bash # -*- mode: sh -*- -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # # Licensed under the Apache License, Version 2.0 (the "License"); From 33f82d76ff776a1cbfa8fb6ab46bcaebd7cf6334 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 6 Feb 2019 16:12:18 +0000 Subject: [PATCH 30/60] Use variable for uid and gid --- mq-advanced-server-rhel/mq-buildah.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 8ca540e..78b9d3b 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -102,7 +102,7 @@ buildah run --user root $ctr_mq -- usermod -aG mqm root # Create the directory for MQ configuration files mkdir -p ${mnt_mq}/etc/mqm -chown 888:888 ${mnt_mq}/etc/mqm +chown ${mqm_uid}:${mqm_gid} ${mnt_mq}/etc/mqm # Install the Go binaries into the image install --mode 0750 --owner ${mqm_uid} --group 0 ./build/runmqserver ${mnt_mq}/usr/local/bin/ From 694b31d6e8e90f090270f5a5febf090353ec1ce6 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Thu, 7 Feb 2019 10:09:53 +0000 Subject: [PATCH 31/60] Use documented way to install golint --- install-build-deps-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-build-deps-ubuntu.sh b/install-build-deps-ubuntu.sh index eab7577..acfc955 100755 --- a/install-build-deps-ubuntu.sh +++ b/install-build-deps-ubuntu.sh @@ -23,4 +23,4 @@ curl https://glide.sh/get | sh sudo curl -Lo /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64 sudo chmod +x /usr/local/bin/dep -go get golang.org/x/lint/golint +go get -u golang.org/x/lint/golint From a19c455ea4d92907fe65a01aef3c5c1481718bbc Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Thu, 7 Feb 2019 10:29:41 +0000 Subject: [PATCH 32/60] up the copyright date --- install-build-deps-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-build-deps-ubuntu.sh b/install-build-deps-ubuntu.sh index acfc955..546200a 100755 --- a/install-build-deps-ubuntu.sh +++ b/install-build-deps-ubuntu.sh @@ -1,6 +1,6 @@ #!/bin/bash # -*- mode: sh -*- -# © Copyright IBM Corporation 2015, 2018 +# © Copyright IBM Corporation 2015, 2019 # # # Licensed under the Apache License, Version 2.0 (the "License"); From c08ca2e79f0e2faf0e004b9eaf9c3c0905e6e13b Mon Sep 17 00:00:00 2001 From: Luke Powlett Date: Thu, 7 Feb 2019 16:05:01 +0000 Subject: [PATCH 33/60] update travis to use Ubuntu 16.04 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index b240462..ea5ff65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +dist: xenial + sudo: required language: go From d834ac7c9c5e85c330a3a5815dc5c582a86af67b Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 27 Feb 2019 15:06:09 +0000 Subject: [PATCH 34/60] Add in system to generate licenses on RHEL --- mq-advanced-server-rhel/mq-buildah.sh | 4 ++++ mq-advanced-server-rhel/writePackages.sh | 30 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100755 mq-advanced-server-rhel/writePackages.sh diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 78b9d3b..758dec9 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -109,6 +109,10 @@ install --mode 0750 --owner ${mqm_uid} --group 0 ./build/runmqserver ${mnt_mq}/u install --mode 6750 --owner ${mqm_uid} --group 0 ./build/chk* ${mnt_mq}/usr/local/bin/ install --mode 0750 --owner ${mqm_uid} --group 0 ./NOTICES.txt ${mnt_mq}/opt/mqm/licenses/notices-container.txt +# Copy in licenses from installed packages +install --mode 0550 --owner root --group root ./mq-advanced-server-rhel/writePackages.sh ${mnt_mq}/usr/local/bin/writePackages +buildah run --user root $ctr_mq -- /usr/local/bin/writePackages + ############################################################################### # Final Buildah commands ############################################################################### diff --git a/mq-advanced-server-rhel/writePackages.sh b/mq-advanced-server-rhel/writePackages.sh new file mode 100755 index 0000000..91ce8f7 --- /dev/null +++ b/mq-advanced-server-rhel/writePackages.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# -*- mode: sh -*- +# © Copyright IBM Corporation 2018, 2019 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copy in licenses from installed packages + +set -e + +rm -f /licenses/installed_package_notices + +for p in $(rpm -qa | sort) +do + rpm -qi $p >> /licenses/installed_package_notices + printf "\n" >> /licenses/installed_package_notices +done + +chmod 0444 /licenses/installed_package_notices From cc0f07290850ff8de287625bc6882d819f6ef4cf Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 25 Feb 2019 15:44:14 +0000 Subject: [PATCH 35/60] Change for running as a non-root user (#276) * Enable running container as mqm * Fix merge problem * Don't force root usage * RHEL image runs as mqm instead of root * Build on host with SELinux enabled * Enable building on node in an OpenShift cluster * Enable running container as mqm * Fix merge problem * Don't force root usage * Merge lastest changes from master * RHEL image runs as mqm instead of root * Fix merge issues * Test changes for non-root * Make timeout properly, and more non-root test fixes * Run tests with fewer/no capabilities * Correct usage docs for non-root * Add security docs * Add temporary debug output * Remove debug code * Fixes for termination-log * Allow init container to run as root * Fixes for CentOS build * Fixes for RHEL build * Logging improvements * Fix Dockerfile RHEL/CentOS build * Fix bash error * Make all builds specify UID * Use redist client for Go SDK * Inspect image before running tests * New test for init container * Log container runtime in runmqdevserver * Add extra capabilities if using a RHEL image --- .travis.yml | 2 +- CHANGELOG.md | 6 +- Dockerfile-server | 15 +- Makefile-UBUNTU | 19 +- cmd/runmqdevserver/logruntime.go | 63 +++++++ cmd/runmqdevserver/main.go | 22 ++- cmd/runmqdevserver/template.go | 4 +- cmd/runmqserver/logging.go | 10 +- cmd/runmqserver/logruntime.go | 79 +++++++++ cmd/runmqserver/main.go | 28 +-- cmd/runmqserver/mqconfig.go | 157 ----------------- cmd/runmqserver/qmgr.go | 2 +- cmd/runmqserver/user.go | 137 --------------- cmd/runmqserver/webserver.go | 19 +- docs/security.md | 39 ++++ docs/usage.md | 14 +- incubating/Dockerfile-sfbridge | 6 +- incubating/mq-explorer/Dockerfile | 6 +- incubating/mq-golang-sdk/Dockerfile | 26 +-- incubating/mq-sdk/Dockerfile | 6 +- incubating/mqadvanced-server-dev/Dockerfile | 27 ++- .../install-extra-packages.sh | 32 ++++ install-mq.sh | 20 ++- internal/containerruntime/runtime.go | 120 +++++++++++++ .../containerruntime/runtime_linux.go | 28 +-- .../containerruntime/runtime_other.go | 4 +- internal/logger/logger.go | 16 +- internal/user/user.go | 82 +++++++++ mq-advanced-server-rhel/go-buildah.sh | 1 + mq-advanced-server-rhel/install-mq-rhel.sh | 16 +- mq-advanced-server-rhel/mq-buildah.sh | 15 +- mq-advanced-server-rhel/mqdev-buildah.sh | 24 ++- test/docker/devconfig_test_util.go | 4 +- test/docker/docker_api_test.go | 160 +++++++++++++---- test/docker/docker_api_test_util.go | 166 ++++++++++++------ 35 files changed, 871 insertions(+), 504 deletions(-) create mode 100644 cmd/runmqdevserver/logruntime.go create mode 100644 cmd/runmqserver/logruntime.go delete mode 100644 cmd/runmqserver/mqconfig.go delete mode 100644 cmd/runmqserver/user.go create mode 100644 docs/security.md create mode 100644 incubating/mqadvanced-server-dev/install-extra-packages.sh create mode 100644 internal/containerruntime/runtime.go rename cmd/runmqserver/mqconfig_linux.go => internal/containerruntime/runtime_linux.go (84%) rename cmd/runmqserver/mqconfig_other.go => internal/containerruntime/runtime_other.go (92%) create mode 100644 internal/user/user.go diff --git a/.travis.yml b/.travis.yml index ea5ff65..5d41df9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ jobs: - BASE_IMAGE=ubuntu:16.04 - DOCKER_DOWNGRADE="echo nothing to be done" - env: - - BASE_IMAGE=centos:latest + - BASE_IMAGE=centos:7 - DOCKER_DOWNGRADE="echo nothing to be done" - if: type IN (pull_request) OR tag IS present env: diff --git a/CHANGELOG.md b/CHANGELOG.md index bcd410f..7a16087 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,12 @@ ## vNext +* Now runs using the "mqm" user instead of root. See new [security doc](https://github.com/ibm-messaging/mq-container/blob/master/docs/security.md) * [New IGNSTATE parameter for runmqsc START and STOP commands](https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_9.1.0/com.ibm.mq.pro.doc/q132310_.htm#q132310___ignstateparm) - From MQ version 9.1.1.0, any MQSC scripts included in the image should make use of the `IGNSTATE(YES)` parameter on any `START` and `STOP` commands. This allows for consistency when executing scripts multiple times (e.g. when a container is restarted) - +* Termination log moved from `/dev/termination-log` to `/run/termination-log`, to make permissions easier to handle +* Fixes for the following issues: + * Brackets no longer appear in termination log + * Test timeouts weren't being used correctly ## 9.1.1.0 (2018-11-30) diff --git a/Dockerfile-server b/Dockerfile-server index 97d70e9..5d96adf 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2015, 2018 +# © Copyright IBM Corporation 2015, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -48,12 +48,15 @@ ARG MQ_URL # The MQ packages to install - see install-mq.sh for default value ARG MQ_PACKAGES +# The UID to use for the "mqm" user +ARG MQM_UID=999 + COPY install-mq.sh /usr/local/bin/ # Install MQ. To avoid a "text file busy" error here, we sleep before installing. RUN chmod u+x /usr/local/bin/install-mq.sh \ && sleep 1 \ - && install-mq.sh + && install-mq.sh $MQM_UID # Create a directory for runtime data from runmqserver RUN mkdir -p /run/runmqserver \ @@ -65,11 +68,17 @@ COPY NOTICES.txt /opt/mqm/licenses/notices-container.txt RUN chmod ug+x /usr/local/bin/runmqserver \ && chown mqm:mqm /usr/local/bin/*mq* \ - && chmod ug+xs /usr/local/bin/chkmq* + && chmod ug+xs /usr/local/bin/chkmq* \ + && install --directory --mode 0775 --owner mqm --group root /run/runmqserver \ + && touch /run/termination-log \ + && chown mqm:root /run/termination-log \ + && chmod 0660 /run/termination-log # Always use port 1414 for MQ & 9157 for the metrics EXPOSE 1414 9157 ENV LANG=en_US.UTF-8 AMQ_DIAGNOSTIC_MSG_SEVERITY=1 AMQ_ADDITIONAL_JSON_LOG=1 LOG_FORMAT=basic +USER $MQM_UID + ENTRYPOINT ["runmqserver"] diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index 7fc2aa5..c4a8f10 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -77,9 +77,11 @@ endif ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu" MQ_ARCHIVE_TYPE=UBUNTU MQ_ARCHIVE_DEV_PLATFORM=ubuntu + MQM_UID=999 else MQ_ARCHIVE_TYPE=LINUX MQ_ARCHIVE_DEV_PLATFORM=linux + MQM_UID=888 endif # Try to figure out which archive to use from the architecture ifeq "$(ARCH)" "x86_64" @@ -149,6 +151,7 @@ test-unit: .PHONY: test-advancedserver test-advancedserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER) on $(shell docker --version)"$(END))) + docker inspect $(MQ_IMAGE_ADVANCEDSERVER) cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) .PHONY: build-devjmstest @@ -159,6 +162,7 @@ build-devjmstest: .PHONY: test-devserver test-devserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER) on $(shell docker --version)"$(END))) + docker inspect $(MQ_IMAGE_DEVSERVER) cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER) coverage: @@ -210,6 +214,7 @@ define docker-build-mq --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" \ --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" \ --build-arg IMAGE_TAG="$1" \ + --build-arg MQM_UID=$(MQM_UID) \ --label IBM_PRODUCT_ID=$4 \ --label IBM_PRODUCT_NAME=$5 \ --label IBM_PRODUCT_VERSION=$6 \ @@ -226,7 +231,7 @@ docker-version: .PHONY: build-advancedserver build-advancedserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE) -build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version build-golang-sdk-ex +build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version build-golang-sdk $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END))) $(call docker-build-mq,$(MQ_IMAGE_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) @@ -238,10 +243,10 @@ else build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm endif build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV) -build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk-ex +build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER_BASE)"$(END))) $(call docker-build-mq,$(MQ_IMAGE_DEVSERVER_BASE),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) - $(DOCKER) build --tag $(MQ_IMAGE_DEVSERVER) --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" --build-arg IMAGE_TAG="$(MQ_IMAGE_DEVSERVER)" --build-arg BASE_IMAGE=$(MQ_IMAGE_DEVSERVER_BASE) --build-arg BUILDER_IMAGE=$(MQ_IMAGE_GOLANG_SDK) --file incubating/mqadvanced-server-dev/Dockerfile . + $(DOCKER) build --tag $(MQ_IMAGE_DEVSERVER) --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" --build-arg IMAGE_TAG="$(MQ_IMAGE_DEVSERVER)" --build-arg BASE_IMAGE=$(MQ_IMAGE_DEVSERVER_BASE) --build-arg BUILDER_IMAGE=$(MQ_IMAGE_GOLANG_SDK) --build-arg MQM_UID=$(MQM_UID) --file incubating/mqadvanced-server-dev/Dockerfile . .PHONY: build-advancedserver-cover build-advancedserver-cover: docker-version @@ -264,12 +269,8 @@ build-sdk-ex: docker-version docker-pull $(call docker-build-mq,$(MQ_IMAGE_SDK),incubating/mq-sdk/Dockerfile,$(MQ_SDK_ARCHIVE),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers SDK (Non-Warranted)",$(MQ_VERSION)) .PHONY: build-golang-sdk -build-golang-sdk: downloads/$(MQ_SDK_ARCHIVE) build-golang-sdk-ex - -.PHONY: build-golang-sdk-ex -build-golang-sdk-ex: docker-version build-sdk-ex - $(DOCKER) build --build-arg BASE_IMAGE=$(MQ_IMAGE_SDK) -t $(MQ_IMAGE_GOLANG_SDK) -f incubating/mq-golang-sdk/Dockerfile . -# $(call docker-build-mq,$(MQ_IMAGE_GOLANG_SDK),incubating/mq-golang-sdk/Dockerfile,$(MQ_IMAGE_SDK),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers SDK (Non-Warranted)",$(MQ_VERSION)) +build-golang-sdk: + $(DOCKER) build -t $(MQ_IMAGE_GOLANG_SDK) -f incubating/mq-golang-sdk/Dockerfile . .PHONY: docker-pull docker-pull: diff --git a/cmd/runmqdevserver/logruntime.go b/cmd/runmqdevserver/logruntime.go new file mode 100644 index 0000000..21aae84 --- /dev/null +++ b/cmd/runmqdevserver/logruntime.go @@ -0,0 +1,63 @@ +/* +© Copyright IBM Corporation 2017, 2019 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "runtime" + "strings" + + containerruntime "github.com/ibm-messaging/mq-container/internal/containerruntime" + "github.com/ibm-messaging/mq-container/internal/user" +) + +func logContainerDetails() { + log.Printf("CPU architecture: %v", runtime.GOARCH) + kv, err := containerruntime.GetKernelVersion() + if err == nil { + log.Printf("Linux kernel version: %v", kv) + } + cr, err := containerruntime.GetContainerRuntime() + if err == nil { + log.Printf("Container runtime: %v", cr) + } + bi, err := containerruntime.GetBaseImage() + if err == nil { + log.Printf("Base image: %v", bi) + } + u, err := user.GetUser() + if err == nil { + if len(u.SupplementalGID) == 0 { + log.Printf("Running as user ID %v (%v) with primary group %v", u.UID, u.Name, u.PrimaryGID) + } else { + log.Printf("Running as user ID %v (%v) with primary group %v, and supplementary groups %v", u.UID, u.Name, u.PrimaryGID, strings.Join(u.SupplementalGID, ",")) + } + } + caps, err := containerruntime.GetCapabilities() + if err == nil { + for k, v := range caps { + if len(v) > 0 { + log.Printf("Capabilities (%s set): %v", strings.ToLower(k), strings.Join(v, ",")) + } + } + } else { + log.Errorf("Error getting capabilities: %v", err) + } + sc, err := containerruntime.GetSeccomp() + if err == nil { + log.Printf("seccomp enforcing mode: %v", sc) + } + log.Printf("Process security attributes: %v", containerruntime.GetSecurityAttributes()) +} diff --git a/cmd/runmqdevserver/main.go b/cmd/runmqdevserver/main.go index c6e9112..0cfc366 100644 --- a/cmd/runmqdevserver/main.go +++ b/cmd/runmqdevserver/main.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ var log *logger.Logger func setPassword(user string, password string) error { // #nosec G204 - cmd := exec.Command("chpasswd") + cmd := exec.Command("sudo", "chpasswd") stdin, err := cmd.StdinPipe() if err != nil { return err @@ -41,9 +41,10 @@ func setPassword(user string, password string) error { if err != nil { log.Errorf("Error closing password stdin: %v", err) } - _, _, err = command.RunCmd(cmd) + out, _, err := command.RunCmd(cmd) if err != nil { - return err + // Include the command output in the error + return fmt.Errorf("%v: %v", err.Error(), out) } log.Printf("Set password for \"%v\" user", user) return nil @@ -93,16 +94,16 @@ func configureWeb(qmName string) error { } func logTerminationf(format string, args ...interface{}) { - logTermination(fmt.Sprintf(format, args)) + logTermination(fmt.Sprintf(format, args...)) } // TODO: Duplicated code func logTermination(args ...interface{}) { - msg := fmt.Sprint(args) - // Write the message to the termination log. This is the default place + msg := fmt.Sprint(args...) + // Write the message to the termination log. This is not the default place // that Kubernetes will look for termination information. log.Debugf("Writing termination message: %v", msg) - err := ioutil.WriteFile("/dev/termination-log", []byte(msg), 0660) + err := ioutil.WriteFile("/run/termination-log", []byte(msg), 0660) if err != nil { log.Debug(err) } @@ -115,6 +116,9 @@ func doMain() error { logTermination(err) return err } + + logContainerDetails() + adminPassword, set := os.LookupEnv("MQ_ADMIN_PASSWORD") if set { err = setPassword("admin", adminPassword) @@ -170,7 +174,7 @@ func main() { } else { // Replace this process with runmqserver // #nosec G204 - err = syscall.Exec("/usr/local/bin/runmqserver", []string{"runmqserver"}, os.Environ()) + err = syscall.Exec("/usr/local/bin/runmqserver", []string{"runmqserver", "-dev"}, os.Environ()) if err != nil { log.Errorf("Error replacing this process with runmqserver: %v", err) } diff --git a/cmd/runmqdevserver/template.go b/cmd/runmqdevserver/template.go index 28f8aa8..2600c51 100644 --- a/cmd/runmqdevserver/template.go +++ b/cmd/runmqdevserver/template.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ func processTemplateFile(templateFile, destFile string, data interface{}) error _, err = os.Stat(dir) if err != nil { if os.IsNotExist(err) { - err = os.MkdirAll(dir, 0660) + err = os.MkdirAll(dir, 0770) if err != nil { log.Error(err) return err diff --git a/cmd/runmqserver/logging.go b/cmd/runmqserver/logging.go index 70bee1d..f2f30a8 100644 --- a/cmd/runmqserver/logging.go +++ b/cmd/runmqserver/logging.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -36,15 +36,15 @@ var log *logger.Logger var collectDiagOnFail = false func logTerminationf(format string, args ...interface{}) { - logTermination(fmt.Sprintf(format, args)) + logTermination(fmt.Sprintf(format, args...)) } func logTermination(args ...interface{}) { - msg := fmt.Sprint(args) - // Write the message to the termination log. This is the default place + msg := fmt.Sprint(args...) + // Write the message to the termination log. This is not the default place // that Kubernetes will look for termination information. log.Debugf("Writing termination message: %v", msg) - err := ioutil.WriteFile("/dev/termination-log", []byte(msg), 0660) + err := ioutil.WriteFile("/run/termination-log", []byte(msg), 0660) if err != nil { log.Debug(err) } diff --git a/cmd/runmqserver/logruntime.go b/cmd/runmqserver/logruntime.go new file mode 100644 index 0000000..10f1539 --- /dev/null +++ b/cmd/runmqserver/logruntime.go @@ -0,0 +1,79 @@ +/* +© Copyright IBM Corporation 2017, 2019 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "fmt" + "runtime" + "strings" + + containerruntime "github.com/ibm-messaging/mq-container/internal/containerruntime" + "github.com/ibm-messaging/mq-container/internal/user" +) + +func logContainerDetails() error { + if runtime.GOOS != "linux" { + return fmt.Errorf("Unsupported platform: %v", runtime.GOOS) + } + log.Printf("CPU architecture: %v", runtime.GOARCH) + kv, err := containerruntime.GetKernelVersion() + if err == nil { + log.Printf("Linux kernel version: %v", kv) + } + cr, err := containerruntime.GetContainerRuntime() + if err == nil { + log.Printf("Container runtime: %v", cr) + } + bi, err := containerruntime.GetBaseImage() + if err == nil { + log.Printf("Base image: %v", bi) + } + u, err := user.GetUser() + if err == nil { + if len(u.SupplementalGID) == 0 { + log.Printf("Running as user ID %v (%v) with primary group %v", u.UID, u.Name, u.PrimaryGID) + } else { + log.Printf("Running as user ID %v (%v) with primary group %v, and supplementary groups %v", u.UID, u.Name, u.PrimaryGID, strings.Join(u.SupplementalGID, ",")) + } + } + caps, err := containerruntime.GetCapabilities() + if err == nil { + for k, v := range caps { + if len(v) > 0 { + log.Printf("Capabilities (%s set): %v", strings.ToLower(k), strings.Join(v, ",")) + } + } + } + sc, err := containerruntime.GetSeccomp() + if err == nil { + log.Printf("seccomp enforcing mode: %v", sc) + } + log.Printf("Process security attributes: %v", containerruntime.GetSecurityAttributes()) + m, err := containerruntime.GetMounts() + if err == nil { + if len(m) == 0 { + log.Print("No volume detected. Persistent messages may be lost") + } else { + for mountPoint, fsType := range m { + log.Printf("Detected '%v' volume mounted to %v", fsType, mountPoint) + if !containerruntime.SupportedFilesystem(fsType) { + return fmt.Errorf("%v uses unsupported filesystem type: %v", mountPoint, fsType) + } + } + } + } + return nil +} diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 01f00d5..2b46a9e 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -30,10 +30,11 @@ import ( ) func doMain() error { + var initFlag = flag.Bool("i", false, "initialize volume only, then exit") var infoFlag = flag.Bool("info", false, "Display debug info, then exit") + var devFlag = flag.Bool("dev", false, "used when running this program from runmqdevserver to control log output") flag.Parse() - // Configure the logger so we can output messages name, nameErr := name.GetQueueManagerName() mf, err := configureLogger(name) if err != nil { @@ -44,7 +45,7 @@ func doMain() error { // Check whether they only want debug info if *infoFlag { logVersionInfo() - logConfig() + logContainerDetails() return nil } @@ -81,16 +82,12 @@ func doMain() error { // Enable diagnostic collecting on failure collectDiagOnFail = true - err = verifyCurrentUser() - if err != nil { - logTermination(err) - return err - } - - err = logConfig() - if err != nil { - logTermination(err) - return err + if *devFlag == false { + err = logContainerDetails() + if err != nil { + logTermination(err) + return err + } } err = createVolume("/mnt/mqm") @@ -104,6 +101,11 @@ func doMain() error { return err } + // If init flag is set, exit now + if *initFlag { + return nil + } + // Print out versioning information logVersionInfo() diff --git a/cmd/runmqserver/mqconfig.go b/cmd/runmqserver/mqconfig.go deleted file mode 100644 index 143415f..0000000 --- a/cmd/runmqserver/mqconfig.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -© Copyright IBM Corporation 2017, 2018 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package main - -import ( - "fmt" - "io/ioutil" - "runtime" - "strings" - - "github.com/genuinetools/amicontained/container" -) - -func logContainerRuntime() { - r, err := container.DetectRuntime() - if err != nil { - log.Printf("Failed to get container runtime: %v", err) - return - } - log.Printf("Container runtime: %v", r) -} - -func logBaseImage() { - buf, err := ioutil.ReadFile("/etc/os-release") - if err != nil { - log.Printf("Failed to read /etc/os-release: %v", err) - return - } - lines := strings.Split(string(buf), "\n") - for _, l := range lines { - if strings.HasPrefix(l, "PRETTY_NAME=") { - words := strings.Split(l, "\"") - if len(words) >= 2 { - log.Printf("Base image: %v", words[1]) - return - } - } - } -} - -// logCapabilities logs the Linux capabilities (e.g. setuid, setgid). See https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities -func logCapabilities() { - caps, err := container.Capabilities() - if err != nil { - log.Printf("Failed to get container capabilities: %v", err) - return - } - for k, v := range caps { - if len(v) > 0 { - log.Printf("Capabilities (%s set): %v", strings.ToLower(k), strings.Join(v, ",")) - } - } -} - -// logSeccomp logs the seccomp enforcing mode, which affects which kernel calls can be made -func logSeccomp() { - s, err := container.SeccompEnforcingMode() - if err != nil { - log.Printf("Failed to get container SeccompEnforcingMode: %v", err) - return - } - log.Printf("seccomp enforcing mode: %v", s) -} - -// logSecurityAttributes logs the security attributes of the current process. -// The security attributes indicate whether AppArmor or SELinux are being used, -// and what the level of confinement is. -func logSecurityAttributes() { - a, err := readProc("/proc/self/attr/current") - // On some systems, if AppArmor or SELinux are not installed, you get an - // error when you try and read `/proc/self/attr/current`, even though the - // file exists. - if err != nil || a == "" { - a = "none" - } - log.Printf("Process security attributes: %v", a) -} - -func readProc(filename string) (value string, err error) { - // #nosec G304 - buf, err := ioutil.ReadFile(filename) - if err != nil { - return "", err - } - return strings.TrimSpace(string(buf)), nil -} - -func readMounts() error { - all, err := readProc("/proc/mounts") - if err != nil { - log.Print("Error: Couldn't read /proc/mounts") - return err - } - lines := strings.Split(all, "\n") - detected := false - for i := range lines { - parts := strings.Split(lines[i], " ") - //dev := parts[0] - mountPoint := parts[1] - fsType := parts[2] - if strings.Contains(mountPoint, "/mnt/mqm") { - log.Printf("Detected '%v' volume mounted to %v", fsType, mountPoint) - detected = true - } - } - if !detected { - log.Print("No volume detected. Persistent messages may be lost") - } else { - return checkFS("/mnt/mqm") - } - return nil -} - -func logConfig() error { - log.Printf("CPU architecture: %v", runtime.GOARCH) - if runtime.GOOS == "linux" { - var err error - osr, err := readProc("/proc/sys/kernel/osrelease") - if err != nil { - log.Print(err) - } else { - log.Printf("Linux kernel version: %v", osr) - } - logContainerRuntime() - logBaseImage() - fileMax, err := readProc("/proc/sys/fs/file-max") - if err != nil { - log.Print(err) - } else { - log.Printf("Maximum file handles: %v", fileMax) - } - logUser() - logCapabilities() - logSeccomp() - logSecurityAttributes() - err = readMounts() - if err != nil { - return err - } - } else { - return fmt.Errorf("Unsupported platform: %v", runtime.GOOS) - } - return nil -} diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index 0082e14..5705f45 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -119,7 +119,7 @@ func configureQueueManager() error { // Run the command and wait for completion out, err := cmd.CombinedOutput() if err != nil { - log.Errorf("Error running MQSC file %v: %v", file.Name(), err) + log.Errorf("Error running MQSC file %v (%v):\n\t%v", file.Name(), err, strings.Replace(string(out), "\n", "\n\t", -1)) return err } // Print the runmqsc output, adding tab characters to make it more readable as part of the log diff --git a/cmd/runmqserver/user.go b/cmd/runmqserver/user.go deleted file mode 100644 index c439b2d..0000000 --- a/cmd/runmqserver/user.go +++ /dev/null @@ -1,137 +0,0 @@ -/* -© Copyright IBM Corporation 2018 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package main - -import ( - "fmt" - "os/user" - "strings" - - "github.com/ibm-messaging/mq-container/internal/command" -) - -const groupName string = "supplgrp" - -func verifyCurrentUser() error { - log.Debug("Verifying current user information") - curUser, err := user.Current() - if err != nil { - return err - } - log.Debugf("Detected current user as: %v+", curUser) - if curUser.Username == "mqm" { - // Not supported yet - return fmt.Errorf("Container is running as mqm user which is not supported. Please run this container as root") - } else if curUser.Username == "root" { - // We're running as root so need to check for supplementary groups. - // We can't use the golang User.GroupIDs as it doesn't seem to detect container supplementary groups.. - groups, err := getCurrentUserGroups() - for _, e := range groups { - _, _, testGroup := command.Run("getent", "group", e) - if testGroup != nil { - log.Printf("Group %s does not exist on the system... Adding to system and MQM user", e) - _, _, err = command.Run("groupadd", "-g", e, groupName) - if err != nil { - log.Errorf("Failed to create group %s as %s", e, groupName) - return err - } - _, _, err = command.Run("usermod", "-aG", groupName, "mqm") - if err != nil { - log.Errorf("Failed to add group %s(%s) to the mqm user.", groupName, e) - return err - } - } - } - } else { - // We're running as an unknown user... - return fmt.Errorf("Container is running as %s user which is not supported. Please run this container as root", curUser.Username) - } - - return nil -} - -func logUser() { - u, usererr := user.Current() - if usererr == nil { - g, err := getCurrentUserGroups() - if err != nil && len(g) == 0 { - log.Printf("Running as user ID %v (%v) with primary group %v", u.Uid, u.Name, u.Gid) - } else { - // Look for the primary group in the list of group IDs - for i, v := range g { - if v == u.Gid { - // Remove the element from the slice - g = append(g[:i], g[i+1:]...) - } - } - log.Printf("Running as user ID %v (%v) with primary group %v, and supplementary groups %v", u.Uid, u.Name, u.Gid, strings.Join(g, ",")) - } - } - - if usererr == nil && u.Username != "mqm" { - mqm, err := user.Lookup("mqm") - // Need to print out mqm user details as well. - g, err := getUserGroups(mqm) - if err != nil && len(g) == 0 { - log.Printf("MQM user ID %v (%v) has primary group %v", mqm.Uid, "mqm", mqm.Gid) - } else { - // Look for the primary group in the list of group IDs - for i, v := range g { - if v == mqm.Gid { - // Remove the element from the slice - g = append(g[:i], g[i+1:]...) - } - } - log.Printf("MQM user ID %v (%v) has primary group %v, and supplementary groups %v", mqm.Uid, "mqm", mqm.Gid, strings.Join(g, ",")) - } - } -} - -func getCurrentUserGroups() ([]string, error) { - var nilArray []string - out, _, err := command.Run("id", "--groups") - if err != nil { - log.Debug("Unable to get current user groups") - return nilArray, err - } - - out = strings.TrimSpace(out) - if out == "" { - // we don't have any groups? - return nilArray, fmt.Errorf("Unable to determine groups for current user") - } - - groups := strings.Split(out, " ") - return groups, nil -} - -func getUserGroups(usr *user.User) ([]string, error) { - var nilArray []string - out, _, err := command.Run("id", "--groups", usr.Uid) - if err != nil { - log.Debugf("Unable to get user %s groups", usr.Uid) - return nilArray, err - } - - out = strings.TrimSpace(out) - if out == "" { - // we don't have any groups? - return nilArray, fmt.Errorf("Unable to determine groups for user %s", usr.Uid) - } - - groups := strings.Split(out, " ") - return groups, nil -} diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index a88786a..dae2bd6 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -1,7 +1,7 @@ // +build mqdev /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,7 +22,9 @@ import ( "io" "os" "os/exec" + "os/user" "path/filepath" + "strconv" "syscall" "github.com/ibm-messaging/mq-container/internal/command" @@ -42,12 +44,23 @@ func startWebServer() error { // Take all current environment variables, and add the app password cmd.Env = append(os.Environ(), "MQ_APP_PASSWORD=passw0rd") } - cmd.SysProcAttr = &syscall.SysProcAttr{} uid, gid, err := command.LookupMQM() if err != nil { return err } - cmd.SysProcAttr.Credential = &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)} + u, err := user.Current() + if err != nil { + return err + } + currentUID, err := strconv.Atoi(u.Uid) + if err != nil { + return fmt.Errorf("Error converting UID to string: %v", err) + } + // Add credentials to run as 'mqm', only if we aren't already 'mqm' + if currentUID != uid { + cmd.SysProcAttr = &syscall.SysProcAttr{} + cmd.SysProcAttr.Credential = &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)} + } out, rc, err := command.RunCmd(cmd) if err != nil { log.Printf("Error %v starting web server: %v", rc, string(out)) diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..d6d6077 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,39 @@ +# Security + +## Container runtime + +### User + +The MQ server image is run using the "mqm" user. On the Ubuntu-based image, this uses the UID and GID of 999. On the Red Hat Enterprise Linux image, it uses the UID and GID of 888. + +### Capabilities + +The MQ Advanced image requires no Linux capabilities, so you can drop any capabilities which are added by default. For example, in Docker you could do the following: + +```sh +docker run \ + --cap-drop=ALL \ + --env LICENSE=accept \ + --env MQ_QMGR_NAME=QM1 \ + --detach \ + mqadvanced-server:9.1.1.0-x86_64-ubuntu-16.04 +``` + +The MQ Advanced for Developers image does requires the "chown", "setuid", "setgid" and "audit_write" capabilities (plus "dac_override" if you're using an image based on Red Hat Enterprise Linux). This is because it uses the "sudo" command to change passwords inside the container. For example, in Docker, you could do the following: + +```sh +docker run \ + --cap-drop=ALL \ + --cap-add=CHOWN \ + --cap-add=SETUID \ + --cap-add=SETGID \ + --cap-add=AUDIT_WRITE \ + --env LICENSE=accept \ + --env MQ_QMGR_NAME=QM1 \ + --detach \ + mqadvanced-server-dev:9.1.1.0-x86_64-ubuntu-16.04 +``` + +### SELinux + +The SELinux label "spc_t" (super-privileged container) is needed to run the MQ container on a host with SELinux enabled. This is due to a current limitation in how MQ data is stored on volumes, which violates the usual policy applied when using the standard "container_t" label. \ No newline at end of file diff --git a/docs/usage.md b/docs/usage.md index cc1ee90..a9311fe 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -66,20 +66,20 @@ The following is an *example* `Dockerfile` for creating your own pre-configured ```dockerfile FROM ibmcom/mq +USER root RUN useradd alice -G mqm && \ echo alice:passw0rd | chpasswd +USER mqm COPY 20-config.mqsc /etc/mqm/ ``` -Here is an example corresponding `20-config.mqsc` script from the [mqdev blog](https://developer.ibm.com/messaging/2018/10/01/archives-getting-going-without-turning-off-ibm-mq-security/), which allows users with passwords to connect on the `PASSWORD.SVRCONN` channel: +The `USER` instructions are necessary to ensure that the `useradd` and `chpasswd` commands are run as the root user. + +Here is an example corresponding `20-config.mqsc` script, which creates two local queues: ```mqsc -DEFINE CHANNEL(PASSWORD.SVRCONN) CHLTYPE(SVRCONN) REPLACE -SET CHLAUTH(PASSWORD.SVRCONN) TYPE(BLOCKUSER) USERLIST('nobody') DESCR('Allow privileged users on this channel') -SET CHLAUTH('*') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(NOACCESS) DESCR('BackStop rule') -SET CHLAUTH(PASSWORD.SVRCONN) TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(CHANNEL) CHCKCLNT(REQUIRED) -ALTER AUTHINFO(SYSTEM.DEFAULT.AUTHINFO.IDPWOS) AUTHTYPE(IDPWOS) ADOPTCTX(YES) -REFRESH SECURITY TYPE(CONNAUTH) +DEFINE QLOCAL(MY.QUEUE.1) REPLACE +DEFINE QLOCAL(MY.QUEUE.2) REPLACE ``` The file `20-config.mqsc` should be saved into the same directory as the `Dockerfile`. diff --git a/incubating/Dockerfile-sfbridge b/incubating/Dockerfile-sfbridge index 0815f43..7a6e9a6 100644 --- a/incubating/Dockerfile-sfbridge +++ b/incubating/Dockerfile-sfbridge @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2015, 2017 +# © Copyright IBM Corporation 2015, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,9 +20,11 @@ ARG MQ_URL=https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messag # The MQ packages to install ARG MQ_PACKAGES="ibmmq-sfbridge" +ARG MQM_UID=999 + ADD install-mq.sh /usr/local/bin/ RUN chmod u+x /usr/local/bin/install-mq.sh \ - && install-mq.sh + && install-mq.sh $MQM_UID ENV LANG=en_US.UTF-8 diff --git a/incubating/mq-explorer/Dockerfile b/incubating/mq-explorer/Dockerfile index 9692099..f4b7fe2 100644 --- a/incubating/mq-explorer/Dockerfile +++ b/incubating/mq-explorer/Dockerfile @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2015, 2017 +# © Copyright IBM Corporation 2015, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ ARG MQ_URL=https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messag # The MQ packages to install ARG MQ_PACKAGES="ibmmq-explorer" +ARG MQM_UID=999 + RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get update \ && apt-get install -y \ @@ -27,7 +29,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \ libxtst6 ADD install-mq.sh /usr/local/bin/ -RUN chmod u+x /usr/local/bin/install-mq.sh \ +RUN chmod u+x /usr/local/bin/install-mq.sh $MQM_UID \ && install-mq.sh ENV LANG=en_US.UTF-8 diff --git a/incubating/mq-golang-sdk/Dockerfile b/incubating/mq-golang-sdk/Dockerfile index 589aad0..b66368f 100644 --- a/incubating/mq-golang-sdk/Dockerfile +++ b/incubating/mq-golang-sdk/Dockerfile @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,22 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG BASE_IMAGE=mq-sdk:9.1.1.0-x86_64-ubuntu-16.04 +FROM golang:1.10 -FROM $BASE_IMAGE - -COPY incubating/mq-golang-sdk/install-golang.sh /usr/local/bin - -ENV GO_VERSION=1.10 - -ENV PATH="${PATH}:/usr/lib/go-${GO_VERSION}/bin:/go/bin:/usr/local/go/bin" \ - CGO_CFLAGS="-I/opt/mqm/inc/" \ - CGO_LDFLAGS_ALLOW="-Wl,-rpath.*" \ - GOPATH="/go" - -# Install the Go compiler and Git -RUN chmod +x /usr/local/bin/install-golang.sh \ - && sleep 1 \ - && install-golang.sh - -WORKDIR $GOPATH +# Install the MQ redistributable client (including header files) into the Go builder image +RUN mkdir -p /opt/mqm \ + && curl -L https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist/9.1.1.0-IBM-MQC-Redist-LinuxX64.tar.gz | tar -xz -C /opt/mqm +ENV CGO_CFLAGS="-I/opt/mqm/inc/" \ + CGO_LDFLAGS_ALLOW="-Wl,-rpath.*" \ No newline at end of file diff --git a/incubating/mq-sdk/Dockerfile b/incubating/mq-sdk/Dockerfile index 7fb304e..a7a8833 100644 --- a/incubating/mq-sdk/Dockerfile +++ b/incubating/mq-sdk/Dockerfile @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,8 @@ ARG MQ_URL # The packages to install in install-mq.sh ARG MQ_PACKAGES +ARG MQM_UID=999 + COPY install-mq.sh /usr/local/bin/ # Install MQ. To avoid a "text file busy" error here, we sleep before installing. @@ -30,6 +32,6 @@ COPY install-mq.sh /usr/local/bin/ # errors with some commands (e.g. `dspmqver`) RUN chmod u+x /usr/local/bin/install-mq.sh \ && sleep 1 \ - && install-mq.sh \ + && install-mq.sh $MQM_UID \ && rm -rf /var/mqm \ && /opt/mqm/bin/crtmqdir -f -s diff --git a/incubating/mqadvanced-server-dev/Dockerfile b/incubating/mqadvanced-server-dev/Dockerfile index 2c996d8..3d80875 100644 --- a/incubating/mqadvanced-server-dev/Dockerfile +++ b/incubating/mqadvanced-server-dev/Dockerfile @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2015, 2018 +# © Copyright IBM Corporation 2015, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -43,6 +43,20 @@ ENV MQ_DEV=true # Default administrator password ENV MQ_ADMIN_PASSWORD=passw0rd +ARG MQM_UID=999 + +USER root + +COPY incubating/mqadvanced-server-dev/install-extra-packages.sh /usr/local/bin/ + +RUN chmod u+x /usr/local/bin/install-extra-packages.sh \ + && sleep 1 \ + && install-extra-packages.sh + +# WARNING: This is what allows the mqm user to change the password of any other user +# It's used by runmqdevserver to change the admin/app passwords. +RUN echo "mqm ALL = NOPASSWD: /usr/sbin/chpasswd" > /etc/sudoers.d/mq-dev-config + ## Add admin and app users, and set a default password for admin RUN useradd admin -G mqm \ && groupadd mqclient \ @@ -55,12 +69,17 @@ RUN mkdir -p /run/runmqdevserver \ COPY --from=builder /go/src/github.com/ibm-messaging/mq-container/runmqserver /usr/local/bin/ COPY --from=builder /go/src/github.com/ibm-messaging/mq-container/runmqdevserver /usr/local/bin/ + # Copy template files -COPY incubating/mqadvanced-server-dev/*.tpl /etc/mqm/ +COPY --chown=mqm:mqm incubating/mqadvanced-server-dev/*.tpl /etc/mqm/ # Copy web XML files for default developer configuration -COPY incubating/mqadvanced-server-dev/web /etc/mqm/web -RUN chmod +x /usr/local/bin/runmq* +COPY --chown=mqm:mqm incubating/mqadvanced-server-dev/web /etc/mqm/web + +RUN chmod +x /usr/local/bin/runmq* \ + && install --directory --mode 0775 --owner mqm --group root /run/runmqdevserver EXPOSE 9443 +USER $MQM_UID + ENTRYPOINT ["runmqdevserver"] diff --git a/incubating/mqadvanced-server-dev/install-extra-packages.sh b/incubating/mqadvanced-server-dev/install-extra-packages.sh new file mode 100644 index 0000000..fd3fa8a --- /dev/null +++ b/incubating/mqadvanced-server-dev/install-extra-packages.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# -*- mode: sh -*- +# © Copyright IBM Corporation 2019 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +test -f /usr/bin/yum && RHEL=true || RHEL=false +test -f /usr/bin/apt-get && UBUNTU=true || UBUNTU=false + +if ($UBUNTU); then + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y --no-install-recommends sudo + rm -rf /var/lib/apt/lists/* +fi + +if ($RHEL); then + yum -y install sudo + yum -y clean all + rm -rf /var/cache/yum/* +fi diff --git a/install-mq.sh b/install-mq.sh index 2acde1b..c61d4e4 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -1,6 +1,6 @@ #!/bin/bash # -*- mode: sh -*- -# © Copyright IBM Corporation 2015, 2018 +# © Copyright IBM Corporation 2015, 2019 # # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,8 @@ # Fail on any non-zero return code set -ex +mqm_uid=${1:-999} + test -f /usr/bin/yum && RHEL=true || RHEL=false test -f /usr/bin/apt-get && UBUNTU=true || UBUNTU=false @@ -102,10 +104,8 @@ $UBUNTU && apt-get purge -y \ $UBUNTU && apt-get autoremove -y # Recommended: Create the mqm user ID with a fixed UID and group, so that the file permissions work between different images -$UBUNTU && groupadd --system --gid 999 mqm -$UBUNTU && useradd --system --uid 999 --gid mqm mqm -$RHEL && groupadd --system --gid 888 mqm -$RHEL && useradd --system --uid 888 --gid mqm mqm +groupadd --system --gid ${mqm_uid} mqm +useradd --system --uid ${mqm_uid} --gid mqm --groups 0 mqm usermod -aG mqm root # Find directory containing .deb files @@ -153,16 +153,18 @@ $UBUNTU && echo "mq:$(dspmqver -b -f 2)" > /etc/debian_chroot # Remove the directory structure under /var/mqm which was created by the installer rm -rf /var/mqm -# Create the mount point for volumes -mkdir -p /mnt/mqm +# Create the mount point for volumes, ensuring MQ has permissions to all directories +install --directory --mode 0775 --owner mqm --group root /mnt +install --directory --mode 0775 --owner mqm --group root /mnt/mqm +install --directory --mode 0775 --owner mqm --group root /mnt/mqm/data # Create the directory for MQ configuration files -mkdir -p /etc/mqm +install --directory --mode 0775 --owner mqm --group root /etc/mqm # Create a symlink for /var/mqm -> /mnt/mqm/data ln -s /mnt/mqm/data /var/mqm -# Optional: Set these values for the Bluemix Vulnerability Report +# Optional: Ensure any passwords expire in a timely manner sed -i 's/PASS_MAX_DAYS\t99999/PASS_MAX_DAYS\t90/' /etc/login.defs sed -i 's/PASS_MIN_DAYS\t0/PASS_MIN_DAYS\t1/' /etc/login.defs diff --git a/internal/containerruntime/runtime.go b/internal/containerruntime/runtime.go new file mode 100644 index 0000000..3b42b09 --- /dev/null +++ b/internal/containerruntime/runtime.go @@ -0,0 +1,120 @@ +/* +© Copyright IBM Corporation 2019 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package runtime + +import ( + "fmt" + "io/ioutil" + "strings" + + "github.com/genuinetools/amicontained/container" +) + +func GetContainerRuntime() (string, error) { + return container.DetectRuntime() +} + +func GetBaseImage() (string, error) { + buf, err := ioutil.ReadFile("/etc/os-release") + if err != nil { + return "", fmt.Errorf("Failed to read /etc/os-release: %v", err) + } + lines := strings.Split(string(buf), "\n") + for _, l := range lines { + if strings.HasPrefix(l, "PRETTY_NAME=") { + words := strings.Split(l, "\"") + if len(words) >= 2 { + return words[1], nil + } + } + } + return "unknown", nil +} + +// GetCapabilities gets the Linux capabilities (e.g. setuid, setgid). See https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities +func GetCapabilities() (map[string][]string, error) { + return container.Capabilities() +} + +// GetSeccomp gets the seccomp enforcing mode, which affects which kernel calls can be made +func GetSeccomp() (string, error) { + s, err := container.SeccompEnforcingMode() + if err != nil { + return "", fmt.Errorf("Failed to get container SeccompEnforcingMode: %v", err) + } + return s, nil +} + +// GetSecurityAttributes gets the security attributes of the current process. +// The security attributes indicate whether AppArmor or SELinux are being used, +// and what the level of confinement is. +func GetSecurityAttributes() string { + a, err := readProc("/proc/self/attr/current") + // On some systems, if AppArmor or SELinux are not installed, you get an + // error when you try and read `/proc/self/attr/current`, even though the + // file exists. + if err != nil || a == "" { + a = "none" + } + return a +} + +// func logUser() { +// u, err := user.GetUser() +// if err == nil { +// if len(u.SupplementalGID) == 0 { +// log.Printf("Running as user ID %v (%v) with primary group %v", u.UID, u.Name, u.PrimaryGID) +// } else { +// log.Printf("Running as user ID %v (%v) with primary group %v, and supplementary groups %v", u.UID, u.Name, u.PrimaryGID, strings.Join(u.SupplementalGID, ",")) +// } +// } +// } + +func readProc(filename string) (value string, err error) { + // #nosec G304 + buf, err := ioutil.ReadFile(filename) + if err != nil { + return "", err + } + return strings.TrimSpace(string(buf)), nil +} + +func GetMounts() (map[string]string, error) { + all, err := readProc("/proc/mounts") + if err != nil { + return nil, fmt.Errorf("Couldn't read /proc/mounts") + } + result := make(map[string]string) + lines := strings.Split(all, "\n") + for i := range lines { + parts := strings.Split(lines[i], " ") + //dev := parts[0] + mountPoint := parts[1] + fsType := parts[2] + if strings.Contains(mountPoint, "/mnt/mqm") { + result[mountPoint] = fsType + } + } + return result, nil +} + +func GetKernelVersion() (string, error) { + return readProc("/proc/sys/kernel/osrelease") +} + +func GetMaxFileHandles() (string, error) { + return readProc("/proc/sys/fs/file-max") +} diff --git a/cmd/runmqserver/mqconfig_linux.go b/internal/containerruntime/runtime_linux.go similarity index 84% rename from cmd/runmqserver/mqconfig_linux.go rename to internal/containerruntime/runtime_linux.go index 2a07aa4..3c92fbb 100644 --- a/cmd/runmqserver/mqconfig_linux.go +++ b/internal/containerruntime/runtime_linux.go @@ -1,7 +1,7 @@ // +build linux /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,11 +15,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package main +package runtime import ( - "fmt" - "golang.org/x/sys/unix" ) @@ -101,24 +99,28 @@ var fsTypes = map[int64]string{ 0x58295829: "zsmalloc", } -func checkFS(path string) error { +// GetFilesystem returns the filesystem type for the specified path +func GetFilesystem(path string) (string, error) { statfs := &unix.Statfs_t{} err := unix.Statfs(path, statfs) if err != nil { - log.Println(err) - return nil + return "", err } // Use a type conversion to make type an int64. On s390x it's a uint32. t, ok := fsTypes[int64(statfs.Type)] if !ok { - log.Printf("WARNING: detected %v has unknown filesystem type %x", path, statfs.Type) - return nil + return "unknown", nil + // log.Printf("WARNING: detected %v has unknown filesystem type %x", path, statfs.Type) } - switch t { + return t, nil +} + +// SupportedFilesystem returns true if the supplied filesystem type is supported for MQ data +func SupportedFilesystem(fsType string) bool { + switch fsType { case "aufs", "overlayfs", "tmpfs": - return fmt.Errorf("%v uses unsupported filesystem type: %v", path, t) + return false default: - log.Printf("Detected %v has filesystem type '%v'", path, t) - return nil + return true } } diff --git a/cmd/runmqserver/mqconfig_other.go b/internal/containerruntime/runtime_other.go similarity index 92% rename from cmd/runmqserver/mqconfig_other.go rename to internal/containerruntime/runtime_other.go index 0c4c233..a46f0ff 100644 --- a/cmd/runmqserver/mqconfig_other.go +++ b/internal/containerruntime/runtime_other.go @@ -1,7 +1,7 @@ // +build !linux /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package main +package runtime // Dummy version of this function, only for non-Linux systems. // Having this allows unit tests to be run on other platforms (e.g. macOS) diff --git a/internal/logger/logger.go b/internal/logger/logger.go index c2b2a5a..7278c98 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ type Logger struct { pid string serverName string host string - user *user.User + userName string } // NewLogger creates a new logger @@ -53,9 +53,13 @@ func NewLogger(writer io.Writer, debug bool, json bool, serverName string) (*Log if err != nil { return nil, err } + // This can fail because the container's running as a random UID which + // is not known by the OS. We don't want this to break the logging + // entirely, so just use a blank user name. user, err := user.Current() - if err != nil { - return nil, err + userName := "" + if err == nil { + userName = user.Username } return &Logger{ mutex: sync.Mutex{}, @@ -66,7 +70,7 @@ func NewLogger(writer io.Writer, debug bool, json bool, serverName string) (*Log pid: strconv.Itoa(os.Getpid()), serverName: serverName, host: hostname, - user: user, + userName: userName, }, nil } @@ -93,7 +97,7 @@ func (l *Logger) log(level string, msg string) { "ibm_serverName": l.serverName, "ibm_processName": l.processName, "ibm_processId": l.pid, - "ibm_userName": l.user.Username, + "ibm_userName": l.userName, "type": "mq_containerlog", } s, err := l.format(entry) diff --git a/internal/user/user.go b/internal/user/user.go new file mode 100644 index 0000000..08ffb2f --- /dev/null +++ b/internal/user/user.go @@ -0,0 +1,82 @@ +/* +© Copyright IBM Corporation 2018, 2019 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package user + +import ( + "fmt" + "os/user" + "strings" + + "github.com/ibm-messaging/mq-container/internal/command" +) + +// User holds information on primary and supplemental OS groups +type User struct { + UID string + Name string + PrimaryGID string + SupplementalGID []string +} + +// GetUser returns the current user and group information +func GetUser() (User, error) { + u, err := user.Current() + if err != nil { + return User{}, err + } + g, err := getCurrentUserGroups() + if err != nil { + return User{}, err + } + if err != nil && len(g) == 0 { + return User{ + UID: u.Uid, + Name: u.Name, + PrimaryGID: u.Gid, + SupplementalGID: []string{}, + }, nil + } + // Look for the primary group in the list of group IDs + for i, v := range g { + if v == u.Gid { + // Remove the element from the slice + g = append(g[:i], g[i+1:]...) + } + } + return User{ + UID: u.Uid, + Name: u.Name, + PrimaryGID: u.Gid, + SupplementalGID: g, + }, nil +} + +func getCurrentUserGroups() ([]string, error) { + var nilArray []string + out, _, err := command.Run("id", "--groups") + if err != nil { + return nilArray, err + } + + out = strings.TrimSpace(out) + if out == "" { + // we don't have any groups? + return nilArray, fmt.Errorf("Unable to determine groups for current user") + } + + groups := strings.Split(out, " ") + return groups, nil +} diff --git a/mq-advanced-server-rhel/go-buildah.sh b/mq-advanced-server-rhel/go-buildah.sh index c0d411f..bdf36d5 100755 --- a/mq-advanced-server-rhel/go-buildah.sh +++ b/mq-advanced-server-rhel/go-buildah.sh @@ -36,6 +36,7 @@ IMAGE_SOURCE=${IMAGE_SOURCE:="Not Applicable"} # Run the build in a container # Note the ":Z" on the volume is to allow the container to access the files when SELinux is enabled +# Note the "podman" network is used explicitly, to avoid problems other CNI networks (e.g. on an OpenShift node) podman run \ --volume ${PWD}:/opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/:Z \ --env IMAGE_REVISION="$IMAGE_REVISION" \ diff --git a/mq-advanced-server-rhel/install-mq-rhel.sh b/mq-advanced-server-rhel/install-mq-rhel.sh index a0e795b..36dd7a7 100755 --- a/mq-advanced-server-rhel/install-mq-rhel.sh +++ b/mq-advanced-server-rhel/install-mq-rhel.sh @@ -34,6 +34,8 @@ readonly mnt_mq=$2 readonly archive=$3 readonly mq_packages=$4 readonly dir_extract=/tmp/extract +readonly mqm_uid=888 +readonly mqm_gid=888 if [ ! -d ${dir_extract}/MQServer ]; then mkdir -p ${dir_extract} @@ -42,9 +44,6 @@ if [ ! -d ${dir_extract}/MQServer ]; then echo Extracting finished fi -# If MQ_PACKAGES isn't specifically set, then choose a valid set of defaults - - # Accept the MQ license buildah run --user root --volume ${dir_extract}:/mnt/mq-download:Z $ctr_mq -- /mnt/mq-download/MQServer/mqlicense.sh -text_only -accept @@ -63,13 +62,20 @@ find $mnt_mq/opt/mqm -name '*.tar.gz' -delete buildah run $ctr_mq -- /opt/mqm/bin/setmqinst -p /opt/mqm -i mkdir -p $mnt_mq/run/runmqserver -chown 888:888 $mnt_mq/run/runmqserver +chown ${mqm_uid}:${mqm_gid} $mnt_mq/run/runmqserver # Remove the directory structure under /var/mqm which was created by the installer rm -rf $mnt_mq/var/mqm -# Create the mount point for volumes +# Create the mount point for volumes, ensuring MQ has permissions to all directories mkdir -p $mnt_mq/mnt/mqm +install --directory --mode 0775 --owner ${mqm_uid} --group root $mnt_mq/mnt +install --directory --mode 0775 --owner ${mqm_uid} --group root $mnt_mq/mnt/mqm +install --directory --mode 0775 --owner ${mqm_uid} --group root $mnt_mq/mnt/mqm/data + +# Create the directory for MQ configuration files +mkdir -p /etc/mqm +install --directory --mode 0775 --owner ${mqm_uid} --group root $mnt_mq/etc/mqm # Create a symlink for /var/mqm -> /mnt/mqm/data buildah run --user root $ctr_mq -- ln -s /mnt/mqm/data /var/mqm diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 78b9d3b..f7c4b40 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -88,13 +88,17 @@ buildah run ${ctr_mq} -- microdnf ${microdnf_opts} install \ util-linux \ which +# Install "sudo" if using MQ Advanced for Developers +if [ "$mqdev" = "TRUE" ]; then + buildah run ${ctr_mq} -- microdnf ${microdnf_opts} install sudo +fi + # Clean up cached files buildah run ${ctr_mq} -- microdnf ${microdnf_opts} clean all rm -rf ${mnt_mq}/etc/yum.repos.d/* buildah run --user root $ctr_mq -- groupadd --system --gid ${mqm_gid} mqm -buildah run --user root $ctr_mq -- useradd --system --uid ${mqm_uid} --gid mqm mqm -buildah run --user root $ctr_mq -- usermod -aG root mqm +buildah run --user root $ctr_mq -- useradd --system --uid ${mqm_uid} --gid mqm --groups 0 mqm buildah run --user root $ctr_mq -- usermod -aG mqm root # Install MQ server packages into the MQ builder image @@ -109,6 +113,11 @@ install --mode 0750 --owner ${mqm_uid} --group 0 ./build/runmqserver ${mnt_mq}/u install --mode 6750 --owner ${mqm_uid} --group 0 ./build/chk* ${mnt_mq}/usr/local/bin/ install --mode 0750 --owner ${mqm_uid} --group 0 ./NOTICES.txt ${mnt_mq}/opt/mqm/licenses/notices-container.txt +install --directory --mode 0775 --owner ${mqm_uid} --group 0 ${mnt_mq}/run/runmqserver +buildah run --user root $ctr_mq -- touch /run/termination-log +buildah run --user root $ctr_mq -- chown mqm:root /run/termination-log +buildah run --user root $ctr_mq -- chmod 0660 /run/termination-log + ############################################################################### # Final Buildah commands ############################################################################### @@ -145,7 +154,7 @@ buildah config \ --env LANG=en_US.UTF-8 \ --env LOG_FORMAT=basic \ --entrypoint runmqserver \ - --user root \ + --user ${mqm_uid} \ $ctr_mq buildah unmount $ctr_mq buildah commit $ctr_mq $tag diff --git a/mq-advanced-server-rhel/mqdev-buildah.sh b/mq-advanced-server-rhel/mqdev-buildah.sh index 8103098..1e7c84a 100755 --- a/mq-advanced-server-rhel/mqdev-buildah.sh +++ b/mq-advanced-server-rhel/mqdev-buildah.sh @@ -53,7 +53,12 @@ fi readonly tag=$2 readonly version=$3 +readonly mqm_uid=888 +readonly mqm_gid=888 +# WARNING: This is what allows the mqm user to change the password of any other user +# It's used by runmqdevserver to change the admin/app passwords. +echo "mqm ALL = NOPASSWD: /usr/sbin/chpasswd" > $mnt_mq/etc/sudoers.d/mq-dev-config # Run these commands inside the container so that the SELinux context is handled correctly buildah run --user root $ctr_mq -- useradd --gid mqm admin @@ -61,17 +66,24 @@ buildah run --user root $ctr_mq -- groupadd --system mqclient buildah run --user root $ctr_mq -- useradd --gid mqclient app buildah run --user root $ctr_mq -- bash -c "echo admin:passw0rd | chpasswd" -mkdir -p $mnt_mq/run/runmqdevserver -chown 888:888 $mnt_mq/run/runmqdevserver +mkdir --parents $mnt_mq/run/runmqdevserver +chown ${mqm_uid}:${mqm_gid} $mnt_mq/run/runmqdevserver # Copy runmqdevserver program -install --mode 0750 --owner 888 --group 888 ./build/runmqdevserver ${mnt_mq}/usr/local/bin/ +install --mode 0750 --owner ${mqm_uid} --group ${mqm_gid} ./build/runmqdevserver ${mnt_mq}/usr/local/bin/ + +install --directory --mode 0775 --owner ${mqm_uid} --group 0 ${mnt_mq}/run/runmqdevserver # Copy template files -cp incubating/mqadvanced-server-dev/*.tpl ${mnt_mq}/etc/mqm/ +cp ./incubating/mqadvanced-server-dev/*.tpl ${mnt_mq}/etc/mqm/ # Copy web XML files for default developer configuration -cp -R incubating/mqadvanced-server-dev/web ${mnt_mq}/etc/mqm/web +mkdir --parents ${mnt_mq}/etc/mqm/web +cp --recursive ./incubating/mqadvanced-server-dev/web/* ${mnt_mq}/etc/mqm/web/ + +# Make "mqm" the owner of all the config files +chown --recursive ${mqm_uid}:${mqm_gid} ${mnt_mq}/etc/mqm/* +chmod --recursive 0750 ${mnt_mq}/etc/mqm/* ############################################################################### # Final Buildah commands @@ -102,7 +114,7 @@ buildah config \ --env MQ_ADMIN_PASSWORD=passw0rd \ --env MQ_DEV=true \ --entrypoint runmqdevserver \ - --user root \ + --user ${mqm_uid} \ $ctr_mq buildah unmount $ctr_mq buildah commit $ctr_mq $tag diff --git a/test/docker/devconfig_test_util.go b/test/docker/devconfig_test_util.go index 77afe15..f202d38 100644 --- a/test/docker/devconfig_test_util.go +++ b/test/docker/devconfig_test_util.go @@ -1,7 +1,7 @@ // +build mqdev /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -114,7 +114,7 @@ func runJMSTests(t *testing.T, cli *client.Client, ID string, tls bool, user, pa t.Fatal(err) } startContainer(t, cli, ctr.ID) - rc := waitForContainer(t, cli, ctr.ID, 10) + rc := waitForContainer(t, cli, ctr.ID, 2*time.Minute) if rc != 0 { t.Errorf("JUnit container failed with rc=%v", rc) } diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 190fbc0..65827e7 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -47,11 +47,11 @@ func TestLicenseNotSet(t *testing.T) { containerConfig := container.Config{} id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 5) + rc := waitForContainer(t, cli, id, 10*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } - expectTerminationMessage(t) + expectTerminationMessage(t, cli, id) } func TestLicenseView(t *testing.T) { @@ -65,7 +65,7 @@ func TestLicenseView(t *testing.T) { } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 5) + rc := waitForContainer(t, cli, id, 10*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } @@ -164,12 +164,12 @@ func TestSecurityVulnerabilitiesRedHat(t *testing.T) { t.Fatal(err) } mnt = strings.TrimSpace(mnt) - _, _, err = command.Run("bash", "-c", "cp /etc/yum.repos.d/* "+ filepath.Join(mnt, "/etc/yum.repos.d/")) + _, _, err = command.Run("bash", "-c", "cp /etc/yum.repos.d/* "+filepath.Join(mnt, "/etc/yum.repos.d/")) if err != nil { t.Fatal(err) } out, ret, _ := command.Run("bash", "-c", "yum --installroot="+mnt+" updateinfo list sec | grep /Sec") - if ret != 1{ + if ret != 1 { t.Errorf("Expected no vulnerabilities, found the following:\n%v", out) } } @@ -279,6 +279,70 @@ func TestNoVolumeWithRestart(t *testing.T) { waitForReady(t, cli, id) } +// TestVolumeRequiresRoot tests the case where only the root user can write +// to the persistent volume. In this case, an "init container" is needed, +// where `runmqserver -i` is run to initialize the storage. Then the +// container can be run as normal. +func TestVolumeRequiresRoot(t *testing.T) { + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + vol := createVolume(t, cli) + defer removeVolume(t, cli, vol.Name) + + // Set permissions on the volume to only allow root to write it + // It's important that read and execute permissions are given to other users + rc, _ := runContainerOneShotWithVolume(t, cli, vol.Name+":/mnt/mqm:nocopy", "bash", "-c", "chown 65534:4294967294 /mnt/mqm/ && chmod 0755 /mnt/mqm/ && ls -lan /mnt/mqm/") + if rc != 0 { + t.Errorf("Expected one shot container to return rc=0, got rc=%v", rc) + } + + containerConfig := container.Config{ + Image: imageName(), + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + } + hostConfig := container.HostConfig{ + Binds: []string{ + coverageBind(t), + vol.Name + ":/mnt/mqm:nocopy", + }, + } + networkingConfig := network.NetworkingConfig{} + + // Run an "init container" as root, with the "-i" option, to initialize the volume + containerConfig = container.Config{ + Image: imageName(), + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1", "DEBUG=true"}, + User: "0", + Entrypoint: []string{"runmqserver", "-i"}, + } + initCtr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()+"Init") + if err != nil { + t.Fatal(err) + } + defer cleanContainer(t, cli, initCtr.ID) + t.Logf("Init container ID=%v", initCtr.ID) + startContainer(t, cli, initCtr.ID) + rc = waitForContainer(t, cli, initCtr.ID, 10*time.Second) + if rc != 0 { + t.Errorf("Expected init container to exit with rc=0, got rc=%v", rc) + } + + containerConfig = container.Config{ + Image: imageName(), + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1", "DEBUG=true"}, + } + ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()+"Main") + if err != nil { + t.Fatal(err) + } + defer cleanContainer(t, cli, ctr.ID) + t.Logf("Main container ID=%v", ctr.ID) + startContainer(t, cli, ctr.ID) + waitForReady(t, cli, ctr.ID) +} + // TestCreateQueueManagerFail causes a failure of `crtmqm` func TestCreateQueueManagerFail(t *testing.T) { t.Parallel() @@ -286,24 +350,31 @@ func TestCreateQueueManagerFail(t *testing.T) { if err != nil { t.Fatal(err) } - img, _, err := cli.ImageInspectWithRaw(context.Background(), imageName()) - if err != nil { - t.Fatal(err) + var files = []struct { + Name, Body string + }{ + {"Dockerfile", fmt.Sprintf(` + FROM %v + USER root + RUN echo '#!/bin/bash\nexit 999' > /opt/mqm/bin/crtmqm + RUN chown mqm:mqm /opt/mqm/bin/crtmqm + RUN chmod 6550 /opt/mqm/bin/crtmqm + USER mqm`, imageName())}, } - oldEntrypoint := strings.Join(img.Config.Entrypoint, " ") + tag := createImage(t, cli, files) + defer deleteImage(t, cli, tag) + containerConfig := container.Config{ - Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, - // Override the entrypoint to create the queue manager directory, but leave it empty. - // This will cause `crtmqm` to return with an exit code of 2. - Entrypoint: []string{"bash", "-c", "mkdir -p /mnt/mqm/data && mkdir -p /var/mqm/qmgrs/qm1 && exec " + oldEntrypoint}, + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + Image: tag, } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 10) + rc := waitForContainer(t, cli, id, 10*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } - expectTerminationMessage(t) + expectTerminationMessage(t, cli, id) } // TestStartQueueManagerFail causes a failure of `strmqm` @@ -313,24 +384,31 @@ func TestStartQueueManagerFail(t *testing.T) { if err != nil { t.Fatal(err) } - img, _, err := cli.ImageInspectWithRaw(context.Background(), imageName()) - if err != nil { - t.Fatal(err) + var files = []struct { + Name, Body string + }{ + {"Dockerfile", fmt.Sprintf(` + FROM %v + USER root + RUN echo '#!/bin/bash\ndltmqm $@ && strmqm $@' > /opt/mqm/bin/strmqm + RUN chown mqm:mqm /opt/mqm/bin/strmqm + RUN chmod 6550 /opt/mqm/bin/strmqm + USER mqm`, imageName())}, } - oldEntrypoint := strings.Join(img.Config.Entrypoint, " ") + tag := createImage(t, cli, files) + defer deleteImage(t, cli, tag) + containerConfig := container.Config{ - Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1", "DEBUG=1"}, - // Override the entrypoint to replace `strmqm` with a script which deletes the queue manager. - // This will cause `strmqm` to return with an exit code of 72. - Entrypoint: []string{"bash", "-c", "echo '#!/bin/bash\ndltmqm $@ && strmqm $@' > /opt/mqm/bin/strmqm && exec " + oldEntrypoint}, + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + Image: tag, } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 10) + rc := waitForContainer(t, cli, id, 10*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } - expectTerminationMessage(t) + expectTerminationMessage(t, cli, id) } // TestVolumeUnmount runs a queue manager with a volume, and then forces an @@ -430,7 +508,13 @@ func TestMQSC(t *testing.T) { var files = []struct { Name, Body string }{ - {"Dockerfile", fmt.Sprintf("FROM %v\nRUN rm -f /etc/mqm/*.mqsc\nADD test.mqsc /etc/mqm/", imageName())}, + {"Dockerfile", fmt.Sprintf(` + FROM %v + USER root + RUN rm -f /etc/mqm/*.mqsc + ADD test.mqsc /etc/mqm/ + RUN chmod 0660 /etc/mqm/test.mqsc + USER mqm`, imageName())}, {"test.mqsc", "DEFINE QLOCAL(test)"}, } tag := createImage(t, cli, files) @@ -461,7 +545,13 @@ func TestInvalidMQSC(t *testing.T) { var files = []struct { Name, Body string }{ - {"Dockerfile", fmt.Sprintf("FROM %v\nRUN rm -f /etc/mqm/*.mqsc\nADD mqscTest.mqsc /etc/mqm/", imageName())}, + {"Dockerfile", fmt.Sprintf(` + FROM %v + USER root + RUN rm -f /etc/mqm/*.mqsc + ADD mqscTest.mqsc /etc/mqm/ + RUN chmod 0660 /etc/mqm/mqscTest.mqsc + USER mqm`, imageName())}, {"mqscTest.mqsc", "DEFINE INVALIDLISTENER('TEST.LISTENER.TCP') TRPTYPE(TCP) PORT(1414) CONTROL(QMGR) REPLACE"}, } tag := createImage(t, cli, files) @@ -473,11 +563,11 @@ func TestInvalidMQSC(t *testing.T) { } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 5) + rc := waitForContainer(t, cli, id, 60*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } - expectTerminationMessage(t) + expectTerminationMessage(t, cli, id) } // TestReadiness creates a new image with large amounts of MQSC in, to @@ -497,7 +587,13 @@ func TestReadiness(t *testing.T) { var files = []struct { Name, Body string }{ - {"Dockerfile", fmt.Sprintf("FROM %v\nRUN rm -f /etc/mqm/*.mqsc\nADD test.mqsc /etc/mqm/", imageName())}, + {"Dockerfile", fmt.Sprintf(` + FROM %v + USER root + RUN rm -f /etc/mqm/*.mqsc + ADD test.mqsc /etc/mqm/ + RUN chmod 0660 /etc/mqm/test.mqsc + USER mqm`, imageName())}, {"test.mqsc", buf.String()}, } tag := createImage(t, cli, files) @@ -656,11 +752,11 @@ func TestBadLogFormat(t *testing.T) { } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 5) + rc := waitForContainer(t, cli, id, 5*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } - expectTerminationMessage(t) + expectTerminationMessage(t, cli, id) } // TestMQJSONDisabled tests the case where MQ's JSON logging feature is diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index fd87991..104511e 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -59,6 +59,18 @@ func imageNameDevJMS() string { return image } +func baseImage(t *testing.T, cli *client.Client) string { + rc, out := runContainerOneShot(t, cli, "bash", "-c", "cat /etc/*release | grep \"^ID=\"") + if rc != 0 { + t.Fatal("Couldn't determine base image") + } + s := strings.Split(out, "=") + if len(s) < 2 { + t.Fatal("Couldn't determine base image string") + } + return s[1] +} + // isWSL return whether we are running in the Windows Subsystem for Linux func isWSL(t *testing.T) bool { if runtime.GOOS == "linux" { @@ -124,46 +136,31 @@ func getTempDir(t *testing.T, unixStylePath bool) string { return "/tmp/" } -// terminationLogUnixPath returns the name of the file to use for the termination log message, with a UNIX path -func terminationLogUnixPath(t *testing.T) string { - // Warning: this directory must be accessible to the Docker daemon, - // in order to enable the bind mount - return getTempDir(t, true) + t.Name() + "-termination-log" -} - -// terminationLogOSPath returns the name of the file to use for the termination log message, with an OS specific path -func terminationLogOSPath(t *testing.T) string { - // Warning: this directory must be accessible to the Docker daemon, - // in order to enable the bind mount - return getTempDir(t, false) + t.Name() + "-termination-log" -} - -// terminationBind returns a string to use to bind-mount a termination log file. -// This is done using a bind, because you can't copy files from /dev out of the container. -func terminationBind(t *testing.T) string { - n := terminationLogUnixPath(t) - // Remove it if it already exists - os.Remove(n) - // Create the empty file - f, err := os.OpenFile(n, os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - t.Fatal(err) - } - f.Close() - return terminationLogOSPath(t) + ":/dev/termination-log" -} - // terminationMessage return the termination message, or an empty string if not set -func terminationMessage(t *testing.T) string { - b, err := ioutil.ReadFile(terminationLogUnixPath(t)) +func terminationMessage(t *testing.T, cli *client.Client, ID string) string { + r, _, err := cli.CopyFromContainer(context.Background(), ID, "/run/termination-log") if err != nil { t.Log(err) + return "" } - return string(b) + b, err := ioutil.ReadAll(r) + tr := tar.NewReader(bytes.NewReader(b)) + _, err = tr.Next() + if err != nil { + t.Log(err) + return "" + } + // read the complete content of the file h.Name into the bs []byte + content, err := ioutil.ReadAll(tr) + if err != nil { + t.Log(err) + return "" + } + return string(content) } -func expectTerminationMessage(t *testing.T) { - m := terminationMessage(t) +func expectTerminationMessage(t *testing.T, cli *client.Client, ID string) { + m := terminationMessage(t, cli, ID) if m == "" { t.Error("Expected termination message to be set") } @@ -195,11 +192,10 @@ func cleanContainer(t *testing.T, cli *client.Client, ID string) { // Log the container output for any container we're about to delete t.Logf("Console log from container %v:\n%v", ID, inspectTextLogs(t, cli, ID)) - m := terminationMessage(t) + m := terminationMessage(t, cli, ID) if m != "" { t.Logf("Termination message: %v", m) } - os.Remove(terminationLogUnixPath(t)) t.Logf("Removing container: %s", ID) opts := types.ContainerRemoveOptions{ @@ -212,6 +208,22 @@ func cleanContainer(t *testing.T, cli *client.Client, ID string) { } } +// devImage returns true if the specified image is a developer image, +// determined by use of the MQ_ADMIN_PASSWORD or MQ_APP_PASSWORD +// environment variables +func devImage(t *testing.T, cli *client.Client, imageID string) bool { + i, _, err := cli.ImageInspectWithRaw(context.Background(), imageID) + if err != nil { + t.Fatal(err) + } + for _, e := range i.ContainerConfig.Env { + if strings.HasPrefix(e, "MQ_ADMIN_PASSWORD") || strings.HasPrefix(e, "MQ_APP_PASSWORD") { + return true + } + } + return false +} + // runContainerWithPorts creates and starts a container, exposing the specified ports on the host. // If no image is specified in the container config, then the image name is retrieved from the TEST_IMAGE // environment variable. @@ -219,15 +231,35 @@ func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *co if containerConfig.Image == "" { containerConfig.Image = imageName() } + // Always run as the "mqm" user, unless the test has specified otherwise + if containerConfig.User == "" { + containerConfig.User = "mqm" + } // if coverage containerConfig.Env = append(containerConfig.Env, "COVERAGE_FILE="+t.Name()+".cov") containerConfig.Env = append(containerConfig.Env, "EXIT_CODE_FILE="+getExitCodeFilename(t)) hostConfig := container.HostConfig{ Binds: []string{ coverageBind(t), - terminationBind(t), + // terminationBind(t), }, PortBindings: nat.PortMap{}, + CapDrop: []string{ + "ALL", + }, + } + if devImage(t, cli, containerConfig.Image) { + t.Logf("Detected MQ Advanced for Developers image — adding extra Linux capabilities to container") + hostConfig.CapAdd = []string{ + "CHOWN", + "SETUID", + "SETGID", + "AUDIT_WRITE", + } + // Only needed for a RHEL-based image + if baseImage(t, cli) != "ubuntu" { + hostConfig.CapAdd = append(hostConfig.CapAdd, "DAC_OVERRIDE") + } } for _, p := range ports { port := nat.Port(fmt.Sprintf("%v/tcp", p)) @@ -254,13 +286,49 @@ func runContainer(t *testing.T, cli *client.Client, containerConfig *container.C return runContainerWithPorts(t, cli, containerConfig, nil) } +// runContainerOneShot runs a container with a custom entrypoint, as the root +// user and with default capabilities func runContainerOneShot(t *testing.T, cli *client.Client, command ...string) (int64, string) { containerConfig := container.Config{ Entrypoint: command, + User: "root", + Image: imageName(), } - id := runContainer(t, cli, &containerConfig) - defer cleanContainer(t, cli, id) - return waitForContainer(t, cli, id, 10), inspectLogs(t, cli, id) + hostConfig := container.HostConfig{} + networkingConfig := network.NetworkingConfig{} + t.Logf("Running one shot container (%s)", containerConfig.Image) + ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()) + if err != nil { + t.Fatal(err) + } + startContainer(t, cli, ctr.ID) + defer cleanContainer(t, cli, ctr.ID) + return waitForContainer(t, cli, ctr.ID, 10*time.Second), inspectLogs(t, cli, ctr.ID) +} + +// runContainerOneShot runs a container with a custom entrypoint, as the root +// user, with default capabilities, and a volume mounted +func runContainerOneShotWithVolume(t *testing.T, cli *client.Client, bind string, command ...string) (int64, string) { + containerConfig := container.Config{ + Entrypoint: command, + User: "root", + Image: imageName(), + } + hostConfig := container.HostConfig{ + Binds: []string{ + bind, + }, + } + networkingConfig := network.NetworkingConfig{} + t.Logf("Running one shot container with volume (%s): %v", containerConfig.Image, command) + ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()) + if err != nil { + t.Fatal(err) + } + t.Logf("One shot container ID: %v", ctr.ID) + startContainer(t, cli, ctr.ID) + defer cleanContainer(t, cli, ctr.ID) + return waitForContainer(t, cli, ctr.ID, 10*time.Second), inspectLogs(t, cli, ctr.ID) } func startContainer(t *testing.T, cli *client.Client, ID string) { @@ -309,19 +377,19 @@ func getCoverageExitCode(t *testing.T, orig int64) int64 { } // waitForContainer waits until a container has exited -func waitForContainer(t *testing.T, cli *client.Client, ID string, timeout int64) int64 { - rc, err := cli.ContainerWait(context.Background(), ID) - +func waitForContainer(t *testing.T, cli *client.Client, ID string, timeout time.Duration) int64 { + c, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + rc, err := cli.ContainerWait(c, ID) + if err != nil { + t.Fatal(err) + } if coverage() { // COVERAGE: When running coverage, the exit code is written to a file, // to allow the coverage to be generated (which doesn't happen for non-zero // exit codes) rc = getCoverageExitCode(t, rc) } - - if err != nil { - t.Fatal(err) - } return rc } @@ -395,7 +463,7 @@ func execContainer(t *testing.T, cli *client.Client, ID string, user string, cmd } func waitForReady(t *testing.T, cli *client.Client, ID string) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() for { From 4f57d1bae293bfd207e5c8404a5af231f920a05b Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 26 Feb 2019 09:15:28 +0000 Subject: [PATCH 36/60] Tidy up commented code --- internal/containerruntime/runtime.go | 11 ----------- internal/containerruntime/runtime_linux.go | 1 - internal/user/user.go | 1 - test/docker/docker_api_test_util.go | 1 - 4 files changed, 14 deletions(-) diff --git a/internal/containerruntime/runtime.go b/internal/containerruntime/runtime.go index 3b42b09..49c53d1 100644 --- a/internal/containerruntime/runtime.go +++ b/internal/containerruntime/runtime.go @@ -72,17 +72,6 @@ func GetSecurityAttributes() string { return a } -// func logUser() { -// u, err := user.GetUser() -// if err == nil { -// if len(u.SupplementalGID) == 0 { -// log.Printf("Running as user ID %v (%v) with primary group %v", u.UID, u.Name, u.PrimaryGID) -// } else { -// log.Printf("Running as user ID %v (%v) with primary group %v, and supplementary groups %v", u.UID, u.Name, u.PrimaryGID, strings.Join(u.SupplementalGID, ",")) -// } -// } -// } - func readProc(filename string) (value string, err error) { // #nosec G304 buf, err := ioutil.ReadFile(filename) diff --git a/internal/containerruntime/runtime_linux.go b/internal/containerruntime/runtime_linux.go index 3c92fbb..94c99c3 100644 --- a/internal/containerruntime/runtime_linux.go +++ b/internal/containerruntime/runtime_linux.go @@ -110,7 +110,6 @@ func GetFilesystem(path string) (string, error) { t, ok := fsTypes[int64(statfs.Type)] if !ok { return "unknown", nil - // log.Printf("WARNING: detected %v has unknown filesystem type %x", path, statfs.Type) } return t, nil } diff --git a/internal/user/user.go b/internal/user/user.go index 08ffb2f..2a61aaf 100644 --- a/internal/user/user.go +++ b/internal/user/user.go @@ -73,7 +73,6 @@ func getCurrentUserGroups() ([]string, error) { out = strings.TrimSpace(out) if out == "" { - // we don't have any groups? return nilArray, fmt.Errorf("Unable to determine groups for current user") } diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index 104511e..2aa1500 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -241,7 +241,6 @@ func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *co hostConfig := container.HostConfig{ Binds: []string{ coverageBind(t), - // terminationBind(t), }, PortBindings: nat.PortMap{}, CapDrop: []string{ From a194545f08ba8d8b6084697075015a53dfec438c Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 27 Feb 2019 14:26:23 +0000 Subject: [PATCH 37/60] Don't add root user into mqm group --- install-mq.sh | 1 - mq-advanced-server-rhel/mq-buildah.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/install-mq.sh b/install-mq.sh index c61d4e4..c4b2926 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -106,7 +106,6 @@ $UBUNTU && apt-get autoremove -y # Recommended: Create the mqm user ID with a fixed UID and group, so that the file permissions work between different images groupadd --system --gid ${mqm_uid} mqm useradd --system --uid ${mqm_uid} --gid mqm --groups 0 mqm -usermod -aG mqm root # Find directory containing .deb files $UBUNTU && DIR_DEB=$(find ${DIR_EXTRACT} -name "*.deb" -printf "%h\n" | sort -u | head -1) diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index f7c4b40..a00c030 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -99,7 +99,6 @@ rm -rf ${mnt_mq}/etc/yum.repos.d/* buildah run --user root $ctr_mq -- groupadd --system --gid ${mqm_gid} mqm buildah run --user root $ctr_mq -- useradd --system --uid ${mqm_uid} --gid mqm --groups 0 mqm -buildah run --user root $ctr_mq -- usermod -aG mqm root # Install MQ server packages into the MQ builder image ./mq-advanced-server-rhel/install-mq-rhel.sh ${ctr_mq} "${mnt_mq}" "${archive}" "${packages}" From c441de7d26cca55a87dad0910f08699519eedcae Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 27 Feb 2019 14:27:12 +0000 Subject: [PATCH 38/60] Fixes for tests on RHEL and test log output --- Makefile-RHEL | 2 +- cmd/runmqdevserver/logruntime.go | 5 ++ cmd/runmqserver/logruntime.go | 7 ++ test/docker/docker_api_test_util.go | 122 ++++++++++++++++++++-------- test/messaging/buildah.sh | 39 +++++---- 5 files changed, 122 insertions(+), 53 deletions(-) diff --git a/Makefile-RHEL b/Makefile-RHEL index ab1b2bf..6185c33 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -181,7 +181,7 @@ build-mqgolang-sdk: check-prereqs downloads/$(MQ_SDK_ARCHIVE) .PHONY: build-go-programs build-go-programs: check-prereqs downloads/$(MQ_SDK_ARCHIVE) build-mqgolang-sdk $(info $(SPACER)$(shell printf $(TITLE)"Build go programs"$(END))) - IMAGE_REVISION=$(IMAGE_REVISION) IMAGE_SOURCE=$(IMAGE_SOURCE) sudo mq-advanced-server-rhel/go-buildah.sh "$(MQ_IMAGE_GOLANG_SDK)" "$(MQDEV)" + IMAGE_REVISION=$(IMAGE_REVISION) IMAGE_SOURCE=$(IMAGE_SOURCE) sudo --preserve-env mq-advanced-server-rhel/go-buildah.sh "$(MQ_IMAGE_GOLANG_SDK)" "$(MQDEV)" .PHONY: build-devjmstest build-devjmstest: check-test-prereqs diff --git a/cmd/runmqdevserver/logruntime.go b/cmd/runmqdevserver/logruntime.go index 21aae84..efa9694 100644 --- a/cmd/runmqdevserver/logruntime.go +++ b/cmd/runmqdevserver/logruntime.go @@ -46,12 +46,17 @@ func logContainerDetails() { } } caps, err := containerruntime.GetCapabilities() + capLogged := false if err == nil { for k, v := range caps { if len(v) > 0 { log.Printf("Capabilities (%s set): %v", strings.ToLower(k), strings.Join(v, ",")) + capLogged = true } } + if !capLogged { + log.Print("Capabilities: none") + } } else { log.Errorf("Error getting capabilities: %v", err) } diff --git a/cmd/runmqserver/logruntime.go b/cmd/runmqserver/logruntime.go index 10f1539..2f552f0 100644 --- a/cmd/runmqserver/logruntime.go +++ b/cmd/runmqserver/logruntime.go @@ -50,12 +50,19 @@ func logContainerDetails() error { } } caps, err := containerruntime.GetCapabilities() + capLogged := false if err == nil { for k, v := range caps { if len(v) > 0 { log.Printf("Capabilities (%s set): %v", strings.ToLower(k), strings.Join(v, ",")) + capLogged = true } } + if !capLogged { + log.Print("Capabilities: none") + } + } else { + log.Errorf("Error getting capabilities: %v", err) } sc, err := containerruntime.GetSeccomp() if err == nil { diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index 2aa1500..f755f5b 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -43,6 +43,18 @@ import ( "github.com/docker/go-connections/nat" ) +type containerDetails struct { + ID string + Name string + Image string + Path string + Args []string + CapAdd []string + CapDrop []string + User string + Env []string +} + func imageName() string { image, ok := os.LookupEnv("TEST_IMAGE") if !ok { @@ -59,8 +71,9 @@ func imageNameDevJMS() string { return image } +// baseImage returns the ID of the underlying base image (e.g. "ubuntu" or "rhel") func baseImage(t *testing.T, cli *client.Client) string { - rc, out := runContainerOneShot(t, cli, "bash", "-c", "cat /etc/*release | grep \"^ID=\"") + rc, out := runContainerOneShot(t, cli, "grep", "^ID=", "/etc/os-release") if rc != 0 { t.Fatal("Couldn't determine base image") } @@ -71,6 +84,16 @@ func baseImage(t *testing.T, cli *client.Client) string { return s[1] } +// devImage returns true if the image under test is a developer image, +// determined by use of the MQ_ADMIN_PASSWORD environment variable +func devImage(t *testing.T, cli *client.Client) bool { + rc, _ := runContainerOneShot(t, cli, "printenv", "MQ_ADMIN_PASSWORD") + if rc == 0 { + return true + } + return false +} + // isWSL return whether we are running in the Windows Subsystem for Linux func isWSL(t *testing.T) bool { if runtime.GOOS == "linux" { @@ -166,21 +189,49 @@ func expectTerminationMessage(t *testing.T, cli *client.Client, ID string) { } } -func cleanContainer(t *testing.T, cli *client.Client, ID string) { +// logContainerDetails logs selected details about the container +func logContainerDetails(t *testing.T, cli *client.Client, ID string) { i, err := cli.ContainerInspect(context.Background(), ID) if err == nil { - // Log the results and continue - t.Logf("Inspected container %v: %#v", ID, i) - s, err := json.MarshalIndent(i, "", " ") - if err != nil { - t.Fatal(err) + d := containerDetails{ + ID: ID, + Name: i.Name, + Image: i.Image, + Path: i.Path, + Args: i.Args, + CapAdd: i.HostConfig.CapAdd, + CapDrop: i.HostConfig.CapDrop, + User: i.Config.User, + Env: i.Config.Env, } - t.Logf("Inspected container %v: %v", ID, string(s)) + // If you need more details, you can always just run `json.MarshalIndent(i, "", " ")` to see everything. + t.Logf("Container details: %+v", d) } +} + +func cleanContainerQuiet(t *testing.T, cli *client.Client, ID string) { + timeout := 10 * time.Second + err := cli.ContainerStop(context.Background(), ID, &timeout) + if err != nil { + // Just log the error and continue + t.Log(err) + } + opts := types.ContainerRemoveOptions{ + RemoveVolumes: true, + Force: true, + } + err = cli.ContainerRemove(context.Background(), ID, opts) + if err != nil { + t.Error(err) + } +} + +func cleanContainer(t *testing.T, cli *client.Client, ID string) { + logContainerDetails(t, cli, ID) t.Logf("Stopping container: %v", ID) timeout := 10 * time.Second // Stop the container. This allows the coverage output to be generated. - err = cli.ContainerStop(context.Background(), ID, &timeout) + err := cli.ContainerStop(context.Background(), ID, &timeout) if err != nil { // Just log the error and continue t.Log(err) @@ -208,22 +259,6 @@ func cleanContainer(t *testing.T, cli *client.Client, ID string) { } } -// devImage returns true if the specified image is a developer image, -// determined by use of the MQ_ADMIN_PASSWORD or MQ_APP_PASSWORD -// environment variables -func devImage(t *testing.T, cli *client.Client, imageID string) bool { - i, _, err := cli.ImageInspectWithRaw(context.Background(), imageID) - if err != nil { - t.Fatal(err) - } - for _, e := range i.ContainerConfig.Env { - if strings.HasPrefix(e, "MQ_ADMIN_PASSWORD") || strings.HasPrefix(e, "MQ_APP_PASSWORD") { - return true - } - } - return false -} - // runContainerWithPorts creates and starts a container, exposing the specified ports on the host. // If no image is specified in the container config, then the image name is retrieved from the TEST_IMAGE // environment variable. @@ -247,7 +282,7 @@ func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *co "ALL", }, } - if devImage(t, cli, containerConfig.Image) { + if devImage(t, cli) { t.Logf("Detected MQ Advanced for Developers image — adding extra Linux capabilities to container") hostConfig.CapAdd = []string{ "CHOWN", @@ -259,6 +294,8 @@ func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *co if baseImage(t, cli) != "ubuntu" { hostConfig.CapAdd = append(hostConfig.CapAdd, "DAC_OVERRIDE") } + } else { + t.Logf("Detected MQ Advanced image - dropping all capabilities") } for _, p := range ports { port := nat.Port(fmt.Sprintf("%v/tcp", p)) @@ -295,14 +332,21 @@ func runContainerOneShot(t *testing.T, cli *client.Client, command ...string) (i } hostConfig := container.HostConfig{} networkingConfig := network.NetworkingConfig{} - t.Logf("Running one shot container (%s)", containerConfig.Image) - ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()) + t.Logf("Running one shot container (%s): %v", containerConfig.Image, command) + ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()+"OneShot") if err != nil { t.Fatal(err) } - startContainer(t, cli, ctr.ID) - defer cleanContainer(t, cli, ctr.ID) - return waitForContainer(t, cli, ctr.ID, 10*time.Second), inspectLogs(t, cli, ctr.ID) + startOptions := types.ContainerStartOptions{} + err = cli.ContainerStart(context.Background(), ctr.ID, startOptions) + if err != nil { + t.Fatal(err) + } + defer cleanContainerQuiet(t, cli, ctr.ID) + rc := waitForContainer(t, cli, ctr.ID, 10*time.Second) + out := inspectLogs(t, cli, ctr.ID) + t.Logf("One shot container finished with rc=%v, output=%v", rc, out) + return rc, out } // runContainerOneShot runs a container with a custom entrypoint, as the root @@ -320,14 +364,20 @@ func runContainerOneShotWithVolume(t *testing.T, cli *client.Client, bind string } networkingConfig := network.NetworkingConfig{} t.Logf("Running one shot container with volume (%s): %v", containerConfig.Image, command) - ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()) + ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name()+"OneShotVolume") if err != nil { t.Fatal(err) } - t.Logf("One shot container ID: %v", ctr.ID) - startContainer(t, cli, ctr.ID) - defer cleanContainer(t, cli, ctr.ID) - return waitForContainer(t, cli, ctr.ID, 10*time.Second), inspectLogs(t, cli, ctr.ID) + startOptions := types.ContainerStartOptions{} + err = cli.ContainerStart(context.Background(), ctr.ID, startOptions) + if err != nil { + t.Fatal(err) + } + defer cleanContainerQuiet(t, cli, ctr.ID) + rc := waitForContainer(t, cli, ctr.ID, 10*time.Second) + out := inspectLogs(t, cli, ctr.ID) + t.Logf("One shot container finished with rc=%v, output=%v", rc, out) + return rc, out } func startContainer(t *testing.T, cli *client.Client, ID string) { diff --git a/test/messaging/buildah.sh b/test/messaging/buildah.sh index 292b4ec..1d8c6a4 100755 --- a/test/messaging/buildah.sh +++ b/test/messaging/buildah.sh @@ -1,5 +1,5 @@ #!/bin/bash -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,24 +22,34 @@ set -e # Use a "scratch" container, so the resulting image has minimal files # Resulting image won't have yum, for example -readonly ctr_mq=$(buildah from rhel7) +readonly ctr_mq=$(buildah from rhel7-minimal) readonly mnt_mq=$(buildah mount $ctr_mq) readonly imagename=$1 -buildah run $ctr_mq -- yum install -y \ - java-1.7.0-openjdk-devel \ - java \ - which \ - wget +microdnf_opts="--nodocs" +# Check whether the host is registered with Red Hat +if subscription-manager status ; then + # Host is subscribed, but the minimal image has no enabled repos + # Note that the "bc" package is the only one in "extras" + microdnf_opts="${microdnf_opts} --enablerepo=rhel-7-server-rpms --enablerepo=rhel-7-server-extras-rpms" +else + # Use the Yum repositories configured on the host + cp -R /etc/yum.repos.d/* ${mnt_mq}/etc/yum.repos.d/ +fi +buildah run ${ctr_mq} -- microdnf ${microdnf_opts} install \ + java-1.8.0-openjdk-devel \ + java \ + which \ + wget -buildah run $ctr_mq -- sh -c "cd /tmp && wget http://mirror.olnevhost.net/pub/apache/maven/binaries/apache-maven-3.2.2-bin.tar.gz" -tar xvf $mnt_mq/tmp/apache-maven-3.2.2-bin.tar.gz -C $mnt_mq/tmp/ +buildah run $ctr_mq -- sh -c "cd /tmp && wget https://www-eu.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz" +tar xvf $mnt_mq/tmp/apache-maven-3.6.0-bin.tar.gz -C $mnt_mq/tmp/ mkdir -p $mnt_mq/usr/src/mymaven cp pom.xml $mnt_mq/usr/src/mymaven/ cp -R src $mnt_mq/usr/src/mymaven/src -buildah run $ctr_mq -- sh -c "cd /usr/src/mymaven && export M2_HOME=/tmp/apache-maven-3.2.2 && export M2=\$M2_HOME/bin && export PATH=\$M2:\$PATH && mvn --version && mvn dependency:go-offline install && mvn --offline install" +buildah run $ctr_mq -- sh -c "cd /usr/src/mymaven && export M2_HOME=/tmp/apache-maven-3.6.0 && export M2=\$M2_HOME/bin && export PATH=\$M2:\$PATH && mvn --version && mvn dependency:go-offline install && mvn --offline install" mkdir -p $mnt_mq/opt/app @@ -53,13 +63,9 @@ cp $mnt_mq/usr/src/mymaven/target/lib/*.jar $mnt_mq/opt/app/ rm -rf $mnt_mq/tmp/* rm -rf $mnt_mq/usr/src/mymaven -# We can't uninstall tar or gzip because they are required -buildah run $ctr_mq -- yum remove -y \ - wget - # Clean up cached files -buildah run $ctr_mq -- yum clean all -rm -rf ${mnt_mq}/var/cache/yum/* +buildah run ${ctr_mq} -- microdnf ${microdnf_opts} clean all +rm -rf ${mnt_mq}/etc/yum.repos.d/* ############################################################################### # Contain image finalization @@ -69,6 +75,7 @@ buildah config \ --os linux \ --label architecture=x86_64 \ --label name="${imagename%:*}" \ + --cmd "" \ --entrypoint '["java", "-classpath", "/opt/app/*", "org.junit.platform.console.ConsoleLauncher", "-p", "com.ibm.mqcontainer.test", "--details", "verbose"]' \ $ctr_mq buildah unmount $ctr_mq From 281cdc4578c197d84064cccc780471168ddc2853 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Mar 2019 09:47:58 +0000 Subject: [PATCH 39/60] Update RHEL build README --- mq-advanced-server-rhel/README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mq-advanced-server-rhel/README.md b/mq-advanced-server-rhel/README.md index c446168..ad6e438 100644 --- a/mq-advanced-server-rhel/README.md +++ b/mq-advanced-server-rhel/README.md @@ -1,7 +1,3 @@ -This is a work-in-progress for a Docker image based on Red Hat Enterprise Linux (RHEL). +# RHEL-based container build -The current MQ container build requires Docker V17.05 or greater (required features include multi-stage Docker build, and "ARG"s in the "FROM" statement). Red Hat Enterprise Linux V7.5 includes Docker up to version V1.13. - -In order to build images with Red Hat Enterprise Linux, license registration is required. The license of the host server can be used, as long as you either use Red Hat's patched version of Docker (which is an old version), or if you use alternative container management tools such as [`buildah`](https://github.com/projectatomic/buildah/) and `podman` (from [`libpod`](https://github.com/projectatomic/libpod)). - -This directory contains scripts for building with `buildah`. The build itself isn't containerized, so more software than usual is needed on the RHEL host, so an Ansible playbook is also provided to help set up the host. \ No newline at end of file +Build scripts for building a container image based on Red Hat Enterprise Linux (RHEL), using the [`buildah`](https://github.com/containers/buildah) tool. buildah is supported on RHEL V7.5 and greater. \ No newline at end of file From 5ef532d2c12ad260e72620ee071503c035641552 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Mar 2019 10:19:21 +0000 Subject: [PATCH 40/60] Don't use redist client for Go SDK because of P+Z support --- Makefile-UBUNTU | 8 ++++++-- incubating/mq-golang-sdk/Dockerfile | 24 ++++++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index c4a8f10..c8b9354 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -266,11 +266,15 @@ else build-sdk-ex: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesSDK-*.rpm MQSeriesSamples*.rpm endif build-sdk-ex: docker-version docker-pull + $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_SDK)"$(END))) $(call docker-build-mq,$(MQ_IMAGE_SDK),incubating/mq-sdk/Dockerfile,$(MQ_SDK_ARCHIVE),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers SDK (Non-Warranted)",$(MQ_VERSION)) .PHONY: build-golang-sdk -build-golang-sdk: - $(DOCKER) build -t $(MQ_IMAGE_GOLANG_SDK) -f incubating/mq-golang-sdk/Dockerfile . +build-golang-sdk: docker-version build-sdk + $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_GOLANG_SDK)"$(END))) + @echo hello + $(DOCKER) build --build-arg BASE_IMAGE=$(MQ_IMAGE_SDK) -t $(MQ_IMAGE_GOLANG_SDK) -f incubating/mq-golang-sdk/Dockerfile . + @echo goodbye .PHONY: docker-pull docker-pull: diff --git a/incubating/mq-golang-sdk/Dockerfile b/incubating/mq-golang-sdk/Dockerfile index b66368f..ff5dc3b 100644 --- a/incubating/mq-golang-sdk/Dockerfile +++ b/incubating/mq-golang-sdk/Dockerfile @@ -11,11 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +ARG BASE_IMAGE=mq-sdk:9.1.1.0-x86_64-ubuntu-16.04 -FROM golang:1.10 +FROM $BASE_IMAGE -# Install the MQ redistributable client (including header files) into the Go builder image -RUN mkdir -p /opt/mqm \ - && curl -L https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist/9.1.1.0-IBM-MQC-Redist-LinuxX64.tar.gz | tar -xz -C /opt/mqm -ENV CGO_CFLAGS="-I/opt/mqm/inc/" \ - CGO_LDFLAGS_ALLOW="-Wl,-rpath.*" \ No newline at end of file +COPY incubating/mq-golang-sdk/install-golang.sh /usr/local/bin + +ENV GO_VERSION=1.10 + +ENV PATH="${PATH}:/usr/lib/go-${GO_VERSION}/bin:/go/bin:/usr/local/go/bin" \ + CGO_CFLAGS="-I/opt/mqm/inc/" \ + CGO_LDFLAGS_ALLOW="-Wl,-rpath.*" \ + GOPATH="/go" + +# Install the Go compiler and Git +RUN chmod +x /usr/local/bin/install-golang.sh \ + && sleep 1 \ + && install-golang.sh + +WORKDIR $GOPATH \ No newline at end of file From 0047301335d05257e3ba599d85a53ce2cdab867f Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Mar 2019 11:31:27 +0000 Subject: [PATCH 41/60] Use Makefile target build-sdk-ex --- Makefile-UBUNTU | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index c8b9354..d8a875d 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -270,7 +270,7 @@ build-sdk-ex: docker-version docker-pull $(call docker-build-mq,$(MQ_IMAGE_SDK),incubating/mq-sdk/Dockerfile,$(MQ_SDK_ARCHIVE),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers SDK (Non-Warranted)",$(MQ_VERSION)) .PHONY: build-golang-sdk -build-golang-sdk: docker-version build-sdk +build-golang-sdk: docker-version build-sdk-ex $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_GOLANG_SDK)"$(END))) @echo hello $(DOCKER) build --build-arg BASE_IMAGE=$(MQ_IMAGE_SDK) -t $(MQ_IMAGE_GOLANG_SDK) -f incubating/mq-golang-sdk/Dockerfile . From d4a81741cc773e99dc43fbb4c3e225361a03c477 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Mar 2019 15:07:51 +0000 Subject: [PATCH 42/60] Prevent re-download of image during build --- Makefile-UBUNTU | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index d8a875d..b6dadc4 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -270,7 +270,10 @@ build-sdk-ex: docker-version docker-pull $(call docker-build-mq,$(MQ_IMAGE_SDK),incubating/mq-sdk/Dockerfile,$(MQ_SDK_ARCHIVE),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers SDK (Non-Warranted)",$(MQ_VERSION)) .PHONY: build-golang-sdk -build-golang-sdk: docker-version build-sdk-ex +build-golang-sdk: downloads/$(MQ_SDK_ARCHIVE) build-golang-sdk-ex + +.PHONY: build-golang-sdk-ex +build-golang-sdk-ex: docker-version build-sdk-ex $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_GOLANG_SDK)"$(END))) @echo hello $(DOCKER) build --build-arg BASE_IMAGE=$(MQ_IMAGE_SDK) -t $(MQ_IMAGE_GOLANG_SDK) -f incubating/mq-golang-sdk/Dockerfile . From 13f620f21a1d0ad7525e816075e71113a86a5fc5 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 5 Mar 2019 15:08:07 +0000 Subject: [PATCH 43/60] Increase test timeouts --- test/docker/docker_api_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 65827e7..f4c4e35 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -47,7 +47,7 @@ func TestLicenseNotSet(t *testing.T) { containerConfig := container.Config{} id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 10*time.Second) + rc := waitForContainer(t, cli, id, 20*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } @@ -65,7 +65,7 @@ func TestLicenseView(t *testing.T) { } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 10*time.Second) + rc := waitForContainer(t, cli, id, 20*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } @@ -324,7 +324,7 @@ func TestVolumeRequiresRoot(t *testing.T) { defer cleanContainer(t, cli, initCtr.ID) t.Logf("Init container ID=%v", initCtr.ID) startContainer(t, cli, initCtr.ID) - rc = waitForContainer(t, cli, initCtr.ID, 10*time.Second) + rc = waitForContainer(t, cli, initCtr.ID, 20*time.Second) if rc != 0 { t.Errorf("Expected init container to exit with rc=0, got rc=%v", rc) } @@ -404,7 +404,7 @@ func TestStartQueueManagerFail(t *testing.T) { } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 10*time.Second) + rc := waitForContainer(t, cli, id, 20*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } @@ -752,7 +752,7 @@ func TestBadLogFormat(t *testing.T) { } id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 5*time.Second) + rc := waitForContainer(t, cli, id, 20*time.Second) if rc != 1 { t.Errorf("Expected rc=1, got rc=%v", rc) } From 68480381657e7157e8c96b1919aadb7d9b10503a Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Thu, 28 Feb 2019 11:20:09 +0000 Subject: [PATCH 44/60] hard code the redist client version --- Makefile-RHEL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile-RHEL b/Makefile-RHEL index 6185c33..1a13dcc 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -28,7 +28,7 @@ MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION_VRM)_LINUX_$(MQ_ARCHIVE_ARCH).tar.gz # for Developers can be installed MQ_ARCHIVE_DEV ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) # MQ_SDK_ARCHIVE specifies the archive to use for the MQ redistributable client, which is used for building the golang programs. -MQ_SDK_ARCHIVE ?= $(MQ_VERSION)-IBM-MQC-Redist-LinuxX64.tar.gz +MQ_SDK_ARCHIVE ?= 9.1.1.0-IBM-MQC-Redist-LinuxX64.tar.gz # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image From 194b04ac13a60165f231d503feb7163c799e9461 Mon Sep 17 00:00:00 2001 From: LPowlett <44359672+LPowlett@users.noreply.github.com> Date: Fri, 8 Mar 2019 09:13:26 +0000 Subject: [PATCH 45/60] Revert MQSC error check (#283) * Revert MQSC error check --- CHANGELOG.md | 2 +- cmd/runmqserver/qmgr.go | 1 - test/docker/docker_api_test.go | 64 +++++++++++++++++----------------- 3 files changed, 33 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a16087..f469689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## vNext * Now runs using the "mqm" user instead of root. See new [security doc](https://github.com/ibm-messaging/mq-container/blob/master/docs/security.md) -* [New IGNSTATE parameter for runmqsc START and STOP commands](https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_9.1.0/com.ibm.mq.pro.doc/q132310_.htm#q132310___ignstateparm) - From MQ version 9.1.1.0, any MQSC scripts included in the image should make use of the `IGNSTATE(YES)` parameter on any `START` and `STOP` commands. This allows for consistency when executing scripts multiple times (e.g. when a container is restarted) +* New [IGNSTATE](https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_9.1.0/com.ibm.mq.pro.doc/q132310_.htm#q132310___ignstateparm) parameter used in default developer config * Termination log moved from `/dev/termination-log` to `/run/termination-log`, to make permissions easier to handle * Fixes for the following issues: * Brackets no longer appear in termination log diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index 5705f45..49b73c1 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -120,7 +120,6 @@ func configureQueueManager() error { out, err := cmd.CombinedOutput() if err != nil { log.Errorf("Error running MQSC file %v (%v):\n\t%v", file.Name(), err, strings.Replace(string(out), "\n", "\n\t", -1)) - return err } // Print the runmqsc output, adding tab characters to make it more readable as part of the log log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1)) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index f4c4e35..a761112 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -536,39 +536,39 @@ func TestMQSC(t *testing.T) { // TestInvalidMQSC creates a new image with an MQSC file containing invalid MQSC, // tries to start a container based on that image, and checks that container terminates -func TestInvalidMQSC(t *testing.T) { - t.Parallel() - cli, err := client.NewEnvClient() - if err != nil { - t.Fatal(err) - } - var files = []struct { - Name, Body string - }{ - {"Dockerfile", fmt.Sprintf(` - FROM %v - USER root - RUN rm -f /etc/mqm/*.mqsc - ADD mqscTest.mqsc /etc/mqm/ - RUN chmod 0660 /etc/mqm/mqscTest.mqsc - USER mqm`, imageName())}, - {"mqscTest.mqsc", "DEFINE INVALIDLISTENER('TEST.LISTENER.TCP') TRPTYPE(TCP) PORT(1414) CONTROL(QMGR) REPLACE"}, - } - tag := createImage(t, cli, files) - defer deleteImage(t, cli, tag) +// func TestInvalidMQSC(t *testing.T) { +// t.Parallel() +// cli, err := client.NewEnvClient() +// if err != nil { +// t.Fatal(err) +// } +// var files = []struct { +// Name, Body string +// }{ +// {"Dockerfile", fmt.Sprintf(` +// FROM %v +// USER root +// RUN rm -f /etc/mqm/*.mqsc +// ADD mqscTest.mqsc /etc/mqm/ +// RUN chmod 0660 /etc/mqm/mqscTest.mqsc +// USER mqm`, imageName())}, +// {"mqscTest.mqsc", "DEFINE INVALIDLISTENER('TEST.LISTENER.TCP') TRPTYPE(TCP) PORT(1414) CONTROL(QMGR) REPLACE"}, +// } +// tag := createImage(t, cli, files) +// defer deleteImage(t, cli, tag) - containerConfig := container.Config{ - Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, - Image: tag, - } - id := runContainer(t, cli, &containerConfig) - defer cleanContainer(t, cli, id) - rc := waitForContainer(t, cli, id, 60*time.Second) - if rc != 1 { - t.Errorf("Expected rc=1, got rc=%v", rc) - } - expectTerminationMessage(t, cli, id) -} +// containerConfig := container.Config{ +// Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, +// Image: tag, +// } +// id := runContainer(t, cli, &containerConfig) +// defer cleanContainer(t, cli, id) +// rc := waitForContainer(t, cli, id, 60*time.Second) +// if rc != 1 { +// t.Errorf("Expected rc=1, got rc=%v", rc) +// } +// expectTerminationMessage(t, cli, id) +// } // TestReadiness creates a new image with large amounts of MQSC in, to // ensure that the readiness check doesn't pass until configuration has finished. From d4df05fd2a9f419a582bf08d25a9febedc7e1b50 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Thu, 28 Feb 2019 12:35:39 +0000 Subject: [PATCH 46/60] Add test fixes and fix mq-explorer build --- Makefile-UBUNTU | 9 +++- incubating/mq-explorer/Dockerfile | 4 +- incubating/mqadvanced-server-dev/Dockerfile | 8 ++-- test/docker/devconfig_test.go | 4 ++ test/docker/docker_api_test.go | 50 ++++++++++++++++++--- test/docker/docker_api_test_util.go | 4 +- test/docker/mqmetric_test.go | 9 ++++ 7 files changed, 74 insertions(+), 14 deletions(-) diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index b6dadc4..95f1c6a 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -231,7 +231,7 @@ docker-version: .PHONY: build-advancedserver build-advancedserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE) -build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version build-golang-sdk +build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version build-golang-sdk-ex $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END))) $(call docker-build-mq,$(MQ_IMAGE_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) @@ -243,7 +243,7 @@ else build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm endif build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV) -build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk +build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk-ex $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER_BASE)"$(END))) $(call docker-build-mq,$(MQ_IMAGE_DEVSERVER_BASE),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) $(DOCKER) build --tag $(MQ_IMAGE_DEVSERVER) --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" --build-arg IMAGE_TAG="$(MQ_IMAGE_DEVSERVER)" --build-arg BASE_IMAGE=$(MQ_IMAGE_DEVSERVER_BASE) --build-arg BUILDER_IMAGE=$(MQ_IMAGE_GOLANG_SDK) --build-arg MQM_UID=$(MQM_UID) --file incubating/mqadvanced-server-dev/Dockerfile . @@ -253,6 +253,11 @@ build-advancedserver-cover: docker-version $(DOCKER) build --build-arg BASE_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_IMAGE_ADVANCEDSERVER)-cover -f Dockerfile-server.cover . .PHONY: build-explorer +ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu" +build-explorer: MQ_PACKAGES=ibmmq-explorer +else +build-explorer: MQ_PACKAGES=MQSeriesRuntime*.rpm MQSeriesJRE*.rpm MQSeriesExplorer*.rpm +endif build-explorer: downloads/$(MQ_ARCHIVE_DEV) docker-pull $(call docker-build-mq,mq-explorer:latest-$(ARCH),incubating/mq-explorer/Dockerfile,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) diff --git a/incubating/mq-explorer/Dockerfile b/incubating/mq-explorer/Dockerfile index f4b7fe2..327c901 100644 --- a/incubating/mq-explorer/Dockerfile +++ b/incubating/mq-explorer/Dockerfile @@ -18,7 +18,7 @@ FROM ubuntu:16.04 ARG MQ_URL=https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev911_ubuntu_x86-64.tar.gz # The MQ packages to install -ARG MQ_PACKAGES="ibmmq-explorer" +ARG MQ_PACKAGES ARG MQM_UID=999 @@ -29,7 +29,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \ libxtst6 ADD install-mq.sh /usr/local/bin/ -RUN chmod u+x /usr/local/bin/install-mq.sh $MQM_UID \ +RUN chmod u+x /usr/local/bin/install-mq.sh \ && install-mq.sh ENV LANG=en_US.UTF-8 diff --git a/incubating/mqadvanced-server-dev/Dockerfile b/incubating/mqadvanced-server-dev/Dockerfile index 3d80875..bf2d93c 100644 --- a/incubating/mqadvanced-server-dev/Dockerfile +++ b/incubating/mqadvanced-server-dev/Dockerfile @@ -71,11 +71,13 @@ COPY --from=builder /go/src/github.com/ibm-messaging/mq-container/runmqserver /u COPY --from=builder /go/src/github.com/ibm-messaging/mq-container/runmqdevserver /usr/local/bin/ # Copy template files -COPY --chown=mqm:mqm incubating/mqadvanced-server-dev/*.tpl /etc/mqm/ +COPY incubating/mqadvanced-server-dev/*.tpl /etc/mqm/ # Copy web XML files for default developer configuration -COPY --chown=mqm:mqm incubating/mqadvanced-server-dev/web /etc/mqm/web +COPY incubating/mqadvanced-server-dev/web /etc/mqm/web -RUN chmod +x /usr/local/bin/runmq* \ +RUN chown -R mqm:mqm /etc/mqm/* \ + && ls -lar /etc/mqm/ \ + && chmod +x /usr/local/bin/runmq* \ && install --directory --mode 0775 --owner mqm --group root /run/runmqdevserver EXPOSE 9443 diff --git a/test/docker/devconfig_test.go b/test/docker/devconfig_test.go index 711e9ee..68122da 100644 --- a/test/docker/devconfig_test.go +++ b/test/docker/devconfig_test.go @@ -33,6 +33,7 @@ import ( // Note: This test requires a separate container image to be available for the JMS tests. func TestDevGoldenPath(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -66,6 +67,7 @@ func TestDevGoldenPath(t *testing.T) { // Note: This test requires a separate container image to be available for the JMS tests func TestDevSecure(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -127,6 +129,7 @@ func TestDevSecure(t *testing.T) { func TestDevWebDisabled(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -157,6 +160,7 @@ func TestDevWebDisabled(t *testing.T) { func TestDevConfigDisabled(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index f4c4e35..09d0f14 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -40,6 +40,7 @@ import ( func TestLicenseNotSet(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -56,6 +57,7 @@ func TestLicenseNotSet(t *testing.T) { func TestLicenseView(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -79,12 +81,14 @@ func TestLicenseView(t *testing.T) { // TestGoldenPath starts a queue manager successfully when metrics are enabled func TestGoldenPathWithMetrics(t *testing.T) { t.Parallel() + goldenPath(t, true) } // TestGoldenPath starts a queue manager successfully when metrics are disabled func TestGoldenPathNoMetrics(t *testing.T) { t.Parallel() + goldenPath(t, false) } @@ -112,6 +116,7 @@ func goldenPath(t *testing.T, metric bool) { // by Ubuntu func TestSecurityVulnerabilitiesUbuntu(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -141,6 +146,7 @@ func TestSecurityVulnerabilitiesUbuntu(t *testing.T) { // by Red Hat func TestSecurityVulnerabilitiesRedHat(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -153,22 +159,25 @@ func TestSecurityVulnerabilitiesRedHat(t *testing.T) { if rc != 0 { t.Skip("Skipping test because container is not RedHat-based") } - id, _, err := command.Run("buildah", "from", imageName()) + id, _, err := command.Run("sudo", "buildah", "from", imageName()) if err != nil { + t.Log(id) t.Fatal(err) } id = strings.TrimSpace(id) defer command.Run("buildah", "rm", id) - mnt, _, err := command.Run("buildah", "mount", id) + mnt, _, err := command.Run("sudo", "buildah", "mount", id) if err != nil { + t.Log(mnt) t.Fatal(err) } mnt = strings.TrimSpace(mnt) - _, _, err = command.Run("bash", "-c", "cp /etc/yum.repos.d/* "+filepath.Join(mnt, "/etc/yum.repos.d/")) + out, _, err := command.Run("bash", "-c", "sudo cp /etc/yum.repos.d/* "+filepath.Join(mnt, "/etc/yum.repos.d/")) if err != nil { + t.Log(out) t.Fatal(err) } - out, ret, _ := command.Run("bash", "-c", "yum --installroot="+mnt+" updateinfo list sec | grep /Sec") + out, ret, _ = command.Run("bash", "-c", "yum --installroot="+mnt+" updateinfo list sec | grep /Sec") if ret != 1 { t.Errorf("Expected no vulnerabilities, found the following:\n%v", out) } @@ -194,11 +203,13 @@ func utilTestNoQueueManagerName(t *testing.T, hostName string, expectedName stri } func TestNoQueueManagerName(t *testing.T) { t.Parallel() + utilTestNoQueueManagerName(t, "test", "test") } func TestNoQueueManagerNameInvalidHostname(t *testing.T) { t.Parallel() + utilTestNoQueueManagerName(t, "test-1", "test1") } @@ -206,6 +217,7 @@ func TestNoQueueManagerNameInvalidHostname(t *testing.T) { // container and starts a new one with same volume. With metrics enabled func TestWithVolumeAndMetrics(t *testing.T) { t.Parallel() + withVolume(t, true) } @@ -213,6 +225,7 @@ func TestWithVolumeAndMetrics(t *testing.T) { // container and starts a new one with same volume. With metrics disabled func TestWithVolumeNoMetrics(t *testing.T) { t.Parallel() + withVolume(t, false) } @@ -264,6 +277,7 @@ func withVolume(t *testing.T, metric bool) { // and restarted cleanly func TestNoVolumeWithRestart(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -284,6 +298,7 @@ func TestNoVolumeWithRestart(t *testing.T) { // where `runmqserver -i` is run to initialize the storage. Then the // container can be run as normal. func TestVolumeRequiresRoot(t *testing.T) { + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -346,6 +361,7 @@ func TestVolumeRequiresRoot(t *testing.T) { // TestCreateQueueManagerFail causes a failure of `crtmqm` func TestCreateQueueManagerFail(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -380,6 +396,7 @@ func TestCreateQueueManagerFail(t *testing.T) { // TestStartQueueManagerFail causes a failure of `strmqm` func TestStartQueueManagerFail(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -417,6 +434,7 @@ func TestStartQueueManagerFail(t *testing.T) { // attached storage gets unmounted. func TestVolumeUnmount(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -465,6 +483,7 @@ func TestVolumeUnmount(t *testing.T) { // created, then checks that no zombies exist (runmqserver should reap them) func TestZombies(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -501,6 +520,7 @@ func TestZombies(t *testing.T) { // on that image, and checks that the MQSC has been applied correctly. func TestMQSC(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -538,6 +558,7 @@ func TestMQSC(t *testing.T) { // tries to start a container based on that image, and checks that container terminates func TestInvalidMQSC(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -575,6 +596,7 @@ func TestInvalidMQSC(t *testing.T) { // WARNING: This test is sensitive to the speed of the machine it's running on. func TestReadiness(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -629,22 +651,34 @@ func TestReadiness(t *testing.T) { func TestErrorLogRotation(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) } + + logsize := 65536 + + rc, _ := runContainerOneShot(t, cli, "bash", "-c", "test -d /etc/apt") + if rc != 0 { + // RHEL + logsize = 32768 + } + qmName := "qm1" containerConfig := container.Config{ Env: []string{ "LICENSE=accept", "MQ_QMGR_NAME=" + qmName, - "MQMAXERRORLOGSIZE=65536", + fmt.Sprintf("MQMAXERRORLOGSIZE=%d", logsize), "LOG_FORMAT=json", + fmt.Sprintf("AMQ_EXTRA_QM_STANZAS=QMErrorLog:ErrorLogSize=%d", logsize), }, ExposedPorts: nat.PortSet{ "1414/tcp": struct{}{}, }, } + id := runContainer(t, cli, &containerConfig) defer cleanContainer(t, cli, id) waitForReady(t, cli, id) @@ -652,8 +686,11 @@ func TestErrorLogRotation(t *testing.T) { // Generate some content for the error logs, by trying to put messages under an unauthorized user // execContainer(t, cli, id, "fred", []string{"bash", "-c", "for i in {1..30} ; do /opt/mqm/samp/bin/amqsput FAKE; done"}) execContainer(t, cli, id, "root", []string{"useradd", "fred"}) + for { execContainer(t, cli, id, "fred", []string{"bash", "-c", "/opt/mqm/samp/bin/amqsput FAKE"}) + _, atoiStr1 := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "wc -c < " + filepath.Join(dir, "AMQERR01.json")}) + amqerr01size, _ := strconv.Atoi(atoiStr1) _, atoiStr := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "wc -c < " + filepath.Join(dir, "AMQERR02.json")}) amqerr02size, _ := strconv.Atoi(atoiStr) @@ -694,12 +731,14 @@ func TestErrorLogRotation(t *testing.T) { // Tests the log comes out in JSON format when JSON format is enabled. With metrics enabled func TestJSONLogFormatWithMetrics(t *testing.T) { t.Parallel() + jsonLogFormat(t, true) } // Tests the log comes out in JSON format when JSON format is enabled. With metrics disabled func TestJSONLogFormatNoMetrics(t *testing.T) { t.Parallel() + jsonLogFormat(t, false) } @@ -740,6 +779,7 @@ func jsonLogFormat(t *testing.T, metric bool) { func TestBadLogFormat(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index f755f5b..b94545f 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -343,7 +343,7 @@ func runContainerOneShot(t *testing.T, cli *client.Client, command ...string) (i t.Fatal(err) } defer cleanContainerQuiet(t, cli, ctr.ID) - rc := waitForContainer(t, cli, ctr.ID, 10*time.Second) + rc := waitForContainer(t, cli, ctr.ID, 20*time.Second) out := inspectLogs(t, cli, ctr.ID) t.Logf("One shot container finished with rc=%v, output=%v", rc, out) return rc, out @@ -374,7 +374,7 @@ func runContainerOneShotWithVolume(t *testing.T, cli *client.Client, bind string t.Fatal(err) } defer cleanContainerQuiet(t, cli, ctr.ID) - rc := waitForContainer(t, cli, ctr.ID, 10*time.Second) + rc := waitForContainer(t, cli, ctr.ID, 20*time.Second) out := inspectLogs(t, cli, ctr.ID) t.Logf("One shot container finished with rc=%v, output=%v", rc, out) return rc, out diff --git a/test/docker/mqmetric_test.go b/test/docker/mqmetric_test.go index 6f08056..ee24ec3 100644 --- a/test/docker/mqmetric_test.go +++ b/test/docker/mqmetric_test.go @@ -27,6 +27,7 @@ import ( func TestGoldenPathMetric(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -53,6 +54,7 @@ func TestGoldenPathMetric(t *testing.T) { func TestMetricNames(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -96,6 +98,7 @@ func TestMetricNames(t *testing.T) { func TestMetricLabels(t *testing.T) { t.Parallel() + requiredLabels := []string{"qmgr"} cli, err := client.NewEnvClient() if err != nil { @@ -144,6 +147,7 @@ func TestMetricLabels(t *testing.T) { func TestRapidFirePrometheus(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -177,6 +181,7 @@ func TestRapidFirePrometheus(t *testing.T) { func TestSlowPrometheus(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -207,6 +212,7 @@ func TestSlowPrometheus(t *testing.T) { func TestContainerRestart(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -254,6 +260,7 @@ func TestContainerRestart(t *testing.T) { func TestQMRestart(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -311,6 +318,7 @@ func TestQMRestart(t *testing.T) { func TestValidValues(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) @@ -346,6 +354,7 @@ func TestValidValues(t *testing.T) { func TestChangingValues(t *testing.T) { t.Parallel() + cli, err := client.NewEnvClient() if err != nil { t.Fatal(err) From f8e057a1d6f36b0abb7fb271501cb167f12e7df9 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Fri, 8 Mar 2019 11:33:28 +0000 Subject: [PATCH 47/60] Fix test breaks from unused code --- incubating/mqadvanced-server-dev/Dockerfile | 1 - test/docker/docker_api_test.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/incubating/mqadvanced-server-dev/Dockerfile b/incubating/mqadvanced-server-dev/Dockerfile index bf2d93c..fb15480 100644 --- a/incubating/mqadvanced-server-dev/Dockerfile +++ b/incubating/mqadvanced-server-dev/Dockerfile @@ -76,7 +76,6 @@ COPY incubating/mqadvanced-server-dev/*.tpl /etc/mqm/ COPY incubating/mqadvanced-server-dev/web /etc/mqm/web RUN chown -R mqm:mqm /etc/mqm/* \ - && ls -lar /etc/mqm/ \ && chmod +x /usr/local/bin/runmq* \ && install --directory --mode 0775 --owner mqm --group root /run/runmqdevserver diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index 77ca9d4..37759f2 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -688,8 +688,6 @@ func TestErrorLogRotation(t *testing.T) { for { execContainer(t, cli, id, "fred", []string{"bash", "-c", "/opt/mqm/samp/bin/amqsput FAKE"}) - _, atoiStr1 := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "wc -c < " + filepath.Join(dir, "AMQERR01.json")}) - amqerr01size, _ := strconv.Atoi(atoiStr1) _, atoiStr := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "wc -c < " + filepath.Join(dir, "AMQERR02.json")}) amqerr02size, _ := strconv.Atoi(atoiStr) From 1fa4f6f148667f3a3de250f08b026c3946bdd5e1 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Fri, 8 Mar 2019 12:59:54 +0000 Subject: [PATCH 48/60] fix gosec failures --- cmd/runmqdevserver/template.go | 1 + cmd/runmqserver/main.go | 5 ++++- cmd/runmqserver/process.go | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/runmqdevserver/template.go b/cmd/runmqdevserver/template.go index 2600c51..de68393 100644 --- a/cmd/runmqdevserver/template.go +++ b/cmd/runmqdevserver/template.go @@ -36,6 +36,7 @@ func processTemplateFile(templateFile, destFile string, data interface{}) error _, err = os.Stat(dir) if err != nil { if os.IsNotExist(err) { + // #nosec G301 err = os.MkdirAll(dir, 0770) if err != nil { log.Error(err) diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 2b46a9e..2814e27 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -45,7 +45,10 @@ func doMain() error { // Check whether they only want debug info if *infoFlag { logVersionInfo() - logContainerDetails() + err = logContainerDetails() + if err != nil { + log.Printf("Error displaying container details: %v", err) + } return nil } diff --git a/cmd/runmqserver/process.go b/cmd/runmqserver/process.go index 03d6c56..dafd9b6 100644 --- a/cmd/runmqserver/process.go +++ b/cmd/runmqserver/process.go @@ -42,6 +42,7 @@ func verifySingleProcess() error { // Verifies that there is only one instance running of the given program name. func verifyOnlyOne(programName string) (int, error) { + // #nosec G104 out, _, _ := command.Run("ps", "-e", "--format", "cmd") //if this goes wrong then assume we are the only one numOfProg := strings.Count(out, programName) From 0dd5f9c818e7d99f4de96c0e18e0c109e69f39e2 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 1 Oct 2018 10:17:38 +0100 Subject: [PATCH 49/60] Replace master with singularity --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 27d1aec..b53d97a 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ [![Build Status](https://travis-ci.org/ibm-messaging/mq-container.svg?branch=master)](https://travis-ci.org/ibm-messaging/mq-container) -**Note**: The `master` branch may be in an *unstable or even broken state* during development. -To get a stable version, please use the correct [branch](https://github.com/ibm-messaging/mq-container/branches) for your MQ version, instead of the `master` branch. +**Note**: The `singularity` branch may be in an *unstable or even broken state* during development. +To get a stable version, please use the correct [branch](https://github.com/ibm-messaging/mq-container/branches) for your MQ version, instead of the `singularity` branch. IBM MQ logo From 568ae6e34e7aa56ba7cc9b5b74d03fa36119ef69 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Wed, 26 Sep 2018 16:17:15 +0100 Subject: [PATCH 50/60] Enable web console for mqadvanced-server --- Dockerfile-server | 7 ++++-- Makefile-RHEL | 3 +-- Makefile-UBUNTU | 10 ++------ .../{post_init_dev.go => post_init.go} | 2 -- cmd/runmqserver/post_init_other.go | 22 ---------------- cmd/runmqserver/webserver.go | 2 -- install-mq.sh | 4 +-- mq-advanced-server-rhel/mq-buildah.sh | 4 +++ mq-advanced-server-rhel/mqdev-buildah.sh | 7 +----- .../Installation1/servers/mqweb/mqwebuser.xml | 25 +++++++++++++++++++ .../Installation1/servers/mqweb/tls.xml | 0 11 files changed, 40 insertions(+), 46 deletions(-) rename cmd/runmqserver/{post_init_dev.go => post_init.go} (98%) delete mode 100644 cmd/runmqserver/post_init_other.go create mode 100644 web/installations/Installation1/servers/mqweb/mqwebuser.xml rename {incubating/mqadvanced-server-dev/web => web}/installations/Installation1/servers/mqweb/tls.xml (100%) diff --git a/Dockerfile-server b/Dockerfile-server index 5d96adf..6e5f1af 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -74,8 +74,11 @@ RUN chmod ug+x /usr/local/bin/runmqserver \ && chown mqm:root /run/termination-log \ && chmod 0660 /run/termination-log -# Always use port 1414 for MQ & 9157 for the metrics -EXPOSE 1414 9157 +# Always use port 1414 for MQ, 9157 for the metrics & 9443 for the web console +EXPOSE 1414 9157 9443 + +# Copy web XML files +COPY web /etc/mqm/web ENV LANG=en_US.UTF-8 AMQ_DIAGNOSTIC_MSG_SEVERITY=1 AMQ_ADDITIONAL_JSON_LOG=1 LOG_FORMAT=basic diff --git a/Makefile-RHEL b/Makefile-RHEL index 1a13dcc..f1f64de 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -40,7 +40,7 @@ MQ_IMAGE_SDK ?=mq-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) # MQ_IMAGE_GOLANG_SDK is the name and tag of the built MQ Advanced for Developers SDK image, plus Go tools MQ_IMAGE_GOLANG_SDK ?=mq-golang-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) # MQ_PACKAGES specifies the MQ packages to install. Defaults vary on base image. -MQ_PACKAGES ?= MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm +MQ_PACKAGES ?= MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm ############################################################################### # Other variables @@ -166,7 +166,6 @@ build-advancedserver: check-prereqs downloads/$(MQ_ARCHIVE) build-go-programs .PHONY: build-devserver build-devserver: MQDEV=TRUE -build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm build-devserver: check-prereqs downloads/$(MQ_ARCHIVE_DEV) build-go-programs $(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER)"$(END))) sudo mq-advanced-server-rhel/mq-buildah.sh "$(MQ_ARCHIVE_DEV)" "$(MQ_PACKAGES)" "$(MQ_IMAGE_DEVSERVER_BASE)" "$(MQ_VERSION)" "$(MQDEV)" diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index 95f1c6a..c45203c 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -134,7 +134,7 @@ downloads/$(MQ_ARCHIVE_DEV): downloads/$(MQ_SDK_ARCHIVE): $(info $(SPACER)$(shell printf $(TITLE)"Downloading IBM MQ Advanced for Developers "$(MQ_VERSION)$(END))) mkdir -p downloads - cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_SDK_ARCHIVE) + cd downloads; curl -LO https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/$(MQ_SDK_ARCHIVE) .PHONY: downloads downloads: downloads/$(MQ_ARCHIVE_DEV) downloads/$(MQ_SDK_ARCHIVE) @@ -236,12 +236,6 @@ build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version build-golang-sdk-ex $(call docker-build-mq,$(MQ_IMAGE_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) .PHONY: build-devserver -# Target-specific variable to add web server into devserver image -ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu" -build-devserver: MQ_PACKAGES=ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams ibmmq-web -else -build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm -endif build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV) build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk-ex $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER_BASE)"$(END))) @@ -267,7 +261,7 @@ build-sdk: downloads/$(MQ_SDK_ARCHIVE) build-sdk-ex .PHONY: build-sdk-ex ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu" build-sdk-ex: MQ_PACKAGES=ibmmq-sdk ibmmq-samples build-essential -else +else build-sdk-ex: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesSDK-*.rpm MQSeriesSamples*.rpm endif build-sdk-ex: docker-version docker-pull diff --git a/cmd/runmqserver/post_init_dev.go b/cmd/runmqserver/post_init.go similarity index 98% rename from cmd/runmqserver/post_init_dev.go rename to cmd/runmqserver/post_init.go index 861c566..59d6157 100644 --- a/cmd/runmqserver/post_init_dev.go +++ b/cmd/runmqserver/post_init.go @@ -1,5 +1,3 @@ -// +build mqdev - /* © Copyright IBM Corporation 2018 diff --git a/cmd/runmqserver/post_init_other.go b/cmd/runmqserver/post_init_other.go deleted file mode 100644 index 71f3458..0000000 --- a/cmd/runmqserver/post_init_other.go +++ /dev/null @@ -1,22 +0,0 @@ -// +build !mqdev - -/* -© Copyright IBM Corporation 2018 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package main - -func postInit(name string) error { - return nil -} diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index dae2bd6..c42849a 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -1,5 +1,3 @@ -// +build mqdev - /* © Copyright IBM Corporation 2018, 2019 diff --git a/install-mq.sh b/install-mq.sh index c4b2926..fe83ced 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -25,8 +25,8 @@ test -f /usr/bin/apt-get && UBUNTU=true || UBUNTU=false # If MQ_PACKAGES isn't specifically set, then choose a valid set of defaults if [ -z "$MQ_PACKAGES" ]; then - $UBUNTU && MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams" - $RHEL && MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm" + $UBUNTU && MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams ibmmq-web" + $RHEL && MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm" fi if ($UBUNTU); then diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index c96bd62..93f09f4 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -121,6 +121,9 @@ buildah run --user root $ctr_mq -- chmod 0660 /run/termination-log install --mode 0550 --owner root --group root ./mq-advanced-server-rhel/writePackages.sh ${mnt_mq}/usr/local/bin/writePackages buildah run --user root $ctr_mq -- /usr/local/bin/writePackages +# Copy web XML files +cp -R web ${mnt_mq}/etc/mqm/web + ############################################################################### # Final Buildah commands ############################################################################### @@ -138,6 +141,7 @@ fi buildah config \ --port 1414/tcp \ --port 9157/tcp \ + --port 9443/tcp \ --os linux \ --label architecture=x86_64 \ --label io.openshift.tags="$OSTAG" \ diff --git a/mq-advanced-server-rhel/mqdev-buildah.sh b/mq-advanced-server-rhel/mqdev-buildah.sh index 1e7c84a..5778ec9 100755 --- a/mq-advanced-server-rhel/mqdev-buildah.sh +++ b/mq-advanced-server-rhel/mqdev-buildah.sh @@ -78,12 +78,7 @@ install --directory --mode 0775 --owner ${mqm_uid} --group 0 ${mnt_mq}/run/runmq cp ./incubating/mqadvanced-server-dev/*.tpl ${mnt_mq}/etc/mqm/ # Copy web XML files for default developer configuration -mkdir --parents ${mnt_mq}/etc/mqm/web -cp --recursive ./incubating/mqadvanced-server-dev/web/* ${mnt_mq}/etc/mqm/web/ - -# Make "mqm" the owner of all the config files -chown --recursive ${mqm_uid}:${mqm_gid} ${mnt_mq}/etc/mqm/* -chmod --recursive 0750 ${mnt_mq}/etc/mqm/* +cp -R incubating/mqadvanced-server-dev/web/ ${mnt_mq}/etc/mqm/web ############################################################################### # Final Buildah commands diff --git a/web/installations/Installation1/servers/mqweb/mqwebuser.xml b/web/installations/Installation1/servers/mqweb/mqwebuser.xml new file mode 100644 index 0000000..7bb6ae1 --- /dev/null +++ b/web/installations/Installation1/servers/mqweb/mqwebuser.xml @@ -0,0 +1,25 @@ + + + + appSecurity-2.0 + + + + + + + + + + + + + + + + + + + + + diff --git a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls.xml b/web/installations/Installation1/servers/mqweb/tls.xml similarity index 100% rename from incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls.xml rename to web/installations/Installation1/servers/mqweb/tls.xml From 9b3b1f7b9ee41307abf8621cfc069202d61d0d76 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Wed, 3 Oct 2018 11:06:17 +0100 Subject: [PATCH 51/60] Move template and keystore functions to internal packages --- cmd/runmqdevserver/main.go | 3 ++- cmd/runmqdevserver/mqsc.go | 4 +++- cmd/runmqdevserver/tls.go | 22 ++++++++++--------- .../keystore}/keystore.go | 9 +++++--- .../mqtemplate/mqtemplate.go | 10 +++++---- 5 files changed, 29 insertions(+), 19 deletions(-) rename {cmd/runmqdevserver => internal/keystore}/keystore.go (95%) rename cmd/runmqdevserver/template.go => internal/mqtemplate/mqtemplate.go (84%) diff --git a/cmd/runmqdevserver/main.go b/cmd/runmqdevserver/main.go index 0cfc366..b6c4247 100644 --- a/cmd/runmqdevserver/main.go +++ b/cmd/runmqdevserver/main.go @@ -24,6 +24,7 @@ import ( "github.com/ibm-messaging/mq-container/internal/command" "github.com/ibm-messaging/mq-container/internal/logger" + "github.com/ibm-messaging/mq-container/internal/mqtemplate" "github.com/ibm-messaging/mq-container/internal/name" ) @@ -90,7 +91,7 @@ func configureLogger() error { func configureWeb(qmName string) error { out := "/etc/mqm/web/installations/Installation1/angular.persistence/admin.json" - return processTemplateFile("/etc/mqm/admin.json.tpl", out, map[string]string{"QueueManagerName": qmName}) + return mqtemplate.ProcessTemplateFile("/etc/mqm/admin.json.tpl", out, map[string]string{"QueueManagerName": qmName}, log) } func logTerminationf(format string, args ...interface{}) { diff --git a/cmd/runmqdevserver/mqsc.go b/cmd/runmqdevserver/mqsc.go index 93d58ea..ef9cce2 100644 --- a/cmd/runmqdevserver/mqsc.go +++ b/cmd/runmqdevserver/mqsc.go @@ -17,6 +17,8 @@ package main import ( "os" + + "github.com/ibm-messaging/mq-container/internal/mqtemplate" ) func updateMQSC(appPasswordRequired bool) error { @@ -30,7 +32,7 @@ func updateMQSC(appPasswordRequired bool) error { if os.Getenv("MQ_DEV") == "true" { const mqscTemplate string = mqsc + ".tpl" // Re-configure channel if app password not set - err := processTemplateFile(mqsc+".tpl", mqsc, map[string]string{"ChckClnt": checkClient}) + err := mqtemplate.ProcessTemplateFile(mqsc+".tpl", mqsc, map[string]string{"ChckClnt": checkClient}, log) if err != nil { return err } diff --git a/cmd/runmqdevserver/tls.go b/cmd/runmqdevserver/tls.go index 8e21ed6..bbfc3af 100644 --- a/cmd/runmqdevserver/tls.go +++ b/cmd/runmqdevserver/tls.go @@ -21,20 +21,22 @@ import ( "path/filepath" "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/keystore" + "github.com/ibm-messaging/mq-container/internal/mqtemplate" ) -func configureWebTLS(cms *KeyStore) error { +func configureWebTLS(cms *keystore.KeyStore) error { dir := "/run/runmqdevserver/tls" - ks := NewJKSKeyStore(filepath.Join(dir, "key.jks"), cms.Password) - ts := NewJKSKeyStore(filepath.Join(dir, "trust.jks"), cms.Password) + ks := keystore.NewJKSKeyStore(filepath.Join(dir, "key.jks"), cms.Password) + ts := keystore.NewJKSKeyStore(filepath.Join(dir, "trust.jks"), cms.Password) log.Debug("Creating key store") - err := ks.Create() + err := ks.Create(log) if err != nil { return err } log.Debug("Creating trust store") - err = ts.Create() + err = ts.Create(log) if err != nil { return err } @@ -105,14 +107,14 @@ func configureTLS(qmName string, inputFile string, passPhrase string) error { } } - cms := NewCMSKeyStore(keyFile, passPhrase) + cms := keystore.NewCMSKeyStore(keyFile, passPhrase) - err = cms.Create() + err = cms.Create(log) if err != nil { return err } - err = cms.CreateStash() + err = cms.CreateStash(log) if err != nil { return err } @@ -146,11 +148,11 @@ func configureTLS(qmName string, inputFile string, passPhrase string) error { const mqsc string = "/etc/mqm/20-dev-tls.mqsc" const mqscTemplate string = mqsc + ".tpl" - err = processTemplateFile(mqscTemplate, mqsc, map[string]string{ + err = mqtemplate.ProcessTemplateFile(mqscTemplate, mqsc, map[string]string{ "SSLKeyR": filepath.Join(dir, "key"), "CertificateLabel": newLabel, "SSLCipherSpec": sslCipherSpec, - }) + }, log) if err != nil { return err } diff --git a/cmd/runmqdevserver/keystore.go b/internal/keystore/keystore.go similarity index 95% rename from cmd/runmqdevserver/keystore.go rename to internal/keystore/keystore.go index 5027ead..d8d97ea 100644 --- a/cmd/runmqdevserver/keystore.go +++ b/internal/keystore/keystore.go @@ -13,7 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package main + +// Package keystore contains code to create and update keystores +package keystore import ( "bufio" @@ -23,6 +25,7 @@ import ( "strings" "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/logger" ) // KeyStore describes information about a keystore file @@ -54,7 +57,7 @@ func NewCMSKeyStore(filename, password string) *KeyStore { } // Create a key store, if it doesn't already exist -func (ks *KeyStore) Create() error { +func (ks *KeyStore) Create(log *logger.Logger) error { _, err := os.Stat(ks.Filename) if err == nil { // Keystore already exists so we should refresh it by deleting it. @@ -111,7 +114,7 @@ func (ks *KeyStore) Create() error { } // CreateStash creates a key stash, if it doesn't already exist -func (ks *KeyStore) CreateStash() error { +func (ks *KeyStore) CreateStash(log *logger.Logger) error { extension := filepath.Ext(ks.Filename) stashFile := ks.Filename[0:len(ks.Filename)-len(extension)] + ".sth" log.Debugf("TLS stash file: %v", stashFile) diff --git a/cmd/runmqdevserver/template.go b/internal/mqtemplate/mqtemplate.go similarity index 84% rename from cmd/runmqdevserver/template.go rename to internal/mqtemplate/mqtemplate.go index de68393..b26ed65 100644 --- a/cmd/runmqdevserver/template.go +++ b/internal/mqtemplate/mqtemplate.go @@ -13,7 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package main + +// Package mqtemplate contains code to process template files +package mqtemplate import ( "os" @@ -21,12 +23,12 @@ import ( "text/template" "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/logger" ) -// processTemplateFile takes a Go templateFile, and processes it with the +// ProcessTemplateFile takes a Go templateFile, and processes it with the // supplied data, writing to destFile -func processTemplateFile(templateFile, destFile string, data interface{}) error { - // Re-configure channel if app password not set +func ProcessTemplateFile(templateFile, destFile string, data interface{}, log *logger.Logger) error { t, err := template.ParseFiles(templateFile) if err != nil { log.Error(err) From 4c1d124484849113af2faa5983b943d5917238d1 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Wed, 3 Oct 2018 13:19:09 +0100 Subject: [PATCH 52/60] Configure Single-Sign-On for the web server --- cmd/runmqserver/post_init.go | 14 ++- cmd/runmqserver/webserver.go | 98 +++++++++++++++++++ internal/keystore/keystore.go | 18 ++++ .../servers/mqweb/mqwebuser.xml.tpl | 44 +++++++++ 4 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl diff --git a/cmd/runmqserver/post_init.go b/cmd/runmqserver/post_init.go index 59d6157..e2bae03 100644 --- a/cmd/runmqserver/post_init.go +++ b/cmd/runmqserver/post_init.go @@ -20,18 +20,26 @@ import ( ) // postInit is run after /var/mqm is set up -// This version of postInit is only included as part of the MQ Advanced for Developers build func postInit(name string) error { disable := os.Getenv("MQ_DISABLE_WEB_CONSOLE") if disable != "true" && disable != "1" { + + // Configure Single-Sign-On for the web server (if enabled) + enableSSO := os.Getenv("MQ_ENABLE_SSO") + if enableSSO == "true" || enableSSO == "1" { + err := configureSSO() + if err != nil { + return err + } + } + // Configure the web server (if installed) err := configureWebServer() if err != nil { return err } // Start the web server, in the background (if installed) - // WARNING: No error handling or health checking available for the web server, - // which is why it's limited to use with MQ Advanced for Developers only + // WARNING: No error handling or health checking available for the web server go func() { startWebServer() }() diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index c42849a..fab67fb 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -23,9 +23,12 @@ import ( "os/user" "path/filepath" "strconv" + "strings" "syscall" "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/keystore" + "github.com/ibm-messaging/mq-container/internal/mqtemplate" ) func startWebServer() error { @@ -88,6 +91,101 @@ func CopyFile(src, dest string) error { return err } +func configureSSO() error { + + // Ensure all required environment variables are set for SSO + requiredEnvVars := []string{ + "MQ_WEB_ADMIN_USERS", + "MQ_OIDC_CLIENT_ID", + "MQ_OIDC_CLIENT_SECRET", + "MQ_OIDC_AUTHORIZATION_ENDPOINT", + "MQ_OIDC_TOKEN_ENDPOINT", + "MQ_OIDC_JWK_ENDPOINT", + "MQ_OIDC_ISSUER_IDENTIFIER", + "MQ_OIDC_CERTIFICATE", + } + for _, envVar := range requiredEnvVars { + if len(os.Getenv(envVar)) == 0 { + return fmt.Errorf("%v must be set when MQ_ENABLE_SSO=true", envVar) + } + } + + // Check mqweb directory exists + const mqwebDir string = "/etc/mqm/web/installations/Installation1/servers/mqweb" + _, err := os.Stat(mqwebDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + // Process SSO template for generating file mqwebuser.xml + adminUsers := strings.Split(os.Getenv("MQ_WEB_ADMIN_USERS"), ",") + err = mqtemplate.ProcessTemplateFile(mqwebDir+"/mqwebuser.xml.tpl", mqwebDir+"/mqwebuser.xml", map[string][]string{"AdminUser": adminUsers}, log) + if err != nil { + return err + } + + // Configure SSO TLS + return configureSSO_TLS() +} + +func configureSSO_TLS() error { + + // Create tls directory + dir := "/run/tls" + _, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(dir, 0770) + if err != nil { + return err + } + mqmUID, mqmGID, err := command.LookupMQM() + if err != nil { + log.Error(err) + return err + } + err = os.Chown(dir, mqmUID, mqmGID) + if err != nil { + log.Error(err) + return err + } + } else { + return err + } + } + + // Setup key store & trust store + ks := keystore.NewJKSKeyStore(filepath.Join(dir, "key.jks"), "password") + ts := keystore.NewJKSKeyStore(filepath.Join(dir, "trust.jks"), "password") + + log.Debug("Creating key store") + err = ks.Create(log) + if err != nil { + return err + } + log.Debug("Creating trust store") + err = ts.Create(log) + if err != nil { + return err + } + log.Debug("Creating self-signed certificate in key store") + err = ks.CreateSelfSignedCertificate("default", "CN=IBMMQWeb,O=IBM,OU=Platform,C=GB") + if err != nil { + return err + } + log.Debug("Importing self-signed certificate into trust store") + err = ts.Import(ks.Filename, ks.Password) + if err != nil { + return err + } + log.Debug("Adding OIDC CA certificate to trust store") + err = ts.Add(os.Getenv("MQ_OIDC_CERTIFICATE"), "OIDC") + return err +} + func configureWebServer() error { _, err := os.Stat("/opt/mqm/bin/strmqweb") if err != nil { diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index d8d97ea..44bf596 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -150,6 +150,24 @@ func (ks *KeyStore) Import(inputFile, password string) error { return nil } +// CreateSelfSignedCertificate creates a self-signed certificate in the keystore +func (ks *KeyStore) CreateSelfSignedCertificate(label, dn string) error { + out, _, err := command.Run(ks.command, "-cert", "-create", "-db", ks.Filename, "-pw", ks.Password, "-label", label, "-dn", dn) + if err != nil { + return fmt.Errorf("error running \"%v -cert -create\": %v %s", ks.command, err, out) + } + return nil +} + +// Add adds a CA certificate to the keystore +func (ks *KeyStore) Add(inputFile, label string) error { + out, _, err := command.Run(ks.command, "-cert", "-add", "-db", ks.Filename, "-type", ks.keyStoreType, "-pw", ks.Password, "-file", inputFile, "-label", label) + if err != nil { + return fmt.Errorf("error running \"%v -cert -add\": %v %s", ks.command, err, out) + } + return nil +} + // GetCertificateLabels returns the labels of all certificates in the key store func (ks *KeyStore) GetCertificateLabels() ([]string, error) { out, _, err := command.Run(ks.command, "-cert", "-list", "-type", ks.keyStoreType, "-db", ks.Filename, "-pw", ks.Password) diff --git a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl new file mode 100644 index 0000000..65a4b60 --- /dev/null +++ b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl @@ -0,0 +1,44 @@ + + + + openidConnectClient-1.0 + ssl-1.0 + + + + + + {{- range $index, $element := .AdminUser}} + + {{- end}} + + + + + + + + + + + + + + + + + + + + + + From c455d696b28dc0dbb55ff259163b4b670317b89a Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Fri, 12 Oct 2018 14:13:20 +0100 Subject: [PATCH 53/60] Split SSO admin user list on newlines (#229) --- cmd/runmqserver/webserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index fab67fb..a58b9fc 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -121,7 +121,7 @@ func configureSSO() error { } // Process SSO template for generating file mqwebuser.xml - adminUsers := strings.Split(os.Getenv("MQ_WEB_ADMIN_USERS"), ",") + adminUsers := strings.Split(os.Getenv("MQ_WEB_ADMIN_USERS"), "\n") err = mqtemplate.ProcessTemplateFile(mqwebDir+"/mqwebuser.xml.tpl", mqwebDir+"/mqwebuser.xml", map[string][]string{"AdminUser": adminUsers}, log) if err != nil { return err From 6f1268ffecc50819ba1fdeec0530bf5516ccf72a Mon Sep 17 00:00:00 2001 From: Rob Parker Date: Thu, 18 Oct 2018 13:43:48 +0100 Subject: [PATCH 54/60] add integration into the docker tag (#233) --- Makefile-RHEL | 4 ++-- Makefile-UBUNTU | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile-RHEL b/Makefile-RHEL index f1f64de..3f22004 100644 --- a/Makefile-RHEL +++ b/Makefile-RHEL @@ -32,9 +32,9 @@ MQ_SDK_ARCHIVE ?= 9.1.1.0-IBM-MQC-Redist-LinuxX64.tar.gz # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image -MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-RHEL-$(ARCH) +MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-integration-$(ARCH) # MQ_IMAGE_DEVSERVER is the name and tag of the built MQ Advanced for Developers image -MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-RHEL-$(ARCH) +MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-integration-$(ARCH) # MQ_IMAGE_SDK is the name and tag of the built MQ Advanced for Developers SDK image MQ_IMAGE_SDK ?=mq-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) # MQ_IMAGE_GOLANG_SDK is the name and tag of the built MQ Advanced for Developers SDK image, plus Go tools diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index c45203c..a90b8e7 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -32,9 +32,9 @@ MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # MQ_IMAGE_ADVANCEDSERVER is the name and tag of the built MQ Advanced image -MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +MQ_IMAGE_ADVANCEDSERVER ?=mqadvanced-server:$(MQ_VERSION)-integration-$(ARCH) # MQ_IMAGE_DEVSERVER is the name and tag of the built MQ Advanced for Developers image -MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) +MQ_IMAGE_DEVSERVER ?=mqadvanced-server-dev:$(MQ_VERSION)-integration-$(ARCH) # MQ_IMAGE_SDK is the name and tag of the built MQ Advanced for Developers SDK image MQ_IMAGE_SDK ?=mq-sdk:$(MQ_VERSION)-$(ARCH)-$(BASE_IMAGE_TAG) # MQ_IMAGE_GOLANG_SDK is the name and tag of the built MQ Advanced for Developers SDK image, plus Go tools From 227db5875a24e51c0c7c5f90d1dcd2c4428e5623 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Thu, 25 Oct 2018 10:17:02 +0100 Subject: [PATCH 55/60] Fix for iframe issue with web console (#238) --- web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl index 65a4b60..3bb4b8c 100644 --- a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl +++ b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl @@ -36,6 +36,8 @@ issuerIdentifier="${env.MQ_OIDC_ISSUER_IDENTIFIER}"> + + From fd262b173ec43808dbf19659f9ae445e36569ce9 Mon Sep 17 00:00:00 2001 From: Stephen Marshall Date: Fri, 26 Oct 2018 14:47:49 +0100 Subject: [PATCH 56/60] Add uniqueUserIdentifier --- cmd/runmqserver/webserver.go | 1 + web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index a58b9fc..e472970 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -98,6 +98,7 @@ func configureSSO() error { "MQ_WEB_ADMIN_USERS", "MQ_OIDC_CLIENT_ID", "MQ_OIDC_CLIENT_SECRET", + "MQ_OIDC_UNIQUE_USER_IDENTIFIER", "MQ_OIDC_AUTHORIZATION_ENDPOINT", "MQ_OIDC_TOKEN_ENDPOINT", "MQ_OIDC_JWK_ENDPOINT", diff --git a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl index 3bb4b8c..5cebb71 100644 --- a/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl +++ b/web/installations/Installation1/servers/mqweb/mqwebuser.xml.tpl @@ -27,6 +27,7 @@ Date: Wed, 7 Nov 2018 11:47:41 +0000 Subject: [PATCH 57/60] Add TLS support (#243) * Add TLS support * Security fix for libsystemd0 systemd systemd-sysv libudev1 --- cmd/runmqserver/webserver.go | 10 +++++----- install-mq.sh | 7 +++++-- internal/keystore/keystore.go | 9 +++++++++ mq-advanced-server-rhel/mq-buildah.sh | 1 + 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index e472970..7897bf5 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -172,17 +172,17 @@ func configureSSO_TLS() error { if err != nil { return err } - log.Debug("Creating self-signed certificate in key store") - err = ks.CreateSelfSignedCertificate("default", "CN=IBMMQWeb,O=IBM,OU=Platform,C=GB") + log.Debug("Generating PKCS12 file") + err = ks.GeneratePKCS12("/mnt/tls/tls.key", "/mnt/tls/tls.crt", "/run/tls/tls.p12", "default", "password") if err != nil { return err } - log.Debug("Importing self-signed certificate into trust store") - err = ts.Import(ks.Filename, ks.Password) + log.Debug("Importing certificate into key store") + err = ks.Import("/run/tls/tls.p12", "password") if err != nil { return err } - log.Debug("Adding OIDC CA certificate to trust store") + log.Debug("Adding OIDC certificate to trust store") err = ts.Add(os.Getenv("MQ_OIDC_CERTIFICATE"), "OIDC") return err } diff --git a/install-mq.sh b/install-mq.sh index fe83ced..c59649d 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -65,7 +65,8 @@ if ($UBUNTU); then procps \ sed \ tar \ - util-linux + util-linux \ + openssl fi # Install additional packages required by MQ, this install process and the runtime scripts @@ -84,7 +85,8 @@ $RHEL && yum -y install \ procps-ng \ sed \ tar \ - util-linux + util-linux \ + openssl # Download and extract the MQ installation files DIR_EXTRACT=/tmp/mq @@ -139,6 +141,7 @@ rm -rf ${DIR_EXTRACT} # Apply any bug fixes not included in base Ubuntu or MQ image. # Don't upgrade everything based on Docker best practices https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run $UBUNTU && apt-get install -y libapparmor1 libsystemd0 systemd systemd-sysv libudev1 perl-base --only-upgrade + # End of bug fixes # Clean up cached files diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index 44bf596..62a368c 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -141,6 +141,15 @@ func (ks *KeyStore) CreateStash(log *logger.Logger) error { return nil } +// GeneratePKCS12 generates a PKCS12 file +func (ks *KeyStore) GeneratePKCS12(keyFile, crtFile, pkcs12File, label, password string) error { + out, _, err := command.Run("openssl", "pkcs12", "-export", "-inkey", keyFile, "-in", crtFile, "-out", pkcs12File, "-name", label, "-passout", "pass:"+password) + if err != nil { + return fmt.Errorf("error running \"openssl pkcs12 -export\": %v %s", err, out) + } + return nil +} + // Import imports a certificate file in the keystore func (ks *KeyStore) Import(inputFile, password string) error { out, _, err := command.Run(ks.command, "-cert", "-import", "-file", inputFile, "-pw", password, "-target", ks.Filename, "-target_pw", ks.Password, "-target_type", ks.keyStoreType) diff --git a/mq-advanced-server-rhel/mq-buildah.sh b/mq-advanced-server-rhel/mq-buildah.sh index 93f09f4..73fb93f 100755 --- a/mq-advanced-server-rhel/mq-buildah.sh +++ b/mq-advanced-server-rhel/mq-buildah.sh @@ -86,6 +86,7 @@ buildah run ${ctr_mq} -- microdnf ${microdnf_opts} install \ shadow-utils \ tar \ util-linux \ + openssl \ which # Install "sudo" if using MQ Advanced for Developers From c3f40c84a7e1f2eca11b7d23585d614a9214f9af Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 19 Mar 2019 11:29:33 +0000 Subject: [PATCH 58/60] Extra changes to support non-root in CIP --- Dockerfile-server | 1 + Makefile-UBUNTU | 2 - cmd/runmqdevserver/tls.go | 34 ---------------- cmd/runmqserver/crtmqvol.go | 66 ++++++++++++++++++++++++++++++- cmd/runmqserver/logging.go | 3 ++ cmd/runmqserver/main.go | 14 +++++++ cmd/runmqserver/qmgr.go | 1 + cmd/runmqserver/webserver.go | 36 ++--------------- internal/keystore/keystore.go | 21 ---------- internal/mqtemplate/mqtemplate.go | 21 ---------- 10 files changed, 88 insertions(+), 111 deletions(-) diff --git a/Dockerfile-server b/Dockerfile-server index 6e5f1af..7086cce 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -70,6 +70,7 @@ RUN chmod ug+x /usr/local/bin/runmqserver \ && chown mqm:mqm /usr/local/bin/*mq* \ && chmod ug+xs /usr/local/bin/chkmq* \ && install --directory --mode 0775 --owner mqm --group root /run/runmqserver \ + && install --directory --mode 0775 --owner mqm --group root /run/tls \ && touch /run/termination-log \ && chown mqm:root /run/termination-log \ && chmod 0660 /run/termination-log diff --git a/Makefile-UBUNTU b/Makefile-UBUNTU index a90b8e7..82b6898 100644 --- a/Makefile-UBUNTU +++ b/Makefile-UBUNTU @@ -274,9 +274,7 @@ build-golang-sdk: downloads/$(MQ_SDK_ARCHIVE) build-golang-sdk-ex .PHONY: build-golang-sdk-ex build-golang-sdk-ex: docker-version build-sdk-ex $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_GOLANG_SDK)"$(END))) - @echo hello $(DOCKER) build --build-arg BASE_IMAGE=$(MQ_IMAGE_SDK) -t $(MQ_IMAGE_GOLANG_SDK) -f incubating/mq-golang-sdk/Dockerfile . - @echo goodbye .PHONY: docker-pull docker-pull: diff --git a/cmd/runmqdevserver/tls.go b/cmd/runmqdevserver/tls.go index bbfc3af..759ef0f 100644 --- a/cmd/runmqdevserver/tls.go +++ b/cmd/runmqdevserver/tls.go @@ -20,7 +20,6 @@ import ( "os" "path/filepath" - "github.com/ibm-messaging/mq-container/internal/command" "github.com/ibm-messaging/mq-container/internal/keystore" "github.com/ibm-messaging/mq-container/internal/mqtemplate" ) @@ -58,16 +57,6 @@ func configureWebTLS(cms *keystore.KeyStore) error { if err != nil { return err } - mqmUID, mqmGID, err := command.LookupMQM() - if err != nil { - log.Error(err) - return err - } - err = os.Chown(tlsConfig, mqmUID, mqmGID) - if err != nil { - log.Error(err) - return err - } return nil } @@ -84,29 +73,6 @@ func configureTLS(qmName string, inputFile string, passPhrase string) error { dir := "/run/runmqdevserver/tls" keyFile := filepath.Join(dir, "key.kdb") - _, err = os.Stat(dir) - if err != nil { - if os.IsNotExist(err) { - // #nosec G301 - err = os.MkdirAll(dir, 0770) - if err != nil { - return err - } - mqmUID, mqmGID, err := command.LookupMQM() - if err != nil { - log.Error(err) - return err - } - err = os.Chown(dir, mqmUID, mqmGID) - if err != nil { - log.Error(err) - return err - } - } else { - return err - } - } - cms := keystore.NewCMSKeyStore(keyFile, passPhrase) err = cms.Create(log) diff --git a/cmd/runmqserver/crtmqvol.go b/cmd/runmqserver/crtmqvol.go index cadd833..0c44869 100644 --- a/cmd/runmqserver/crtmqvol.go +++ b/cmd/runmqserver/crtmqvol.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -60,3 +60,67 @@ func createVolume(path string) error { } return nil } + +func createWebConsoleTLSDirStructure() error { + // Create tls directory + dir := "/run/tls" + _, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(dir, 0770) + if err != nil { + return err + } + mqmUID, mqmGID, err := command.LookupMQM() + if err != nil { + log.Error(err) + return err + } + err = os.Chown(dir, mqmUID, mqmGID) + if err != nil { + log.Error(err) + return err + } + } else { + return err + } + } + + return nil +} + +func createDevTLSDir() error { + // TODO: Use a persisted file (on the volume) instead? + par := "/run/runmqdevserver" + dir := filepath.Join(par, "tls") + + _, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + // #nosec G301 + err = os.MkdirAll(dir, 0770) + if err != nil { + return err + } + mqmUID, mqmGID, err := command.LookupMQM() + if err != nil { + log.Error(err) + return err + } + err = os.Chown(dir, mqmUID, mqmGID) + if err != nil { + log.Error(err) + return err + } + err = os.Chown(par, mqmUID, mqmGID) + if err != nil { + log.Error(err) + return err + } + + } else { + return err + } + } + return nil +} diff --git a/cmd/runmqserver/logging.go b/cmd/runmqserver/logging.go index f2f30a8..12ec6ca 100644 --- a/cmd/runmqserver/logging.go +++ b/cmd/runmqserver/logging.go @@ -138,6 +138,9 @@ func logDiagnostics() { out, _, _ = command.Run("ls", "-l", "/mnt/mqm/data") log.Debugf("/mnt/mqm/data:\n%s", out) // #nosec G104 + out, _, _ = command.Run("ls", "-l", "/etc/mqm") + log.Debugf("/etc/mqm:\n%s", out) + // #nosec G104 out, _, _ = command.Run("ls", "-l", "/var/mqm") log.Debugf("/var/mqm:\n%s", out) // #nosec G104 diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 2814e27..acee110 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -104,6 +104,20 @@ func doMain() error { return err } + err = createWebConsoleTLSDirStructure() + if err != nil { + logTermination(err) + return err + } + + if *devFlag == true { + err = createDevTLSDir() + if err != nil { + logTermination(err) + return err + } + } + // If init flag is set, exit now if *initFlag { return nil diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index 49b73c1..5515cfe 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -34,6 +34,7 @@ func createDirStructure() error { return err } log.Println("Created directory structure under /var/mqm") + return nil } diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go index 7897bf5..7537864 100644 --- a/cmd/runmqserver/webserver.go +++ b/cmd/runmqserver/webserver.go @@ -136,34 +136,14 @@ func configureSSO_TLS() error { // Create tls directory dir := "/run/tls" - _, err := os.Stat(dir) - if err != nil { - if os.IsNotExist(err) { - err = os.MkdirAll(dir, 0770) - if err != nil { - return err - } - mqmUID, mqmGID, err := command.LookupMQM() - if err != nil { - log.Error(err) - return err - } - err = os.Chown(dir, mqmUID, mqmGID) - if err != nil { - log.Error(err) - return err - } - } else { - return err - } - } + mntdir := "/mnt/tls/" // Setup key store & trust store ks := keystore.NewJKSKeyStore(filepath.Join(dir, "key.jks"), "password") ts := keystore.NewJKSKeyStore(filepath.Join(dir, "trust.jks"), "password") log.Debug("Creating key store") - err = ks.Create(log) + err := ks.Create(log) if err != nil { return err } @@ -173,12 +153,12 @@ func configureSSO_TLS() error { return err } log.Debug("Generating PKCS12 file") - err = ks.GeneratePKCS12("/mnt/tls/tls.key", "/mnt/tls/tls.crt", "/run/tls/tls.p12", "default", "password") + err = ks.GeneratePKCS12(filepath.Join(mntdir, "tls.key"), filepath.Join(mntdir, "tls.crt"), filepath.Join(dir, "tls.p12"), "default", "password") if err != nil { return err } log.Debug("Importing certificate into key store") - err = ks.Import("/run/tls/tls.p12", "password") + err = ks.Import(filepath.Join(dir, "tls.p12"), "password") if err != nil { return err } @@ -203,10 +183,6 @@ func configureWebServer() error { } return err } - uid, gid, err := command.LookupMQM() - if err != nil { - return err - } const prefix string = "/etc/mqm/web" err = filepath.Walk(prefix, func(from string, info os.FileInfo, err error) error { if err != nil { @@ -242,10 +218,6 @@ func configureWebServer() error { return err } } - err = os.Chown(to, uid, gid) - if err != nil { - return err - } return nil }) return err diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index 62a368c..6743779 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -99,17 +99,6 @@ func (ks *KeyStore) Create(log *logger.Logger) error { if err != nil { return fmt.Errorf("error running \"%v -keydb -create\": %v %s", ks.command, err, out) } - - mqmUID, mqmGID, err := command.LookupMQM() - if err != nil { - log.Error(err) - return err - } - err = os.Chown(ks.Filename, mqmUID, mqmGID) - if err != nil { - log.Error(err) - return err - } return nil } @@ -128,16 +117,6 @@ func (ks *KeyStore) CreateStash(log *logger.Logger) error { } return err } - mqmUID, mqmGID, err := command.LookupMQM() - if err != nil { - log.Error(err) - return err - } - err = os.Chown(stashFile, mqmUID, mqmGID) - if err != nil { - log.Error(err) - return err - } return nil } diff --git a/internal/mqtemplate/mqtemplate.go b/internal/mqtemplate/mqtemplate.go index b26ed65..e035b2c 100644 --- a/internal/mqtemplate/mqtemplate.go +++ b/internal/mqtemplate/mqtemplate.go @@ -22,7 +22,6 @@ import ( "path" "text/template" - "github.com/ibm-messaging/mq-container/internal/command" "github.com/ibm-messaging/mq-container/internal/logger" ) @@ -44,16 +43,6 @@ func ProcessTemplateFile(templateFile, destFile string, data interface{}, log *l log.Error(err) return err } - mqmUID, mqmGID, err := command.LookupMQM() - if err != nil { - log.Error(err) - return err - } - err = os.Chown(dir, mqmUID, mqmGID) - if err != nil { - log.Error(err) - return err - } } else { return err } @@ -66,15 +55,5 @@ func ProcessTemplateFile(templateFile, destFile string, data interface{}, log *l log.Error(err) return err } - mqmUID, mqmGID, err := command.LookupMQM() - if err != nil { - log.Error(err) - return err - } - err = os.Chown(destFile, mqmUID, mqmGID) - if err != nil { - log.Error(err) - return err - } return nil } From 7b5e34e59ef0362678709c3d7dde0345ee7f44b9 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 19 Mar 2019 13:45:40 +0000 Subject: [PATCH 59/60] update copyright dates --- Makefile | 2 +- cmd/chkmqhealthy/main.go | 2 +- cmd/chkmqready/main.go | 2 +- cmd/runmqdevserver/mqsc.go | 2 +- cmd/runmqdevserver/tls.go | 2 +- cmd/runmqserver/license.go | 2 +- cmd/runmqserver/mirror.go | 2 +- cmd/runmqserver/process.go | 2 +- cmd/runmqserver/signals.go | 2 +- cmd/runmqserver/version.go | 2 +- .../Installation1/servers/mqweb/mqwebuser.xml | 25 +++++++++++++++++++ .../Installation1/servers/mqweb/tls-dev.xml | 25 +++++++++++++++++++ internal/command/command.go | 2 +- internal/keystore/keystore.go | 2 +- internal/metrics/metrics.go | 2 +- internal/metrics/update.go | 2 +- test/docker/Gopkg.toml | 2 +- test/messaging/pom.xml | 2 +- 18 files changed, 66 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 5afa075..7aeb76a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2018 +# © Copyright IBM Corporation 2018, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/cmd/chkmqhealthy/main.go b/cmd/chkmqhealthy/main.go index 1420805..02dbbfa 100644 --- a/cmd/chkmqhealthy/main.go +++ b/cmd/chkmqhealthy/main.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/chkmqready/main.go b/cmd/chkmqready/main.go index 3f978d1..e75f1c5 100644 --- a/cmd/chkmqready/main.go +++ b/cmd/chkmqready/main.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/runmqdevserver/mqsc.go b/cmd/runmqdevserver/mqsc.go index ef9cce2..f35512d 100644 --- a/cmd/runmqdevserver/mqsc.go +++ b/cmd/runmqdevserver/mqsc.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/runmqdevserver/tls.go b/cmd/runmqdevserver/tls.go index 759ef0f..e664409 100644 --- a/cmd/runmqdevserver/tls.go +++ b/cmd/runmqdevserver/tls.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/runmqserver/license.go b/cmd/runmqserver/license.go index 19f199e..aad7810 100644 --- a/cmd/runmqserver/license.go +++ b/cmd/runmqserver/license.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/runmqserver/mirror.go b/cmd/runmqserver/mirror.go index 19b44e7..94c7dfd 100644 --- a/cmd/runmqserver/mirror.go +++ b/cmd/runmqserver/mirror.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/runmqserver/process.go b/cmd/runmqserver/process.go index dafd9b6..3f38c25 100644 --- a/cmd/runmqserver/process.go +++ b/cmd/runmqserver/process.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/runmqserver/signals.go b/cmd/runmqserver/signals.go index 003565d..f5460c8 100644 --- a/cmd/runmqserver/signals.go +++ b/cmd/runmqserver/signals.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/runmqserver/version.go b/cmd/runmqserver/version.go index 0142364..ec21f37 100644 --- a/cmd/runmqserver/version.go +++ b/cmd/runmqserver/version.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml index d575484..d4c2945 100644 --- a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml +++ b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml @@ -1,5 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + appSecurity-2.0 basicAuthenticationMQ-1.0 diff --git a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls-dev.xml b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls-dev.xml index 903656c..37645dd 100644 --- a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls-dev.xml +++ b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls-dev.xml @@ -1,5 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/internal/command/command.go b/internal/command/command.go index 7790c55..75322e8 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2018 +© Copyright IBM Corporation 2017, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index 6743779..26b6eea 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 0947465..3e3a099 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/internal/metrics/update.go b/internal/metrics/update.go index f2f470a..eb390f2 100644 --- a/internal/metrics/update.go +++ b/internal/metrics/update.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2018 +© Copyright IBM Corporation 2018, 2019 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/test/docker/Gopkg.toml b/test/docker/Gopkg.toml index 9dbde48..056e2d2 100644 --- a/test/docker/Gopkg.toml +++ b/test/docker/Gopkg.toml @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2017, 2018 +# © Copyright IBM Corporation 2017, 2019 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/messaging/pom.xml b/test/messaging/pom.xml index 90a3a99..9b32804 100644 --- a/test/messaging/pom.xml +++ b/test/messaging/pom.xml @@ -1,5 +1,5 @@