Merge pull request #185 from arthurbarr/9.1.0

Port latest fixes to 9.1.0

Dockerfile-server

@@ -21,12 +21,11 @@ ARG BUILDER_IMAGE=mq-golang-sdk:9.0.5.0-x86_64-ubuntu-16.04
 FROM $BUILDER_IMAGE as builder
 WORKDIR /go/src/github.com/ibm-messaging/mq-container/
 ARG IMAGE_REVISION="Not specified"
-ARG IMAGE_CREATED="Not specified"
 ARG IMAGE_SOURCE="Not specified"
 COPY cmd/ ./cmd
 COPY internal/ ./internal
 COPY vendor/ ./vendor
-RUN go build -ldflags "-X \"main.ImageCreated=$IMAGE_CREATED\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\"" ./cmd/runmqserver/
+RUN go build -ldflags "-X \"main.ImageCreated=$(date --iso-8601=seconds)\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\"" ./cmd/runmqserver/
 RUN go build ./cmd/chkmqready/
 RUN go build ./cmd/chkmqhealthy/
 # Run all unit tests

Makefile

@@ -62,8 +62,6 @@ DEV_JMS_IMAGE=mq-dev-jms-test
 # Variables for versioning
 IMAGE_REVISION=$(shell git rev-parse HEAD)
 IMAGE_SOURCE=$(shell git config --get remote.origin.url)
-IMAGE_CREATED=$(shell date -u +%Y-%m-%dT%H:%M:%S%:z)
-
 
 ifneq (,$(findstring Microsoft,$(shell uname -r)))
 	DOWNLOADS_DIR=$(patsubst /mnt/c%,C:%,$(realpath ./downloads/))
@@ -223,7 +221,6 @@ define docker-build-mq
 	  --build-arg BASE_IMAGE=$(BASE_IMAGE) \
 	  --build-arg BUILDER_IMAGE=$(MQ_IMAGE_GOLANG_SDK) \
 	  --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" \
-	  --build-arg IMAGE_CREATED="$(IMAGE_CREATED)" \
 	  --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" \
 	  --label IBM_PRODUCT_ID=$4 \
 	  --label IBM_PRODUCT_NAME=$5 \
@@ -241,7 +238,7 @@ docker-version:
 
 .PHONY: build-advancedserver
 build-advancedserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE)
-build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version build-golang-sdk
+build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version build-golang-sdk-ex
 	$(info $(SPACER)$(shell printf $(TITLE)"Build $(MQ_IMAGE_ADVANCEDSERVER)"$(END)))
 	$(call docker-build-mq,$(MQ_IMAGE_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION))
 
@@ -253,10 +250,10 @@ else
 build-devserver: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm MQSeriesWeb-*.rpm
 endif
 build-devserver: MQ_SDK_ARCHIVE=$(MQ_ARCHIVE_DEV)
-build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk
+build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version build-golang-sdk-ex
 	$(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER_BASE)"$(END)))
 	$(call docker-build-mq,$(MQ_IMAGE_DEVSERVER_BASE),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION))
-	$(DOCKER) build --tag $(MQ_IMAGE_DEVSERVER) --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" --build-arg IMAGE_CREATED="$(IMAGE_CREATED)" --build-arg BASE_IMAGE=$(MQ_IMAGE_DEVSERVER_BASE) --build-arg BUILDER_IMAGE=$(MQ_IMAGE_GOLANG_SDK) --file incubating/mqadvanced-server-dev/Dockerfile .
+	$(DOCKER) build --tag $(MQ_IMAGE_DEVSERVER) --build-arg IMAGE_SOURCE="$(IMAGE_SOURCE)" --build-arg IMAGE_REVISION="$(IMAGE_REVISION)" --build-arg BASE_IMAGE=$(MQ_IMAGE_DEVSERVER_BASE) --build-arg BUILDER_IMAGE=$(MQ_IMAGE_GOLANG_SDK) --file incubating/mqadvanced-server-dev/Dockerfile .
 
 .PHONY: build-advancedserver-cover
 build-advancedserver-cover: docker-version
@@ -266,15 +263,23 @@ build-advancedserver-cover: docker-version
 build-explorer: downloads/$(MQ_ARCHIVE_DEV)
 	$(call docker-build-mq,mq-explorer:latest-$(ARCH),incubating/mq-explorer/Dockerfile-mq-explorer,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION))
 
+.PHONY: build-sdk
+build-sdk: downloads/$(MQ_SDK_ARCHIVE) build-sdk-ex
+
+.PHONY: build-sdk-ex
 ifeq "$(findstring ubuntu,$(BASE_IMAGE))" "ubuntu"
-build-sdk: MQ_PACKAGES=ibmmq-sdk ibmmq-samples build-essential
+build-sdk-ex: MQ_PACKAGES=ibmmq-sdk ibmmq-samples build-essential
 else 
-build-sdk: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesSDK-*.rpm MQSeriesSamples*.rpm
+build-sdk-ex: MQ_PACKAGES=MQSeriesRuntime-*.rpm MQSeriesSDK-*.rpm MQSeriesSamples*.rpm
 endif
-build-sdk: downloads/$(MQ_SDK_ARCHIVE) docker-version docker-pull
+build-sdk-ex: docker-version docker-pull
 	$(call docker-build-mq,$(MQ_IMAGE_SDK),incubating/mq-sdk/Dockerfile,$(MQ_SDK_ARCHIVE),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers SDK (Non-Warranted)",$(MQ_VERSION))
 
-build-golang-sdk: downloads/$(MQ_SDK_ARCHIVE) docker-version build-sdk
+.PHONY: build-golang-sdk
+build-golang-sdk: downloads/$(MQ_SDK_ARCHIVE) build-golang-sdk-ex
+
+.PHONY: build-golang-sdk-ex
+build-golang-sdk-ex: docker-version build-sdk-ex
 	$(DOCKER) build --build-arg BASE_IMAGE=$(MQ_IMAGE_SDK) -t $(MQ_IMAGE_GOLANG_SDK) -f incubating/mq-golang-sdk/Dockerfile .
 #	$(call docker-build-mq,$(MQ_IMAGE_GOLANG_SDK),incubating/mq-golang-sdk/Dockerfile,$(MQ_IMAGE_SDK),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers SDK (Non-Warranted)",$(MQ_VERSION))
 

cmd/chkmqhealthy/main.go

@@ -39,7 +39,7 @@ func queueManagerHealthy() (bool, error) {
 		fmt.Println(err)
 		return false, err
 	}
-	fmt.Println(out)
+	fmt.Printf("%s", out)
 	if !strings.Contains(string(out), "(RUNNING)") {
 		return false, nil
 	}

cmd/runmqserver/mqconfig.go

@@ -131,7 +131,7 @@ func readMounts() error {
 		//dev := parts[0]
 		mountPoint := parts[1]
 		fsType := parts[2]
-		if strings.Contains(mountPoint, "/mnt") {
+		if strings.Contains(mountPoint, "/mnt/mqm") {
 			log.Printf("Detected '%v' volume mounted to %v", fsType, mountPoint)
 			detected = true
 		}

cmd/runmqserver/webserver.go

@@ -21,7 +21,9 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"syscall"
 
 	"github.com/ibm-messaging/mq-container/internal/command"
 )
@@ -33,7 +35,20 @@ func startWebServer() error {
 		return nil
 	}
 	log.Println("Starting web server")
-	out, rc, err := command.RunAsMQM("strmqweb")
+	cmd := exec.Command("strmqweb")
+	// Set a default app password for the web server, if one isn't already set
+	_, set := os.LookupEnv("MQ_APP_PASSWORD")
+	if !set {
+		// Take all current environment variables, and add the app password
+		cmd.Env = append(os.Environ(), "MQ_APP_PASSWORD=passw0rd")
+	}
+	cmd.SysProcAttr = &syscall.SysProcAttr{}
+	uid, gid, err := command.LookupMQM()
+	if err != nil {
+		return err
+	}
+	cmd.SysProcAttr.Credential = &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}
+	out, rc, err := command.RunCmd(cmd)
 	if err != nil {
 		log.Printf("Error %v starting web server: %v", rc, string(out))
 		return err

incubating/mqadvanced-server-dev/Dockerfile

@@ -20,14 +20,13 @@ ARG BUILDER_IMAGE=mq-golang-sdk:9.0.5.0-x86_64-ubuntu-16.04
 ###############################################################################
 FROM $BUILDER_IMAGE as builder
 ARG IMAGE_REVISION="Not specified"
-ARG IMAGE_CREATED="Not specified"
 ARG IMAGE_SOURCE="Not specified"
 WORKDIR /go/src/github.com/ibm-messaging/mq-container/
 COPY cmd/ ./cmd
 COPY internal/ ./internal
 COPY vendor/ ./vendor
 # Re-build runmqserver, with code tagged with 'mqdev' enabled
-RUN go build -ldflags "-X \"main.ImageCreated=$IMAGE_CREATED\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\"" --tags 'mqdev'  ./cmd/runmqserver
+RUN go build -ldflags "-X \"main.ImageCreated=$(date --iso-8601=seconds)\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\"" --tags 'mqdev'  ./cmd/runmqserver
 RUN go build ./cmd/runmqdevserver/
 # Run all unit tests
 RUN go test -v ./cmd/runmqdevserver/...

incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml

@@ -16,13 +16,23 @@
             <security-role name="MQWebAdmin">
                 <group name="MQWebUI" realm="defaultRealm"/>
             </security-role>
+            <security-role name="MQWebUser">
+                <group name="MQWebMessaging" realm="defaultRealm"/>
+            </security-role>
         </application-bnd>
     </enterpriseApplication>
     <basicRegistry id="basic" realm="defaultRealm">
         <user name="admin" password="${env.MQ_ADMIN_PASSWORD}"/>
+        <!-- The app user will always get a default password of "passw0rd",
+             even if you don't set the environment variable.
+             See `webserver.go` -->
+        <user name="app" password="${env.MQ_APP_PASSWORD}"/>
         <group name="MQWebUI">
             <member name="admin"/>
         </group>
+        <group name="MQWebMessaging">
+            <member name="app"/>
+        </group>
     </basicRegistry>
     <variable name="httpHost" value="*"/>
     <include location="tls.xml"/>

install-mq.sh

@@ -106,7 +106,7 @@ $UBUNTU && groupadd --system --gid 999 mqm
 $UBUNTU && useradd --system --uid 999 --gid mqm mqm
 $RHEL && groupadd --system --gid 888 mqm
 $RHEL && useradd --system --uid 888 --gid mqm mqm
-usermod -G mqm root
+usermod -aG mqm root
 
 # Find directory containing .deb files
 $UBUNTU && DIR_DEB=$(find ${DIR_EXTRACT} -name "*.deb" -printf "%h\n" | sort -u | head -1)

test/docker/devconfig_test.go

@@ -37,10 +37,11 @@ func TestDevGoldenPath(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	qm := "qm1"
 	containerConfig := container.Config{
 		Env: []string{
 			"LICENSE=accept",
-			"MQ_QMGR_NAME=qm1",
+			"MQ_QMGR_NAME=" + qm,
 		},
 	}
 	id := runContainerWithPorts(t, cli, &containerConfig, []int{9443})
@@ -49,7 +50,13 @@ func TestDevGoldenPath(t *testing.T) {
 	waitForWebReady(t, cli, id, insecureTLSConfig)
 	t.Run("JMS", func(t *testing.T) {
 		// Run the JMS tests, with no password specified
-		runJMSTests(t, cli, id, false, "app", "")
+		runJMSTests(t, cli, id, false, "app", defaultAppPasswordOS)
+	})
+	t.Run("REST admin", func(t *testing.T) {
+		testRESTAdmin(t, cli, id, insecureTLSConfig)
+	})
+	t.Run("REST messaging", func(t *testing.T) {
+		testRESTMessaging(t, cli, id, insecureTLSConfig, qm, "app", defaultAppPasswordWeb)
 	})
 	// Stop the container cleanly
 	stopContainer(t, cli, id)
@@ -64,11 +71,13 @@ func TestDevSecure(t *testing.T) {
 		t.Fatal(err)
 	}
 	const tlsPassPhrase string = "passw0rd"
+	qm := "qm1"
+	appPassword := "differentPassw0rd"
 	containerConfig := container.Config{
 		Env: []string{
 			"LICENSE=accept",
-			"MQ_QMGR_NAME=qm1",
+			"MQ_QMGR_NAME=" + qm,
-			"MQ_APP_PASSWORD=" + devAppPassword,
+			"MQ_APP_PASSWORD=" + appPassword,
 			"MQ_TLS_KEYSTORE=/var/tls/server.p12",
 			"MQ_TLS_PASSPHRASE=" + tlsPassPhrase,
 			"DEBUG=1",
@@ -100,7 +109,17 @@ func TestDevSecure(t *testing.T) {
 	waitForReady(t, cli, ctr.ID)
 	cert := filepath.Join(tlsDir(t, true), "server.crt")
 	waitForWebReady(t, cli, ctr.ID, createTLSConfig(t, cert, tlsPassPhrase))
-	runJMSTests(t, cli, ctr.ID, true, "app", devAppPassword)
+
+	t.Run("JMS", func(t *testing.T) {
+		runJMSTests(t, cli, ctr.ID, true, "app", appPassword)
+	})
+	t.Run("REST admin", func(t *testing.T) {
+		testRESTAdmin(t, cli, ctr.ID, insecureTLSConfig)
+	})
+	t.Run("REST messaging", func(t *testing.T) {
+		testRESTMessaging(t, cli, ctr.ID, insecureTLSConfig, qm, "app", appPassword)
+	})
+
 	// Stop the container cleanly
 	stopContainer(t, cli, ctr.ID)
 }
@@ -129,7 +148,7 @@ func TestDevWebDisabled(t *testing.T) {
 	})
 	t.Run("JMS", func(t *testing.T) {
 		// Run the JMS tests, with no password specified
-		runJMSTests(t, cli, id, false, "app", "")
+		runJMSTests(t, cli, id, false, "app", defaultAppPasswordOS)
 	})
 	// Stop the container cleanly
 	stopContainer(t, cli, id)

test/docker/devconfig_test_util.go

@@ -18,12 +18,14 @@ limitations under the License.
 package main
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httputil"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -34,8 +36,9 @@ import (
 	"github.com/docker/docker/client"
 )
 
-const devAdminPassword string = "passw0rd"
+const defaultAdminPassword string = "passw0rd"
-const devAppPassword string = "passw0rd"
+const defaultAppPasswordOS string = ""
+const defaultAppPasswordWeb string = "passw0rd"
 
 // Disable TLS verification (server uses a self-signed certificate by default,
 // so verification isn't useful anyway)
@@ -58,7 +61,7 @@ func waitForWebReady(t *testing.T, cli *client.Client, ID string, tlsConfig *tls
 		select {
 		case <-time.After(1 * time.Second):
 			req, err := http.NewRequest("GET", url, nil)
-			req.SetBasicAuth("admin", devAdminPassword)
+			req.SetBasicAuth("admin", defaultAdminPassword)
 			resp, err := httpClient.Do(req.WithContext(ctx))
 			if err == nil && resp.StatusCode == http.StatusOK {
 				t.Log("MQ web server is ready")
@@ -140,17 +143,16 @@ func createTLSConfig(t *testing.T, certFile, password string) *tls.Config {
 	}
 }
 
-func testREST(t *testing.T, cli *client.Client, ID string, tlsConfig *tls.Config) {
+func testRESTAdmin(t *testing.T, cli *client.Client, ID string, tlsConfig *tls.Config) {
 	httpClient := http.Client{
 		Timeout: time.Duration(30 * time.Second),
 		Transport: &http.Transport{
 			TLSClientConfig: tlsConfig,
 		},
 	}
-
 	url := fmt.Sprintf("https://localhost:%s/ibmmq/rest/v1/admin/installation", getPort(t, cli, ID, 9443))
 	req, err := http.NewRequest("GET", url, nil)
-	req.SetBasicAuth("admin", devAdminPassword)
+	req.SetBasicAuth("admin", defaultAdminPassword)
 	resp, err := httpClient.Do(req)
 	if err != nil {
 		t.Fatal(err)
@@ -159,3 +161,70 @@ func testREST(t *testing.T, cli *client.Client, ID string, tlsConfig *tls.Config
 		t.Errorf("Expected HTTP status code %v from 'GET installation'; got %v", http.StatusOK, resp.StatusCode)
 	}
 }
+
+// curl -i -k https://localhost:1234/ibmmq/rest/v1/messaging/qmgr/qm1/queue/DEV.QUEUE.1/message -X POST -u app -H “ibm-mq-rest-csrf-token: N/A” -H “Content-Type: text/plain;charset=utf-8" -d “Hello World”
+
+func logHTTPRequest(t *testing.T, req *http.Request) {
+	d, err := httputil.DumpRequestOut(req, true)
+	if err != nil {
+		t.Error(err)
+	}
+	t.Logf("HTTP request: %v", string(d))
+}
+
+func logHTTPResponse(t *testing.T, resp *http.Response) {
+	d, err := httputil.DumpResponse(resp, true)
+	if err != nil {
+		t.Error(err)
+	}
+	t.Logf("HTTP response: %v", string(d))
+}
+
+func testRESTMessaging(t *testing.T, cli *client.Client, ID string, tlsConfig *tls.Config, qmName string, user string, password string) {
+	httpClient := http.Client{
+		Timeout: time.Duration(30 * time.Second),
+		Transport: &http.Transport{
+			TLSClientConfig: tlsConfig,
+		},
+	}
+	q := "DEV.QUEUE.1"
+	url := fmt.Sprintf("https://localhost:%s/ibmmq/rest/v1/messaging/qmgr/%s/queue/%s/message", getPort(t, cli, ID, 9443), qmName, q)
+	putMessage := []byte("Hello")
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(putMessage))
+	req.SetBasicAuth(user, password)
+	req.Header.Add("ibm-mq-rest-csrf-token", "n/a")
+	req.Header.Add("Content-Type", "text/plain;charset=utf-8")
+	logHTTPRequest(t, req)
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	logHTTPResponse(t, resp)
+	if resp.StatusCode != http.StatusCreated {
+		t.Errorf("Expected HTTP status code %v from 'POST to queue'; got %v", http.StatusOK, resp.StatusCode)
+		t.Logf("HTTP response: %+v", resp)
+		t.Fail()
+	}
+
+	req, err = http.NewRequest("DELETE", url, nil)
+	req.Header.Add("ibm-mq-rest-csrf-token", "n/a")
+	req.SetBasicAuth(user, password)
+	logHTTPRequest(t, req)
+	resp, err = httpClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	logHTTPResponse(t, resp)
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("Expected HTTP status code %v from 'DELETE from queue'; got %v", http.StatusOK, resp.StatusCode)
+		t.Logf("HTTP response: %+v", resp)
+		t.Fail()
+	}
+	gotMessage, err := ioutil.ReadAll(resp.Body)
+	//gotMessage := string(b)
+	if string(gotMessage) != string(putMessage) {
+		t.Errorf("Expected payload to be \"%s\"; got \"%s\"", putMessage, gotMessage)
+	}
+}

test/messaging/src/main/java/com/ibm/mqcontainer/test/JMSTests.java

@@ -73,6 +73,10 @@ class JMSTests {
         factory.setTransportType(WMQConstants.WMQ_CM_CLIENT);
         factory.setChannel(channel);
         factory.setConnectionNameList(String.format("%s(1414)", addr));
+        // If a password is set, make sure it gets sent to the queue manager for authentication
+        if (password != null) {
+            factory.setBooleanProperty(WMQConstants.USER_AUTHENTICATION_MQCSP, true);
+        }
         // factory.setClientReconnectOptions(WMQConstants.WMQ_CLIENT_RECONNECT);
         if (TRUSTSTORE == null) {
             LOGGER.info("Not using TLS");