Use v2.0.0 of mq-golang now it has been created (#159)

* Use v2.0.0 of mq-golang now it has been created

* do a strip vendor

vendor/golang.org/x/sys/unix/linux/Dockerfile

@@ -1,23 +1,25 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 # Dependencies to get the git sources and go binaries
-RUN apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y  --no-install-recommends \
+        ca-certificates \
         curl \
         git \
+    && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 # Get the git sources. If not cached, this takes O(5 minutes).
 WORKDIR /git
 RUN git config --global advice.detachedHead false
-# Linux Kernel: Released 19 Feb 2017
-RUN git clone --branch v4.10 --depth 1 https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux
-# GNU C library: Released 05 Feb 2017 (we should try to get a secure way to clone this)
-RUN git clone --branch glibc-2.25 --depth 1 git://sourceware.org/git/glibc.git
+# Linux Kernel: Released 03 Jun 2018
+RUN git clone --branch v4.17 --depth 1 https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux
+# GNU C library: Released 01 Feb 2018 (we should try to get a secure way to clone this)
+RUN git clone --branch glibc-2.27 --depth 1 git://sourceware.org/git/glibc.git
 
-# Get Go 1.8 (https://github.com/docker-library/golang/blob/master/1.8/Dockerfile)
-ENV GOLANG_VERSION 1.8
+# Get Go
+ENV GOLANG_VERSION 1.11beta1
 ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz
-ENV GOLANG_DOWNLOAD_SHA256 53ab94104ee3923e228a2cb2116e5e462ad3ebaeea06ff04463479d7f12d27ca
+ENV GOLANG_DOWNLOAD_SHA256 df7fe096ffab5d331d35c6d038d2ec0426b45ce17f55a93037e371d3af9d4e6d
 
 RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
     && echo "$GOLANG_DOWNLOAD_SHA256  golang.tar.gz" | sha256sum -c - \
@@ -26,20 +28,22 @@ RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
 
 ENV PATH /usr/local/go/bin:$PATH
 
-# Linux and Glibc build dependencies
-RUN apt-get update && apt-get install -y \
-        gawk make python \
+# Linux and Glibc build dependencies and emulator
+RUN apt-get update && apt-get install -y  --no-install-recommends \
+        bison gawk make python \
         gcc gcc-multilib \
-        gettext  texinfo \
-    && rm -rf /var/lib/apt/lists/*
-# Emulator and cross compilers
-RUN apt-get update && apt-get install -y \
+        gettext texinfo \
         qemu \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+# Cross compilers (install recommended packages to get cross libc-dev)
+RUN apt-get update && apt-get install -y \
         gcc-aarch64-linux-gnu       gcc-arm-linux-gnueabi     \
         gcc-mips-linux-gnu          gcc-mips64-linux-gnuabi64 \
         gcc-mips64el-linux-gnuabi64 gcc-mipsel-linux-gnu      \
         gcc-powerpc64-linux-gnu     gcc-powerpc64le-linux-gnu \
         gcc-s390x-linux-gnu         gcc-sparc64-linux-gnu     \
+    && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 # Let the scripts know they are in the docker environment

vendor/golang.org/x/sys/unix/linux/mkall.go

@@ -15,12 +15,20 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
+	"debug/elf"
+	"encoding/binary"
+	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
+	"unicode"
 )
 
 // These will be paths to the appropriate source directories.
@@ -128,6 +136,15 @@ var targets = []target{
 	// },
 }
 
+// ptracePairs is a list of pairs of targets that can, in some cases,
+// run each other's binaries.
+var ptracePairs = []struct{ a1, a2 string }{
+	{"386", "amd64"},
+	{"arm", "arm64"},
+	{"mips", "mips64"},
+	{"mipsle", "mips64le"},
+}
+
 func main() {
 	if runtime.GOOS != GOOS || runtime.GOARCH != BuildArch {
 		fmt.Printf("Build system has GOOS_GOARCH = %s_%s, need %s_%s\n",
@@ -158,6 +175,18 @@ func main() {
 			fmt.Printf("----- SUCCESS:    %s -----\n\n", t.GoArch)
 		}
 	}
+
+	fmt.Printf("----- GENERATING ptrace pairs -----\n")
+	ok := true
+	for _, p := range ptracePairs {
+		if err := generatePtracePair(p.a1, p.a2); err != nil {
+			fmt.Printf("%v\n***** FAILURE: %s/%s *****\n\n", err, p.a1, p.a2)
+			ok = false
+		}
+	}
+	if ok {
+		fmt.Printf("----- SUCCESS ptrace pairs    -----\n\n")
+	}
 }
 
 // Makes an exec.Cmd with Stderr attached to os.Stderr
@@ -266,7 +295,7 @@ func (t *target) generateFiles() error {
 	return nil
 }
 
-// Create the Linux and glibc headers in the include directory.
+// Create the Linux, glibc and ABI (C compiler convention) headers in the include directory.
 func (t *target) makeHeaders() error {
 	// Make the Linux headers we need for this architecture
 	linuxMake := makeCommand("make", "headers_install", "ARCH="+t.LinuxArch, "INSTALL_HDR_PATH="+TempDir)
@@ -301,6 +330,114 @@ func (t *target) makeHeaders() error {
 		file.Close()
 	}
 
+	// ABI headers will specify C compiler behavior for the target platform.
+	return t.makeABIHeaders()
+}
+
+// makeABIHeaders generates C header files based on the platform's calling convention.
+// While many platforms have formal Application Binary Interfaces, in practice, whatever the
+// dominant C compilers generate is the de-facto calling convention.
+//
+// We generate C headers instead of a Go file, so as to enable references to the ABI from Cgo.
+func (t *target) makeABIHeaders() (err error) {
+	abiDir := filepath.Join(IncludeDir, "abi")
+	if err = os.Mkdir(abiDir, os.ModePerm); err != nil {
+		return err
+	}
+
+	cc := os.Getenv("CC")
+	if cc == "" {
+		return errors.New("CC (compiler) env var not set")
+	}
+
+	// Build a sacrificial ELF file, to mine for C compiler behavior.
+	binPath := filepath.Join(TempDir, "tmp_abi.o")
+	bin, err := t.buildELF(cc, cCode, binPath)
+	if err != nil {
+		return fmt.Errorf("cannot build ELF to analyze: %v", err)
+	}
+	defer bin.Close()
+	defer os.Remove(binPath)
+
+	// Right now, we put everything in abi.h, but we may change this later.
+	abiFile, err := os.Create(filepath.Join(abiDir, "abi.h"))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if cerr := abiFile.Close(); cerr != nil && err == nil {
+			err = cerr
+		}
+	}()
+
+	if err = t.writeBitFieldMasks(bin, abiFile); err != nil {
+		return fmt.Errorf("cannot write bitfield masks: %v", err)
+	}
+
+	return nil
+}
+
+func (t *target) buildELF(cc, src, path string) (*elf.File, error) {
+	// Compile the cCode source using the set compiler - we will need its .data section.
+	// Do not link the binary, so that we can find .data section offsets from the symbol values.
+	ccCmd := makeCommand(cc, "-o", path, "-gdwarf", "-x", "c", "-c", "-")
+	ccCmd.Stdin = strings.NewReader(src)
+	ccCmd.Stdout = os.Stdout
+	if err := ccCmd.Run(); err != nil {
+		return nil, fmt.Errorf("compiler error: %v", err)
+	}
+
+	bin, err := elf.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read ELF file %s: %v", path, err)
+	}
+
+	return bin, nil
+}
+
+func (t *target) writeBitFieldMasks(bin *elf.File, out io.Writer) error {
+	symbols, err := bin.Symbols()
+	if err != nil {
+		return fmt.Errorf("getting ELF symbols: %v", err)
+	}
+	var masksSym *elf.Symbol
+
+	for _, sym := range symbols {
+		if sym.Name == "masks" {
+			masksSym = &sym
+		}
+	}
+
+	if masksSym == nil {
+		return errors.New("could not find the 'masks' symbol in ELF symtab")
+	}
+
+	dataSection := bin.Section(".data")
+	if dataSection == nil {
+		return errors.New("ELF file has no .data section")
+	}
+
+	data, err := dataSection.Data()
+	if err != nil {
+		return fmt.Errorf("could not read .data section: %v\n", err)
+	}
+
+	var bo binary.ByteOrder
+	if t.BigEndian {
+		bo = binary.BigEndian
+	} else {
+		bo = binary.LittleEndian
+	}
+
+	// 64 bit masks of type uint64 are stored in the data section starting at masks.Value.
+	// Here we are running on AMD64, but these values may be big endian or little endian,
+	// depending on target architecture.
+	for i := uint64(0); i < 64; i++ {
+		off := masksSym.Value + i*8
+		// Define each mask in native by order, so as to match target endian.
+		fmt.Fprintf(out, "#define BITFIELD_MASK_%d %dULL\n", i, bo.Uint64(data[off:off+8]))
+	}
+
 	return nil
 }
 
@@ -377,3 +514,237 @@ func (t *target) mksyscallFlags() (flags []string) {
 	}
 	return
 }
+
+// generatePtracePair takes a pair of GOARCH values that can run each
+// other's binaries, such as 386 and amd64. It extracts the PtraceRegs
+// type for each one. It writes a new file defining the types
+// PtraceRegsArch1 and PtraceRegsArch2 and the corresponding functions
+// Ptrace{Get,Set}Regs{arch1,arch2}. This permits debugging the other
+// binary on a native system.
+func generatePtracePair(arch1, arch2 string) error {
+	def1, err := ptraceDef(arch1)
+	if err != nil {
+		return err
+	}
+	def2, err := ptraceDef(arch2)
+	if err != nil {
+		return err
+	}
+	f, err := os.Create(fmt.Sprintf("zptrace%s_linux.go", arch1))
+	if err != nil {
+		return err
+	}
+	buf := bufio.NewWriter(f)
+	fmt.Fprintf(buf, "// Code generated by linux/mkall.go generatePtracePair(%s, %s). DO NOT EDIT.\n", arch1, arch2)
+	fmt.Fprintf(buf, "\n")
+	fmt.Fprintf(buf, "// +build linux\n")
+	fmt.Fprintf(buf, "// +build %s %s\n", arch1, arch2)
+	fmt.Fprintf(buf, "\n")
+	fmt.Fprintf(buf, "package unix\n")
+	fmt.Fprintf(buf, "\n")
+	fmt.Fprintf(buf, "%s\n", `import "unsafe"`)
+	fmt.Fprintf(buf, "\n")
+	writeOnePtrace(buf, arch1, def1)
+	fmt.Fprintf(buf, "\n")
+	writeOnePtrace(buf, arch2, def2)
+	if err := buf.Flush(); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ptraceDef returns the definition of PtraceRegs for arch.
+func ptraceDef(arch string) (string, error) {
+	filename := fmt.Sprintf("ztypes_linux_%s.go", arch)
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return "", fmt.Errorf("reading %s: %v", filename, err)
+	}
+	start := bytes.Index(data, []byte("type PtraceRegs struct"))
+	if start < 0 {
+		return "", fmt.Errorf("%s: no definition of PtraceRegs", filename)
+	}
+	data = data[start:]
+	end := bytes.Index(data, []byte("\n}\n"))
+	if end < 0 {
+		return "", fmt.Errorf("%s: can't find end of PtraceRegs definition", filename)
+	}
+	return string(data[:end+2]), nil
+}
+
+// writeOnePtrace writes out the ptrace definitions for arch.
+func writeOnePtrace(w io.Writer, arch, def string) {
+	uarch := string(unicode.ToUpper(rune(arch[0]))) + arch[1:]
+	fmt.Fprintf(w, "// PtraceRegs%s is the registers used by %s binaries.\n", uarch, arch)
+	fmt.Fprintf(w, "%s\n", strings.Replace(def, "PtraceRegs", "PtraceRegs"+uarch, 1))
+	fmt.Fprintf(w, "\n")
+	fmt.Fprintf(w, "// PtraceGetRegs%s fetches the registers used by %s binaries.\n", uarch, arch)
+	fmt.Fprintf(w, "func PtraceGetRegs%s(pid int, regsout *PtraceRegs%s) error {\n", uarch, uarch)
+	fmt.Fprintf(w, "\treturn ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout)))\n")
+	fmt.Fprintf(w, "}\n")
+	fmt.Fprintf(w, "\n")
+	fmt.Fprintf(w, "// PtraceSetRegs%s sets the registers used by %s binaries.\n", uarch, arch)
+	fmt.Fprintf(w, "func PtraceSetRegs%s(pid int, regs *PtraceRegs%s) error {\n", uarch, uarch)
+	fmt.Fprintf(w, "\treturn ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs)))\n")
+	fmt.Fprintf(w, "}\n")
+}
+
+// cCode is compiled for the target architecture, and the resulting data section is carved for
+// the statically initialized bit masks.
+const cCode = `
+// Bit fields are used in some system calls and other ABIs, but their memory layout is
+// implementation-defined [1]. Even with formal ABIs, bit fields are a source of subtle bugs [2].
+// Here we generate the offsets for all 64 bits in an uint64.
+// 1: http://en.cppreference.com/w/c/language/bit_field
+// 2: https://lwn.net/Articles/478657/
+
+#include <stdint.h>
+
+struct bitfield {
+	union {
+		uint64_t val;
+		struct {
+			uint64_t u64_bit_0 : 1;
+			uint64_t u64_bit_1 : 1;
+			uint64_t u64_bit_2 : 1;
+			uint64_t u64_bit_3 : 1;
+			uint64_t u64_bit_4 : 1;
+			uint64_t u64_bit_5 : 1;
+			uint64_t u64_bit_6 : 1;
+			uint64_t u64_bit_7 : 1;
+			uint64_t u64_bit_8 : 1;
+			uint64_t u64_bit_9 : 1;
+			uint64_t u64_bit_10 : 1;
+			uint64_t u64_bit_11 : 1;
+			uint64_t u64_bit_12 : 1;
+			uint64_t u64_bit_13 : 1;
+			uint64_t u64_bit_14 : 1;
+			uint64_t u64_bit_15 : 1;
+			uint64_t u64_bit_16 : 1;
+			uint64_t u64_bit_17 : 1;
+			uint64_t u64_bit_18 : 1;
+			uint64_t u64_bit_19 : 1;
+			uint64_t u64_bit_20 : 1;
+			uint64_t u64_bit_21 : 1;
+			uint64_t u64_bit_22 : 1;
+			uint64_t u64_bit_23 : 1;
+			uint64_t u64_bit_24 : 1;
+			uint64_t u64_bit_25 : 1;
+			uint64_t u64_bit_26 : 1;
+			uint64_t u64_bit_27 : 1;
+			uint64_t u64_bit_28 : 1;
+			uint64_t u64_bit_29 : 1;
+			uint64_t u64_bit_30 : 1;
+			uint64_t u64_bit_31 : 1;
+			uint64_t u64_bit_32 : 1;
+			uint64_t u64_bit_33 : 1;
+			uint64_t u64_bit_34 : 1;
+			uint64_t u64_bit_35 : 1;
+			uint64_t u64_bit_36 : 1;
+			uint64_t u64_bit_37 : 1;
+			uint64_t u64_bit_38 : 1;
+			uint64_t u64_bit_39 : 1;
+			uint64_t u64_bit_40 : 1;
+			uint64_t u64_bit_41 : 1;
+			uint64_t u64_bit_42 : 1;
+			uint64_t u64_bit_43 : 1;
+			uint64_t u64_bit_44 : 1;
+			uint64_t u64_bit_45 : 1;
+			uint64_t u64_bit_46 : 1;
+			uint64_t u64_bit_47 : 1;
+			uint64_t u64_bit_48 : 1;
+			uint64_t u64_bit_49 : 1;
+			uint64_t u64_bit_50 : 1;
+			uint64_t u64_bit_51 : 1;
+			uint64_t u64_bit_52 : 1;
+			uint64_t u64_bit_53 : 1;
+			uint64_t u64_bit_54 : 1;
+			uint64_t u64_bit_55 : 1;
+			uint64_t u64_bit_56 : 1;
+			uint64_t u64_bit_57 : 1;
+			uint64_t u64_bit_58 : 1;
+			uint64_t u64_bit_59 : 1;
+			uint64_t u64_bit_60 : 1;
+			uint64_t u64_bit_61 : 1;
+			uint64_t u64_bit_62 : 1;
+			uint64_t u64_bit_63 : 1;
+		};
+	};
+};
+
+struct bitfield masks[] = {
+	{.u64_bit_0 = 1},
+	{.u64_bit_1 = 1},
+	{.u64_bit_2 = 1},
+	{.u64_bit_3 = 1},
+	{.u64_bit_4 = 1},
+	{.u64_bit_5 = 1},
+	{.u64_bit_6 = 1},
+	{.u64_bit_7 = 1},
+	{.u64_bit_8 = 1},
+	{.u64_bit_9 = 1},
+	{.u64_bit_10 = 1},
+	{.u64_bit_11 = 1},
+	{.u64_bit_12 = 1},
+	{.u64_bit_13 = 1},
+	{.u64_bit_14 = 1},
+	{.u64_bit_15 = 1},
+	{.u64_bit_16 = 1},
+	{.u64_bit_17 = 1},
+	{.u64_bit_18 = 1},
+	{.u64_bit_19 = 1},
+	{.u64_bit_20 = 1},
+	{.u64_bit_21 = 1},
+	{.u64_bit_22 = 1},
+	{.u64_bit_23 = 1},
+	{.u64_bit_24 = 1},
+	{.u64_bit_25 = 1},
+	{.u64_bit_26 = 1},
+	{.u64_bit_27 = 1},
+	{.u64_bit_28 = 1},
+	{.u64_bit_29 = 1},
+	{.u64_bit_30 = 1},
+	{.u64_bit_31 = 1},
+	{.u64_bit_32 = 1},
+	{.u64_bit_33 = 1},
+	{.u64_bit_34 = 1},
+	{.u64_bit_35 = 1},
+	{.u64_bit_36 = 1},
+	{.u64_bit_37 = 1},
+	{.u64_bit_38 = 1},
+	{.u64_bit_39 = 1},
+	{.u64_bit_40 = 1},
+	{.u64_bit_41 = 1},
+	{.u64_bit_42 = 1},
+	{.u64_bit_43 = 1},
+	{.u64_bit_44 = 1},
+	{.u64_bit_45 = 1},
+	{.u64_bit_46 = 1},
+	{.u64_bit_47 = 1},
+	{.u64_bit_48 = 1},
+	{.u64_bit_49 = 1},
+	{.u64_bit_50 = 1},
+	{.u64_bit_51 = 1},
+	{.u64_bit_52 = 1},
+	{.u64_bit_53 = 1},
+	{.u64_bit_54 = 1},
+	{.u64_bit_55 = 1},
+	{.u64_bit_56 = 1},
+	{.u64_bit_57 = 1},
+	{.u64_bit_58 = 1},
+	{.u64_bit_59 = 1},
+	{.u64_bit_60 = 1},
+	{.u64_bit_61 = 1},
+	{.u64_bit_62 = 1},
+	{.u64_bit_63 = 1}
+};
+
+int main(int argc, char **argv) {
+	struct bitfield *mask_ptr = &masks[0];
+	return mask_ptr->val;
+}
+
+`