Merge branch '2860-fix-random-unknown-signal-panics-for-windows-docker' into 'master'

Update github.com/Microsoft/go-winio dependency

Closes #2860

See merge request !1348

Gopkg.lock

@@ -48,11 +48,12 @@
   revision = "a368813c5e648fee92e5f6c30e3944ff9d5e8895"
 
 [[projects]]
-  digest = "1:f6c9fe343d9c60ca0604237a80d3b46edd6deae0c7ab0471fbf1d8bdb645ac21"
+  digest = "1:767982ab2a5a0e94d51545496626182d678b5590e6eddc05353c3b39fab45c9a"
   name = "github.com/Microsoft/go-winio"
   packages = ["."]
   pruneopts = "N"
-  revision = "ce2922f643c8fd76b46cadc7f404a06282678b34"
+  revision = "1a8911d1ed007260465c3bfbbc785ac6915a0bb8"
+  version = "v0.4.12"
 
 [[projects]]
   branch = "master"

Gopkg.toml

@@ -172,7 +172,7 @@ ignored = ["test", "appengine"]
 
 [[override]]
   name = "github.com/Microsoft/go-winio"
-  revision = "ce2922f643c8fd76b46cadc7f404a06282678b34"
+  version = "v0.4.12"
 
 [[override]]
   name = "github.com/blang/semver"

vendor/github.com/Microsoft/go-winio/backup.go

+// +build windows
+
 package winio
 
 import (
@@ -66,10 +68,20 @@ func NewBackupStreamReader(r io.Reader) *BackupStreamReader {
 	return &BackupStreamReader{r, 0}
 }
 
-// Next returns the next backup stream and prepares for calls to Write(). It skips the remainder of the current stream if
+// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if
 // it was not completely read.
 func (r *BackupStreamReader) Next() (*BackupHeader, error) {
 	if r.bytesLeft > 0 {
+		if s, ok := r.r.(io.Seeker); ok {
+			// Make sure Seek on io.SeekCurrent sometimes succeeds
+			// before trying the actual seek.
+			if _, err := s.Seek(0, io.SeekCurrent); err == nil {
+				if _, err = s.Seek(r.bytesLeft, io.SeekCurrent); err != nil {
+					return nil, err
+				}
+				r.bytesLeft = 0
+			}
+		}
 		if _, err := io.Copy(ioutil.Discard, r); err != nil {
 			return nil, err
 		}
@@ -183,7 +195,6 @@ type BackupFileReader struct {
 // Read will attempt to read the security descriptor of the file.
 func NewBackupFileReader(f *os.File, includeSecurity bool) *BackupFileReader {
 	r := &BackupFileReader{f, includeSecurity, 0}
-	runtime.SetFinalizer(r, func(r *BackupFileReader) { r.Close() })
 	return r
 }
 
@@ -194,6 +205,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) {
 	if err != nil {
 		return 0, &os.PathError{"BackupRead", r.f.Name(), err}
 	}
+	runtime.KeepAlive(r.f)
 	if bytesRead == 0 {
 		return 0, io.EOF
 	}
@@ -205,6 +217,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) {
 func (r *BackupFileReader) Close() error {
 	if r.ctx != 0 {
 		backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
+		runtime.KeepAlive(r.f)
 		r.ctx = 0
 	}
 	return nil
@@ -217,11 +230,10 @@ type BackupFileWriter struct {
 	ctx             uintptr
 }
 
-// NewBackupFileWrtier returns a new BackupFileWriter from a file handle. If includeSecurity is true,
+// NewBackupFileWriter returns a new BackupFileWriter from a file handle. If includeSecurity is true,
 // Write() will attempt to restore the security descriptor from the stream.
 func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter {
 	w := &BackupFileWriter{f, includeSecurity, 0}
-	runtime.SetFinalizer(w, func(w *BackupFileWriter) { w.Close() })
 	return w
 }
 
@@ -232,6 +244,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) {
 	if err != nil {
 		return 0, &os.PathError{"BackupWrite", w.f.Name(), err}
 	}
+	runtime.KeepAlive(w.f)
 	if int(bytesWritten) != len(b) {
 		return int(bytesWritten), errors.New("not all bytes could be written")
 	}
@@ -243,6 +256,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) {
 func (w *BackupFileWriter) Close() error {
 	if w.ctx != 0 {
 		backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
+		runtime.KeepAlive(w.f)
 		w.ctx = 0
 	}
 	return nil

vendor/github.com/Microsoft/go-winio/backuptar/noop.go

+// +build !windows
+// This file only exists to allow go get on non-Windows platforms.
+
+package backuptar

vendor/github.com/Microsoft/go-winio/backuptar/tar.go

+// +build windows
+
 package backuptar
 
 import (
@@ -34,6 +36,7 @@ const (
 	hdrSecurityDescriptor    = "sd"
 	hdrRawSecurityDescriptor = "rawsd"
 	hdrMountPoint            = "mountpoint"
+	hdrEaPrefix              = "xattr."
 )
 
 func writeZeroes(w io.Writer, count int64) error {
@@ -116,6 +119,21 @@ func BasicInfoHeader(name string, size int64, fileInfo *winio.FileBasicInfo) *ta
 func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error {
 	name = filepath.ToSlash(name)
 	hdr := BasicInfoHeader(name, size, fileInfo)
+
+	// If r can be seeked, then this function is two-pass: pass 1 collects the
+	// tar header data, and pass 2 copies the data stream. If r cannot be
+	// seeked, then some header data (in particular EAs) will be silently lost.
+	var (
+		restartPos int64
+		err        error
+	)
+	sr, readTwice := r.(io.Seeker)
+	if readTwice {
+		if restartPos, err = sr.Seek(0, io.SeekCurrent); err != nil {
+			readTwice = false
+		}
+	}
+
 	br := winio.NewBackupStreamReader(r)
 	var dataHdr *winio.BackupHeader
 	for dataHdr == nil {
@@ -129,7 +147,9 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
 		switch bhdr.Id {
 		case winio.BackupData:
 			hdr.Mode |= c_ISREG
-			dataHdr = bhdr
+			if !readTwice {
+				dataHdr = bhdr
+			}
 		case winio.BackupSecurity:
 			sd, err := ioutil.ReadAll(br)
 			if err != nil {
@@ -149,18 +169,54 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
 				hdr.Winheaders[hdrMountPoint] = "1"
 			}
 			hdr.Linkname = rp.Target
-		case winio.BackupEaData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData:
+
+		case winio.BackupEaData:
+			eab, err := ioutil.ReadAll(br)
+			if err != nil {
+				return err
+			}
+			eas, err := winio.DecodeExtendedAttributes(eab)
+			if err != nil {
+				return err
+			}
+			for _, ea := range eas {
+				// Use base64 encoding for the binary value. Note that there
+				// is no way to encode the EA's flags, since their use doesn't
+				// make any sense for persisted EAs.
+				hdr.Winheaders[hdrEaPrefix+ea.Name] = base64.StdEncoding.EncodeToString(ea.Value)
+			}
+
+		case winio.BackupAlternateData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData:
 			// ignore these streams
 		default:
 			return fmt.Errorf("%s: unknown stream ID %d", name, bhdr.Id)
 		}
 	}
 
-	err := t.WriteHeader(hdr)
+	err = t.WriteHeader(hdr)
 	if err != nil {
 		return err
 	}
 
+	if readTwice {
+		// Get back to the data stream.
+		if _, err = sr.Seek(restartPos, io.SeekStart); err != nil {
+			return err
+		}
+		for dataHdr == nil {
+			bhdr, err := br.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			if bhdr.Id == winio.BackupData {
+				dataHdr = bhdr
+			}
+		}
+	}
+
 	if dataHdr != nil {
 		// A data stream was found. Copy the data.
 		if (dataHdr.Attributes & winio.StreamSparseAttributes) == 0 {
@@ -247,7 +303,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 		if err != nil {
 			return "", 0, nil, err
 		}
-		fileInfo.FileAttributes = uintptr(attr)
+		fileInfo.FileAttributes = uint32(attr)
 	} else {
 		if hdr.Typeflag == tar.TypeDir {
 			fileInfo.FileAttributes |= syscall.FILE_ATTRIBUTE_DIRECTORY
@@ -291,6 +347,38 @@ func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (
 			return nil, err
 		}
 	}
+	var eas []winio.ExtendedAttribute
+	for k, v := range hdr.Winheaders {
+		if !strings.HasPrefix(k, hdrEaPrefix) {
+			continue
+		}
+		data, err := base64.StdEncoding.DecodeString(v)
+		if err != nil {
+			return nil, err
+		}
+		eas = append(eas, winio.ExtendedAttribute{
+			Name:  k[len(hdrEaPrefix):],
+			Value: data,
+		})
+	}
+	if len(eas) != 0 {
+		eadata, err := winio.EncodeExtendedAttributes(eas)
+		if err != nil {
+			return nil, err
+		}
+		bhdr := winio.BackupHeader{
+			Id:   winio.BackupEaData,
+			Size: int64(len(eadata)),
+		}
+		err = bw.WriteHeader(&bhdr)
+		if err != nil {
+			return nil, err
+		}
+		_, err = bw.Write(eadata)
+		if err != nil {
+			return nil, err
+		}
+	}
 	if hdr.Typeflag == tar.TypeSymlink {
 		_, isMountPoint := hdr.Winheaders[hdrMountPoint]
 		rp := winio.ReparsePoint{
@@ -337,7 +425,7 @@ func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (
 		bhdr := winio.BackupHeader{
 			Id:   winio.BackupAlternateData,
 			Size: ahdr.Size,
-			Name: ahdr.Name[len(hdr.Name)+1:] + ":$DATA",
+			Name: ahdr.Name[len(hdr.Name):] + ":$DATA",
 		}
 		err = bw.WriteHeader(&bhdr)
 		if err != nil {

vendor/github.com/Microsoft/go-winio/backuptar/tar_test.go

@@ -80,5 +80,5 @@ func TestRoundTrip(t *testing.T) {
 		t.Errorf("got %#v, expected %#v", *bi, *bi2)
 	}
 
-	ensurePresent(t, hdr.Winheaders, "fileattr", "sd")
+	ensurePresent(t, hdr.Winheaders, "fileattr", "rawsd")
 }

vendor/github.com/Microsoft/go-winio/ea.go

+package winio
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+)
+
+type fileFullEaInformation struct {
+	NextEntryOffset uint32
+	Flags           uint8
+	NameLength      uint8
+	ValueLength     uint16
+}
+
+var (
+	fileFullEaInformationSize = binary.Size(&fileFullEaInformation{})
+
+	errInvalidEaBuffer = errors.New("invalid extended attribute buffer")
+	errEaNameTooLarge  = errors.New("extended attribute name too large")
+	errEaValueTooLarge = errors.New("extended attribute value too large")
+)
+
+// ExtendedAttribute represents a single Windows EA.
+type ExtendedAttribute struct {
+	Name  string
+	Value []byte
+	Flags uint8
+}
+
+func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
+	var info fileFullEaInformation
+	err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
+	if err != nil {
+		err = errInvalidEaBuffer
+		return
+	}
+
+	nameOffset := fileFullEaInformationSize
+	nameLen := int(info.NameLength)
+	valueOffset := nameOffset + int(info.NameLength) + 1
+	valueLen := int(info.ValueLength)
+	nextOffset := int(info.NextEntryOffset)
+	if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
+		err = errInvalidEaBuffer
+		return
+	}
+
+	ea.Name = string(b[nameOffset : nameOffset+nameLen])
+	ea.Value = b[valueOffset : valueOffset+valueLen]
+	ea.Flags = info.Flags
+	if info.NextEntryOffset != 0 {
+		nb = b[info.NextEntryOffset:]
+	}
+	return
+}
+
+// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
+// buffer retrieved from BackupRead, ZwQueryEaFile, etc.
+func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
+	for len(b) != 0 {
+		ea, nb, err := parseEa(b)
+		if err != nil {
+			return nil, err
+		}
+
+		eas = append(eas, ea)
+		b = nb
+	}
+	return
+}
+
+func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {
+	if int(uint8(len(ea.Name))) != len(ea.Name) {
+		return errEaNameTooLarge
+	}
+	if int(uint16(len(ea.Value))) != len(ea.Value) {
+		return errEaValueTooLarge
+	}
+	entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value))
+	withPadding := (entrySize + 3) &^ 3
+	nextOffset := uint32(0)
+	if !last {
+		nextOffset = withPadding
+	}
+	info := fileFullEaInformation{
+		NextEntryOffset: nextOffset,
+		Flags:           ea.Flags,
+		NameLength:      uint8(len(ea.Name)),
+		ValueLength:     uint16(len(ea.Value)),
+	}
+
+	err := binary.Write(buf, binary.LittleEndian, &info)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write([]byte(ea.Name))
+	if err != nil {
+		return err
+	}
+
+	err = buf.WriteByte(0)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write(ea.Value)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize])
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION
+// buffer for use with BackupWrite, ZwSetEaFile, etc.
+func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) {
+	var buf bytes.Buffer
+	for i := range eas {
+		last := false
+		if i == len(eas)-1 {
+			last = true
+		}
+
+		err := writeEa(&buf, &eas[i], last)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return buf.Bytes(), nil
+}

vendor/github.com/Microsoft/go-winio/ea_test.go

+package winio
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"syscall"
+	"testing"
+	"unsafe"
+)
+
+var (
+	testEas = []ExtendedAttribute{
+		{Name: "foo", Value: []byte("bar")},
+		{Name: "fizz", Value: []byte("buzz")},
+	}
+
+	testEasEncoded   = []byte{16, 0, 0, 0, 0, 3, 3, 0, 102, 111, 111, 0, 98, 97, 114, 0, 0, 0, 0, 0, 0, 4, 4, 0, 102, 105, 122, 122, 0, 98, 117, 122, 122, 0, 0, 0}
+	testEasNotPadded = testEasEncoded[0 : len(testEasEncoded)-3]
+	testEasTruncated = testEasEncoded[0:20]
+)
+
+func Test_RoundTripEas(t *testing.T) {
+	b, err := EncodeExtendedAttributes(testEas)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(testEasEncoded, b) {
+		t.Fatalf("encoded mismatch %v %v", testEasEncoded, b)
+	}
+	eas, err := DecodeExtendedAttributes(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(testEas, eas) {
+		t.Fatalf("mismatch %+v %+v", testEas, eas)
+	}
+}
+
+func Test_EasDontNeedPaddingAtEnd(t *testing.T) {
+	eas, err := DecodeExtendedAttributes(testEasNotPadded)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(testEas, eas) {
+		t.Fatalf("mismatch %+v %+v", testEas, eas)
+	}
+}
+
+func Test_TruncatedEasFailCorrectly(t *testing.T) {
+	_, err := DecodeExtendedAttributes(testEasTruncated)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+}
+
+func Test_NilEasEncodeAndDecodeAsNil(t *testing.T) {
+	b, err := EncodeExtendedAttributes(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(b) != 0 {
+		t.Fatal("expected empty")
+	}
+	eas, err := DecodeExtendedAttributes(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(eas) != 0 {
+		t.Fatal("expected empty")
+	}
+}
+
+// Test_SetFileEa makes sure that the test buffer is actually parsable by NtSetEaFile.
+func Test_SetFileEa(t *testing.T) {
+	f, err := ioutil.TempFile("", "winio")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(f.Name())
+	defer f.Close()
+	ntdll := syscall.MustLoadDLL("ntdll.dll")
+	ntSetEaFile := ntdll.MustFindProc("NtSetEaFile")
+	var iosb [2]uintptr
+	r, _, _ := ntSetEaFile.Call(f.Fd(), uintptr(unsafe.Pointer(&iosb[0])), uintptr(unsafe.Pointer(&testEasEncoded[0])), uintptr(len(testEasEncoded)))
+	if r != 0 {
+		t.Fatalf("NtSetEaFile failed with %08x", r)
+	}
+}

vendor/github.com/Microsoft/go-winio/file.go

+// +build windows
+
 package winio
 
 import (
@@ -5,6 +7,7 @@ import (
 	"io"
 	"runtime"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 )
@@ -13,7 +16,19 @@ import (
 //sys createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) = CreateIoCompletionPort
 //sys getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) = GetQueuedCompletionStatus
 //sys setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) = SetFileCompletionNotificationModes
-//sys timeBeginPeriod(period uint32) (n int32) = winmm.timeBeginPeriod
+
+type atomicBool int32
+
+func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 }
+func (b *atomicBool) setFalse()   { atomic.StoreInt32((*int32)(b), 0) }
+func (b *atomicBool) setTrue()    { atomic.StoreInt32((*int32)(b), 1) }
+func (b *atomicBool) swap(new bool) bool {
+	var newInt int32
+	if new {
+		newInt = 1
+	}
+	return atomic.SwapInt32((*int32)(b), newInt) == 1
+}
 
 const (
 	cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1
@@ -31,6 +46,8 @@ func (e *timeoutError) Error() string   { return "i/o timeout" }
 func (e *timeoutError) Timeout() bool   { return true }
 func (e *timeoutError) Temporary() bool { return true }
 
+type timeoutChan chan struct{}
+
 var ioInitOnce sync.Once
 var ioCompletionPort syscall.Handle
 
@@ -60,9 +77,18 @@ func initIo() {
 type win32File struct {
 	handle        syscall.Handle
 	wg            sync.WaitGroup
-	closing       bool
-	readDeadline  time.Time
-	writeDeadline time.Time
+	wgLock        sync.RWMutex
+	closing       atomicBool
+	readDeadline  deadlineHandler
+	writeDeadline deadlineHandler
+}
+
+type deadlineHandler struct {
+	setLock     sync.Mutex
+	channel     timeoutChan
+	channelLock sync.RWMutex
+	timer       *time.Timer
+	timedout    atomicBool
 }
 
 // makeWin32File makes a new win32File from an existing file handle
@@ -77,7 +103,8 @@ func makeWin32File(h syscall.Handle) (*win32File, error) {
 	if err != nil {
 		return nil, err
 	}
-	runtime.SetFinalizer(f, (*win32File).closeHandle)
+	f.readDeadline.channel = make(timeoutChan)
+	f.writeDeadline.channel = make(timeoutChan)
 	return f, nil