renameio.WriteFile writes files atomically by renaming temporary files

proxy/router.go

@@ -14,6 +14,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/goproxyio/goproxy/renameio"
 )
 
 const ListExpire = 5 * time.Minute
@@ -66,19 +68,30 @@ func NewRouter(srv *Server, opts *RouterOptions) *Router {
 			if r.StatusCode == http.StatusOK {
 				var buf []byte
 				if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") {
-					if gr, err := gzip.NewReader(r.Body); err == nil {
+					gr, err := gzip.NewReader(r.Body)
-						defer gr.Close()
+					if err != nil {
-						buf, _ = ioutil.ReadAll(gr)
+						return err
-						r.Header.Del("Content-Encoding")
 					}
+					defer gr.Close()
+					buf, err = ioutil.ReadAll(gr)
+					if err != nil {
+						return err
+					}
+					r.Header.Del("Content-Encoding")
 				} else {
-					buf, _ = ioutil.ReadAll(r.Body)
+					buf, err = ioutil.ReadAll(r.Body)
+					if err != nil {
+						return err
+					}
 				}
 				r.Body = ioutil.NopCloser(bytes.NewReader(buf))
 				if buf != nil {
 					file := filepath.Join(opts.DownloadRoot, r.Request.URL.Path)
 					os.MkdirAll(path.Dir(file), os.ModePerm)
-					ioutil.WriteFile(file, buf, 0666)
+					err = renameio.WriteFile(file, buf, 0666)
+					if err != nil {
+						return err
+					}
 				}
 			}
 			return nil

renameio/renameio.go

@@ -0,0 +1,93 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package renameio writes files atomically by renaming temporary files.
+package renameio
+
+import (
+	"bytes"
+	"io"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/goproxyio/goproxy/robustio"
+)
+
+const patternSuffix = ".tmp"
+
+// Pattern returns a glob pattern that matches the unrenamed temporary files
+// created when writing to filename.
+func Pattern(filename string) string {
+	return filepath.Join(filepath.Dir(filename), filepath.Base(filename)+patternSuffix)
+}
+
+// WriteFile is like ioutil.WriteFile, but first writes data to an arbitrary
+// file in the same directory as filename, then renames it atomically to the
+// final name.
+//
+// That ensures that the final location, if it exists, is always a complete file.
+func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
+	return WriteToFile(filename, bytes.NewReader(data), perm)
+}
+
+// WriteToFile is a variant of WriteFile that accepts the data as an io.Reader
+// instead of a slice.
+func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) {
+	f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		// Only call os.Remove on f.Name() if we failed to rename it: otherwise,
+		// some other process may have created a new file with the same name after
+		// that.
+		if err != nil {
+			f.Close()
+			os.Remove(f.Name())
+		}
+	}()
+
+	if _, err := io.Copy(f, data); err != nil {
+		return err
+	}
+	// Sync the file before renaming it: otherwise, after a crash the reader may
+	// observe a 0-length file instead of the actual contents.
+	// See https://golang.org/issue/22397#issuecomment-380831736.
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+
+	return robustio.Rename(f.Name(), filename)
+}
+
+// ReadFile is like ioutil.ReadFile, but on Windows retries spurious errors that
+// may occur if the file is concurrently replaced.
+//
+// Errors are classified heuristically and retries are bounded, so even this
+// function may occasionally return a spurious error on Windows.
+// If so, the error will likely wrap one of:
+// 	- syscall.ERROR_ACCESS_DENIED
+// 	- syscall.ERROR_FILE_NOT_FOUND
+// 	- internal/syscall/windows.ERROR_SHARING_VIOLATION
+func ReadFile(filename string) ([]byte, error) {
+	return robustio.ReadFile(filename)
+}
+
+// tempFile creates a new temporary file with given permission bits.
+func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) {
+	for i := 0; i < 10000; i++ {
+		name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix)
+		f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
+		if os.IsExist(err) {
+			continue
+		}
+		break
+	}
+	return
+}

renameio/renameio_test.go

@@ -0,0 +1,145 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !plan9
+
+package renameio
+
+import (
+	"encoding/binary"
+	"errors"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/goproxyio/goproxy/robustio"
+)
+
+func TestConcurrentReadsAndWrites(t *testing.T) {
+	dir, err := ioutil.TempDir("", "renameio")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	path := filepath.Join(dir, "blob.bin")
+
+	const chunkWords = 8 << 10
+	buf := make([]byte, 2*chunkWords*8)
+	for i := uint64(0); i < 2*chunkWords; i++ {
+		binary.LittleEndian.PutUint64(buf[i*8:], i)
+	}
+
+	var attempts int64 = 128
+	if !testing.Short() {
+		attempts *= 16
+	}
+	const parallel = 32
+
+	var sem = make(chan bool, parallel)
+
+	var (
+		writeSuccesses, readSuccesses int64 // atomic
+		writeErrnoSeen, readErrnoSeen sync.Map
+	)
+
+	for n := attempts; n > 0; n-- {
+		sem <- true
+		go func() {
+			defer func() { <-sem }()
+
+			time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond)
+			offset := rand.Intn(chunkWords)
+			chunk := buf[offset*8 : (offset+chunkWords)*8]
+			if err := WriteFile(path, chunk, 0666); err == nil {
+				atomic.AddInt64(&writeSuccesses, 1)
+			} else if robustio.IsEphemeralError(err) {
+				var (
+					errno syscall.Errno
+					dup   bool
+				)
+				if errors.As(err, &errno) {
+					_, dup = writeErrnoSeen.LoadOrStore(errno, true)
+				}
+				if !dup {
+					t.Logf("ephemeral error: %v", err)
+				}
+			} else {
+				t.Errorf("unexpected error: %v", err)
+			}
+
+			time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond)
+			data, err := ReadFile(path)
+			if err == nil {
+				atomic.AddInt64(&readSuccesses, 1)
+			} else if robustio.IsEphemeralError(err) {
+				var (
+					errno syscall.Errno
+					dup   bool
+				)
+				if errors.As(err, &errno) {
+					_, dup = readErrnoSeen.LoadOrStore(errno, true)
+				}
+				if !dup {
+					t.Logf("ephemeral error: %v", err)
+				}
+				return
+			} else {
+				t.Errorf("unexpected error: %v", err)
+				return
+			}
+
+			if len(data) != 8*chunkWords {
+				t.Errorf("read %d bytes, but each write is a %d-byte file", len(data), 8*chunkWords)
+				return
+			}
+
+			u := binary.LittleEndian.Uint64(data)
+			for i := 1; i < chunkWords; i++ {
+				next := binary.LittleEndian.Uint64(data[i*8:])
+				if next != u+1 {
+					t.Errorf("wrote sequential integers, but read integer out of sequence at offset %d", i)
+					return
+				}
+				u = next
+			}
+		}()
+	}
+
+	for n := parallel; n > 0; n-- {
+		sem <- true
+	}
+
+	var minWriteSuccesses int64 = attempts
+	if runtime.GOOS == "windows" {
+		// Windows produces frequent "Access is denied" errors under heavy rename load.
+		// As long as those are the only errors and *some* of the writes succeed, we're happy.
+		minWriteSuccesses = attempts / 4
+	}
+
+	if writeSuccesses < minWriteSuccesses {
+		t.Errorf("%d (of %d) writes succeeded; want ≥ %d", writeSuccesses, attempts, minWriteSuccesses)
+	} else {
+		t.Logf("%d (of %d) writes succeeded (ok: ≥ %d)", writeSuccesses, attempts, minWriteSuccesses)
+	}
+
+	var minReadSuccesses int64 = attempts
+	if runtime.GOOS == "windows" {
+		// Windows produces frequent "Access is denied" errors under heavy rename load.
+		// As long as those are the only errors and *some* of the writes succeed, we're happy.
+		minReadSuccesses = attempts / 4
+	}
+
+	if readSuccesses < minReadSuccesses {
+		t.Errorf("%d (of %d) reads succeeded; want ≥ %d", readSuccesses, attempts, minReadSuccesses)
+	} else {
+		t.Logf("%d (of %d) reads succeeded (ok: ≥ %d)", readSuccesses, attempts, minReadSuccesses)
+	}
+}

renameio/umask_test.go

@@ -0,0 +1,42 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !nacl,!plan9,!windows,!js
+
+package renameio
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+)
+
+func TestWriteFileModeAppliesUmask(t *testing.T) {
+	dir, err := ioutil.TempDir("", "renameio")
+	if err != nil {
+		t.Fatalf("Failed to create temporary directory: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	const mode = 0644
+	const umask = 0007
+	defer syscall.Umask(syscall.Umask(umask))
+
+	file := filepath.Join(dir, "testWrite")
+	err = WriteFile(file, []byte("go-build"), mode)
+	if err != nil {
+		t.Fatalf("Failed to write file: %v", err)
+	}
+
+	fi, err := os.Stat(file)
+	if err != nil {
+		t.Fatalf("Stat %q (looking for mode %#o): %s", file, mode, err)
+	}
+
+	if fi.Mode()&os.ModePerm != 0640 {
+		t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&os.ModePerm, 0640)
+	}
+}

robustio/robustio.go

@@ -0,0 +1,53 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package robustio wraps I/O functions that are prone to failure on Windows,
+// transparently retrying errors up to an arbitrary timeout.
+//
+// Errors are classified heuristically and retries are bounded, so the functions
+// in this package do not completely eliminate spurious errors. However, they do
+// significantly reduce the rate of failure in practice.
+//
+// If so, the error will likely wrap one of:
+// The functions in this package do not completely eliminate spurious errors,
+// but substantially reduce their rate of occurrence in practice.
+package robustio
+
+// Rename is like os.Rename, but on Windows retries errors that may occur if the
+// file is concurrently read or overwritten.
+//
+// (See golang.org/issue/31247 and golang.org/issue/32188.)
+func Rename(oldpath, newpath string) error {
+	return rename(oldpath, newpath)
+}
+
+// ReadFile is like ioutil.ReadFile, but on Windows retries errors that may
+// occur if the file is concurrently replaced.
+//
+// (See golang.org/issue/31247 and golang.org/issue/32188.)
+func ReadFile(filename string) ([]byte, error) {
+	return readFile(filename)
+}
+
+// RemoveAll is like os.RemoveAll, but on Windows retries errors that may occur
+// if an executable file in the directory has recently been executed.
+//
+// (See golang.org/issue/19491.)
+func RemoveAll(path string) error {
+	return removeAll(path)
+}
+
+// IsEphemeralError reports whether err is one of the errors that the functions
+// in this package attempt to mitigate.
+//
+// Errors considered ephemeral include:
+// 	- syscall.ERROR_ACCESS_DENIED
+// 	- syscall.ERROR_FILE_NOT_FOUND
+// 	- internal/syscall/windows.ERROR_SHARING_VIOLATION
+//
+// This set may be expanded in the future; programs must not rely on the
+// non-ephemerality of any given error.
+func IsEphemeralError(err error) bool {
+	return isEphemeralError(err)
+}

robustio/robustio_other.go

@@ -0,0 +1,28 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !windows
+
+package robustio
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+func rename(oldpath, newpath string) error {
+	return os.Rename(oldpath, newpath)
+}
+
+func readFile(filename string) ([]byte, error) {
+	return ioutil.ReadFile(filename)
+}
+
+func removeAll(path string) error {
+	return os.RemoveAll(path)
+}
+
+func isEphemeralError(err error) bool {
+	return false
+}

robustio/robustio_windows.go

@@ -0,0 +1,105 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package robustio
+
+import (
+	"errors"
+	"internal/syscall/windows"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"syscall"
+	"time"
+)
+
+const arbitraryTimeout = 500 * time.Millisecond
+
+// retry retries ephemeral errors from f up to an arbitrary timeout
+// to work around spurious filesystem errors on Windows
+func retry(f func() (err error, mayRetry bool)) error {
+	var (
+		bestErr     error
+		lowestErrno syscall.Errno
+		start       time.Time
+		nextSleep   time.Duration = 1 * time.Millisecond
+	)
+	for {
+		err, mayRetry := f()
+		if err == nil || !mayRetry {
+			return err
+		}
+
+		var errno syscall.Errno
+		if errors.As(err, &errno) && (lowestErrno == 0 || errno < lowestErrno) {
+			bestErr = err
+			lowestErrno = errno
+		} else if bestErr == nil {
+			bestErr = err
+		}
+
+		if start.IsZero() {
+			start = time.Now()
+		} else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout {
+			break
+		}
+		time.Sleep(nextSleep)
+		nextSleep += time.Duration(rand.Int63n(int64(nextSleep)))
+	}
+
+	return bestErr
+}
+
+// rename is like os.Rename, but retries ephemeral errors.
+//
+// It wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with
+// MOVEFILE_REPLACE_EXISTING.
+//
+// Windows also provides a different system call, ReplaceFile,
+// that provides similar semantics, but perhaps preserves more metadata. (The
+// documentation on the differences between the two is very sparse.)
+//
+// Empirical error rates with MoveFileEx are lower under modest concurrency, so
+// for now we're sticking with what the os package already provides.
+func rename(oldpath, newpath string) (err error) {
+	return retry(func() (err error, mayRetry bool) {
+		err = os.Rename(oldpath, newpath)
+		return err, isEphemeralError(err)
+	})
+}
+
+// readFile is like ioutil.ReadFile, but retries ephemeral errors.
+func readFile(filename string) ([]byte, error) {
+	var b []byte
+	err := retry(func() (err error, mayRetry bool) {
+		b, err = ioutil.ReadFile(filename)
+
+		// Unlike in rename, we do not retry ERROR_FILE_NOT_FOUND here: it can occur
+		// as a spurious error, but the file may also genuinely not exist, so the
+		// increase in robustness is probably not worth the extra latency.
+		return err, isEphemeralError(err) && !errors.Is(err, syscall.ERROR_FILE_NOT_FOUND)
+	})
+	return b, err
+}
+
+func removeAll(path string) error {
+	return retry(func() (err error, mayRetry bool) {
+		err = os.RemoveAll(path)
+		return err, isEphemeralError(err)
+	})
+}
+
+// isEphemeralError returns true if err may be resolved by waiting.
+func isEphemeralError(err error) bool {
+	var errno syscall.Errno
+	if errors.As(err, &errno) {
+		switch errno {
+		case syscall.ERROR_ACCESS_DENIED,
+			syscall.ERROR_FILE_NOT_FOUND,
+			windows.ERROR_SHARING_VIOLATION:
+			return true
+		}
+	}
+	return false
+}