diff --git a/proxy/router.go b/proxy/router.go index b87a361..a6663df 100644 --- a/proxy/router.go +++ b/proxy/router.go @@ -14,6 +14,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/goproxyio/goproxy/renameio" ) const ListExpire = 5 * time.Minute @@ -66,19 +68,30 @@ func NewRouter(srv *Server, opts *RouterOptions) *Router { if r.StatusCode == http.StatusOK { var buf []byte if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") { - if gr, err := gzip.NewReader(r.Body); err == nil { - defer gr.Close() - buf, _ = ioutil.ReadAll(gr) - r.Header.Del("Content-Encoding") + gr, err := gzip.NewReader(r.Body) + if err != nil { + return err } + defer gr.Close() + buf, err = ioutil.ReadAll(gr) + if err != nil { + return err + } + r.Header.Del("Content-Encoding") } else { - buf, _ = ioutil.ReadAll(r.Body) + buf, err = ioutil.ReadAll(r.Body) + if err != nil { + return err + } } r.Body = ioutil.NopCloser(bytes.NewReader(buf)) if buf != nil { file := filepath.Join(opts.DownloadRoot, r.Request.URL.Path) os.MkdirAll(path.Dir(file), os.ModePerm) - ioutil.WriteFile(file, buf, 0666) + err = renameio.WriteFile(file, buf, 0666) + if err != nil { + return err + } } } return nil diff --git a/renameio/renameio.go b/renameio/renameio.go new file mode 100644 index 0000000..5d08f6b --- /dev/null +++ b/renameio/renameio.go @@ -0,0 +1,93 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package renameio writes files atomically by renaming temporary files. +package renameio + +import ( + "bytes" + "io" + "math/rand" + "os" + "path/filepath" + "strconv" + + "github.com/goproxyio/goproxy/robustio" +) + +const patternSuffix = ".tmp" + +// Pattern returns a glob pattern that matches the unrenamed temporary files +// created when writing to filename. +func Pattern(filename string) string { + return filepath.Join(filepath.Dir(filename), filepath.Base(filename)+patternSuffix) +} + +// WriteFile is like ioutil.WriteFile, but first writes data to an arbitrary +// file in the same directory as filename, then renames it atomically to the +// final name. +// +// That ensures that the final location, if it exists, is always a complete file. +func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { + return WriteToFile(filename, bytes.NewReader(data), perm) +} + +// WriteToFile is a variant of WriteFile that accepts the data as an io.Reader +// instead of a slice. +func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) { + f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm) + if err != nil { + return err + } + defer func() { + // Only call os.Remove on f.Name() if we failed to rename it: otherwise, + // some other process may have created a new file with the same name after + // that. + if err != nil { + f.Close() + os.Remove(f.Name()) + } + }() + + if _, err := io.Copy(f, data); err != nil { + return err + } + // Sync the file before renaming it: otherwise, after a crash the reader may + // observe a 0-length file instead of the actual contents. + // See https://golang.org/issue/22397#issuecomment-380831736. + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + + return robustio.Rename(f.Name(), filename) +} + +// ReadFile is like ioutil.ReadFile, but on Windows retries spurious errors that +// may occur if the file is concurrently replaced. +// +// Errors are classified heuristically and retries are bounded, so even this +// function may occasionally return a spurious error on Windows. +// If so, the error will likely wrap one of: +// - syscall.ERROR_ACCESS_DENIED +// - syscall.ERROR_FILE_NOT_FOUND +// - internal/syscall/windows.ERROR_SHARING_VIOLATION +func ReadFile(filename string) ([]byte, error) { + return robustio.ReadFile(filename) +} + +// tempFile creates a new temporary file with given permission bits. +func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) { + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix) + f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) + if os.IsExist(err) { + continue + } + break + } + return +} diff --git a/renameio/renameio_test.go b/renameio/renameio_test.go new file mode 100644 index 0000000..172df35 --- /dev/null +++ b/renameio/renameio_test.go @@ -0,0 +1,145 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9 + +package renameio + +import ( + "encoding/binary" + "errors" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/goproxyio/goproxy/robustio" +) + +func TestConcurrentReadsAndWrites(t *testing.T) { + dir, err := ioutil.TempDir("", "renameio") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + path := filepath.Join(dir, "blob.bin") + + const chunkWords = 8 << 10 + buf := make([]byte, 2*chunkWords*8) + for i := uint64(0); i < 2*chunkWords; i++ { + binary.LittleEndian.PutUint64(buf[i*8:], i) + } + + var attempts int64 = 128 + if !testing.Short() { + attempts *= 16 + } + const parallel = 32 + + var sem = make(chan bool, parallel) + + var ( + writeSuccesses, readSuccesses int64 // atomic + writeErrnoSeen, readErrnoSeen sync.Map + ) + + for n := attempts; n > 0; n-- { + sem <- true + go func() { + defer func() { <-sem }() + + time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) + offset := rand.Intn(chunkWords) + chunk := buf[offset*8 : (offset+chunkWords)*8] + if err := WriteFile(path, chunk, 0666); err == nil { + atomic.AddInt64(&writeSuccesses, 1) + } else if robustio.IsEphemeralError(err) { + var ( + errno syscall.Errno + dup bool + ) + if errors.As(err, &errno) { + _, dup = writeErrnoSeen.LoadOrStore(errno, true) + } + if !dup { + t.Logf("ephemeral error: %v", err) + } + } else { + t.Errorf("unexpected error: %v", err) + } + + time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) + data, err := ReadFile(path) + if err == nil { + atomic.AddInt64(&readSuccesses, 1) + } else if robustio.IsEphemeralError(err) { + var ( + errno syscall.Errno + dup bool + ) + if errors.As(err, &errno) { + _, dup = readErrnoSeen.LoadOrStore(errno, true) + } + if !dup { + t.Logf("ephemeral error: %v", err) + } + return + } else { + t.Errorf("unexpected error: %v", err) + return + } + + if len(data) != 8*chunkWords { + t.Errorf("read %d bytes, but each write is a %d-byte file", len(data), 8*chunkWords) + return + } + + u := binary.LittleEndian.Uint64(data) + for i := 1; i < chunkWords; i++ { + next := binary.LittleEndian.Uint64(data[i*8:]) + if next != u+1 { + t.Errorf("wrote sequential integers, but read integer out of sequence at offset %d", i) + return + } + u = next + } + }() + } + + for n := parallel; n > 0; n-- { + sem <- true + } + + var minWriteSuccesses int64 = attempts + if runtime.GOOS == "windows" { + // Windows produces frequent "Access is denied" errors under heavy rename load. + // As long as those are the only errors and *some* of the writes succeed, we're happy. + minWriteSuccesses = attempts / 4 + } + + if writeSuccesses < minWriteSuccesses { + t.Errorf("%d (of %d) writes succeeded; want ≥ %d", writeSuccesses, attempts, minWriteSuccesses) + } else { + t.Logf("%d (of %d) writes succeeded (ok: ≥ %d)", writeSuccesses, attempts, minWriteSuccesses) + } + + var minReadSuccesses int64 = attempts + if runtime.GOOS == "windows" { + // Windows produces frequent "Access is denied" errors under heavy rename load. + // As long as those are the only errors and *some* of the writes succeed, we're happy. + minReadSuccesses = attempts / 4 + } + + if readSuccesses < minReadSuccesses { + t.Errorf("%d (of %d) reads succeeded; want ≥ %d", readSuccesses, attempts, minReadSuccesses) + } else { + t.Logf("%d (of %d) reads succeeded (ok: ≥ %d)", readSuccesses, attempts, minReadSuccesses) + } +} diff --git a/renameio/umask_test.go b/renameio/umask_test.go new file mode 100644 index 0000000..1d5e594 --- /dev/null +++ b/renameio/umask_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !nacl,!plan9,!windows,!js + +package renameio + +import ( + "io/ioutil" + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestWriteFileModeAppliesUmask(t *testing.T) { + dir, err := ioutil.TempDir("", "renameio") + if err != nil { + t.Fatalf("Failed to create temporary directory: %v", err) + } + defer os.RemoveAll(dir) + + const mode = 0644 + const umask = 0007 + defer syscall.Umask(syscall.Umask(umask)) + + file := filepath.Join(dir, "testWrite") + err = WriteFile(file, []byte("go-build"), mode) + if err != nil { + t.Fatalf("Failed to write file: %v", err) + } + + fi, err := os.Stat(file) + if err != nil { + t.Fatalf("Stat %q (looking for mode %#o): %s", file, mode, err) + } + + if fi.Mode()&os.ModePerm != 0640 { + t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&os.ModePerm, 0640) + } +} diff --git a/robustio/robustio.go b/robustio/robustio.go new file mode 100644 index 0000000..76e47ad --- /dev/null +++ b/robustio/robustio.go @@ -0,0 +1,53 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package robustio wraps I/O functions that are prone to failure on Windows, +// transparently retrying errors up to an arbitrary timeout. +// +// Errors are classified heuristically and retries are bounded, so the functions +// in this package do not completely eliminate spurious errors. However, they do +// significantly reduce the rate of failure in practice. +// +// If so, the error will likely wrap one of: +// The functions in this package do not completely eliminate spurious errors, +// but substantially reduce their rate of occurrence in practice. +package robustio + +// Rename is like os.Rename, but on Windows retries errors that may occur if the +// file is concurrently read or overwritten. +// +// (See golang.org/issue/31247 and golang.org/issue/32188.) +func Rename(oldpath, newpath string) error { + return rename(oldpath, newpath) +} + +// ReadFile is like ioutil.ReadFile, but on Windows retries errors that may +// occur if the file is concurrently replaced. +// +// (See golang.org/issue/31247 and golang.org/issue/32188.) +func ReadFile(filename string) ([]byte, error) { + return readFile(filename) +} + +// RemoveAll is like os.RemoveAll, but on Windows retries errors that may occur +// if an executable file in the directory has recently been executed. +// +// (See golang.org/issue/19491.) +func RemoveAll(path string) error { + return removeAll(path) +} + +// IsEphemeralError reports whether err is one of the errors that the functions +// in this package attempt to mitigate. +// +// Errors considered ephemeral include: +// - syscall.ERROR_ACCESS_DENIED +// - syscall.ERROR_FILE_NOT_FOUND +// - internal/syscall/windows.ERROR_SHARING_VIOLATION +// +// This set may be expanded in the future; programs must not rely on the +// non-ephemerality of any given error. +func IsEphemeralError(err error) bool { + return isEphemeralError(err) +} diff --git a/robustio/robustio_other.go b/robustio/robustio_other.go new file mode 100644 index 0000000..56e6ad6 --- /dev/null +++ b/robustio/robustio_other.go @@ -0,0 +1,28 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !windows + +package robustio + +import ( + "io/ioutil" + "os" +) + +func rename(oldpath, newpath string) error { + return os.Rename(oldpath, newpath) +} + +func readFile(filename string) ([]byte, error) { + return ioutil.ReadFile(filename) +} + +func removeAll(path string) error { + return os.RemoveAll(path) +} + +func isEphemeralError(err error) bool { + return false +} diff --git a/robustio/robustio_windows.go b/robustio/robustio_windows.go new file mode 100644 index 0000000..a3d94e5 --- /dev/null +++ b/robustio/robustio_windows.go @@ -0,0 +1,105 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package robustio + +import ( + "errors" + "internal/syscall/windows" + "io/ioutil" + "math/rand" + "os" + "syscall" + "time" +) + +const arbitraryTimeout = 500 * time.Millisecond + +// retry retries ephemeral errors from f up to an arbitrary timeout +// to work around spurious filesystem errors on Windows +func retry(f func() (err error, mayRetry bool)) error { + var ( + bestErr error + lowestErrno syscall.Errno + start time.Time + nextSleep time.Duration = 1 * time.Millisecond + ) + for { + err, mayRetry := f() + if err == nil || !mayRetry { + return err + } + + var errno syscall.Errno + if errors.As(err, &errno) && (lowestErrno == 0 || errno < lowestErrno) { + bestErr = err + lowestErrno = errno + } else if bestErr == nil { + bestErr = err + } + + if start.IsZero() { + start = time.Now() + } else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout { + break + } + time.Sleep(nextSleep) + nextSleep += time.Duration(rand.Int63n(int64(nextSleep))) + } + + return bestErr +} + +// rename is like os.Rename, but retries ephemeral errors. +// +// It wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with +// MOVEFILE_REPLACE_EXISTING. +// +// Windows also provides a different system call, ReplaceFile, +// that provides similar semantics, but perhaps preserves more metadata. (The +// documentation on the differences between the two is very sparse.) +// +// Empirical error rates with MoveFileEx are lower under modest concurrency, so +// for now we're sticking with what the os package already provides. +func rename(oldpath, newpath string) (err error) { + return retry(func() (err error, mayRetry bool) { + err = os.Rename(oldpath, newpath) + return err, isEphemeralError(err) + }) +} + +// readFile is like ioutil.ReadFile, but retries ephemeral errors. +func readFile(filename string) ([]byte, error) { + var b []byte + err := retry(func() (err error, mayRetry bool) { + b, err = ioutil.ReadFile(filename) + + // Unlike in rename, we do not retry ERROR_FILE_NOT_FOUND here: it can occur + // as a spurious error, but the file may also genuinely not exist, so the + // increase in robustness is probably not worth the extra latency. + return err, isEphemeralError(err) && !errors.Is(err, syscall.ERROR_FILE_NOT_FOUND) + }) + return b, err +} + +func removeAll(path string) error { + return retry(func() (err error, mayRetry bool) { + err = os.RemoveAll(path) + return err, isEphemeralError(err) + }) +} + +// isEphemeralError returns true if err may be resolved by waiting. +func isEphemeralError(err error) bool { + var errno syscall.Errno + if errors.As(err, &errno) { + switch errno { + case syscall.ERROR_ACCESS_DENIED, + syscall.ERROR_FILE_NOT_FOUND, + windows.ERROR_SHARING_VIOLATION: + return true + } + } + return false +}