mirror of https://github.com/goproxyio/goproxy
renameio.WriteFile writes files atomically by renaming temporary files
parent
389bbbec05
commit
2b89997884
|
@ -14,6 +14,8 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/goproxyio/goproxy/renameio"
|
||||
)
|
||||
|
||||
const ListExpire = 5 * time.Minute
|
||||
|
@ -66,19 +68,30 @@ func NewRouter(srv *Server, opts *RouterOptions) *Router {
|
|||
if r.StatusCode == http.StatusOK {
|
||||
var buf []byte
|
||||
if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") {
|
||||
if gr, err := gzip.NewReader(r.Body); err == nil {
|
||||
defer gr.Close()
|
||||
buf, _ = ioutil.ReadAll(gr)
|
||||
r.Header.Del("Content-Encoding")
|
||||
gr, err := gzip.NewReader(r.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gr.Close()
|
||||
buf, err = ioutil.ReadAll(gr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Header.Del("Content-Encoding")
|
||||
} else {
|
||||
buf, _ = ioutil.ReadAll(r.Body)
|
||||
buf, err = ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
r.Body = ioutil.NopCloser(bytes.NewReader(buf))
|
||||
if buf != nil {
|
||||
file := filepath.Join(opts.DownloadRoot, r.Request.URL.Path)
|
||||
os.MkdirAll(path.Dir(file), os.ModePerm)
|
||||
ioutil.WriteFile(file, buf, 0666)
|
||||
err = renameio.WriteFile(file, buf, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
|
|
@ -0,0 +1,93 @@
|
|||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package renameio writes files atomically by renaming temporary files.
|
||||
package renameio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/goproxyio/goproxy/robustio"
|
||||
)
|
||||
|
||||
const patternSuffix = ".tmp"
|
||||
|
||||
// Pattern returns a glob pattern that matches the unrenamed temporary files
|
||||
// created when writing to filename.
|
||||
func Pattern(filename string) string {
|
||||
return filepath.Join(filepath.Dir(filename), filepath.Base(filename)+patternSuffix)
|
||||
}
|
||||
|
||||
// WriteFile is like ioutil.WriteFile, but first writes data to an arbitrary
|
||||
// file in the same directory as filename, then renames it atomically to the
|
||||
// final name.
|
||||
//
|
||||
// That ensures that the final location, if it exists, is always a complete file.
|
||||
func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
|
||||
return WriteToFile(filename, bytes.NewReader(data), perm)
|
||||
}
|
||||
|
||||
// WriteToFile is a variant of WriteFile that accepts the data as an io.Reader
|
||||
// instead of a slice.
|
||||
func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) {
|
||||
f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
// Only call os.Remove on f.Name() if we failed to rename it: otherwise,
|
||||
// some other process may have created a new file with the same name after
|
||||
// that.
|
||||
if err != nil {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := io.Copy(f, data); err != nil {
|
||||
return err
|
||||
}
|
||||
// Sync the file before renaming it: otherwise, after a crash the reader may
|
||||
// observe a 0-length file instead of the actual contents.
|
||||
// See https://golang.org/issue/22397#issuecomment-380831736.
|
||||
if err := f.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return robustio.Rename(f.Name(), filename)
|
||||
}
|
||||
|
||||
// ReadFile is like ioutil.ReadFile, but on Windows retries spurious errors that
|
||||
// may occur if the file is concurrently replaced.
|
||||
//
|
||||
// Errors are classified heuristically and retries are bounded, so even this
|
||||
// function may occasionally return a spurious error on Windows.
|
||||
// If so, the error will likely wrap one of:
|
||||
// - syscall.ERROR_ACCESS_DENIED
|
||||
// - syscall.ERROR_FILE_NOT_FOUND
|
||||
// - internal/syscall/windows.ERROR_SHARING_VIOLATION
|
||||
func ReadFile(filename string) ([]byte, error) {
|
||||
return robustio.ReadFile(filename)
|
||||
}
|
||||
|
||||
// tempFile creates a new temporary file with given permission bits.
|
||||
func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) {
|
||||
for i := 0; i < 10000; i++ {
|
||||
name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix)
|
||||
f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
|
||||
if os.IsExist(err) {
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
return
|
||||
}
|
|
@ -0,0 +1,145 @@
|
|||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !plan9
|
||||
|
||||
package renameio
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/goproxyio/goproxy/robustio"
|
||||
)
|
||||
|
||||
func TestConcurrentReadsAndWrites(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "renameio")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
path := filepath.Join(dir, "blob.bin")
|
||||
|
||||
const chunkWords = 8 << 10
|
||||
buf := make([]byte, 2*chunkWords*8)
|
||||
for i := uint64(0); i < 2*chunkWords; i++ {
|
||||
binary.LittleEndian.PutUint64(buf[i*8:], i)
|
||||
}
|
||||
|
||||
var attempts int64 = 128
|
||||
if !testing.Short() {
|
||||
attempts *= 16
|
||||
}
|
||||
const parallel = 32
|
||||
|
||||
var sem = make(chan bool, parallel)
|
||||
|
||||
var (
|
||||
writeSuccesses, readSuccesses int64 // atomic
|
||||
writeErrnoSeen, readErrnoSeen sync.Map
|
||||
)
|
||||
|
||||
for n := attempts; n > 0; n-- {
|
||||
sem <- true
|
||||
go func() {
|
||||
defer func() { <-sem }()
|
||||
|
||||
time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond)
|
||||
offset := rand.Intn(chunkWords)
|
||||
chunk := buf[offset*8 : (offset+chunkWords)*8]
|
||||
if err := WriteFile(path, chunk, 0666); err == nil {
|
||||
atomic.AddInt64(&writeSuccesses, 1)
|
||||
} else if robustio.IsEphemeralError(err) {
|
||||
var (
|
||||
errno syscall.Errno
|
||||
dup bool
|
||||
)
|
||||
if errors.As(err, &errno) {
|
||||
_, dup = writeErrnoSeen.LoadOrStore(errno, true)
|
||||
}
|
||||
if !dup {
|
||||
t.Logf("ephemeral error: %v", err)
|
||||
}
|
||||
} else {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond)
|
||||
data, err := ReadFile(path)
|
||||
if err == nil {
|
||||
atomic.AddInt64(&readSuccesses, 1)
|
||||
} else if robustio.IsEphemeralError(err) {
|
||||
var (
|
||||
errno syscall.Errno
|
||||
dup bool
|
||||
)
|
||||
if errors.As(err, &errno) {
|
||||
_, dup = readErrnoSeen.LoadOrStore(errno, true)
|
||||
}
|
||||
if !dup {
|
||||
t.Logf("ephemeral error: %v", err)
|
||||
}
|
||||
return
|
||||
} else {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(data) != 8*chunkWords {
|
||||
t.Errorf("read %d bytes, but each write is a %d-byte file", len(data), 8*chunkWords)
|
||||
return
|
||||
}
|
||||
|
||||
u := binary.LittleEndian.Uint64(data)
|
||||
for i := 1; i < chunkWords; i++ {
|
||||
next := binary.LittleEndian.Uint64(data[i*8:])
|
||||
if next != u+1 {
|
||||
t.Errorf("wrote sequential integers, but read integer out of sequence at offset %d", i)
|
||||
return
|
||||
}
|
||||
u = next
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for n := parallel; n > 0; n-- {
|
||||
sem <- true
|
||||
}
|
||||
|
||||
var minWriteSuccesses int64 = attempts
|
||||
if runtime.GOOS == "windows" {
|
||||
// Windows produces frequent "Access is denied" errors under heavy rename load.
|
||||
// As long as those are the only errors and *some* of the writes succeed, we're happy.
|
||||
minWriteSuccesses = attempts / 4
|
||||
}
|
||||
|
||||
if writeSuccesses < minWriteSuccesses {
|
||||
t.Errorf("%d (of %d) writes succeeded; want ≥ %d", writeSuccesses, attempts, minWriteSuccesses)
|
||||
} else {
|
||||
t.Logf("%d (of %d) writes succeeded (ok: ≥ %d)", writeSuccesses, attempts, minWriteSuccesses)
|
||||
}
|
||||
|
||||
var minReadSuccesses int64 = attempts
|
||||
if runtime.GOOS == "windows" {
|
||||
// Windows produces frequent "Access is denied" errors under heavy rename load.
|
||||
// As long as those are the only errors and *some* of the writes succeed, we're happy.
|
||||
minReadSuccesses = attempts / 4
|
||||
}
|
||||
|
||||
if readSuccesses < minReadSuccesses {
|
||||
t.Errorf("%d (of %d) reads succeeded; want ≥ %d", readSuccesses, attempts, minReadSuccesses)
|
||||
} else {
|
||||
t.Logf("%d (of %d) reads succeeded (ok: ≥ %d)", readSuccesses, attempts, minReadSuccesses)
|
||||
}
|
||||
}
|
|
@ -0,0 +1,42 @@
|
|||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !nacl,!plan9,!windows,!js
|
||||
|
||||
package renameio
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWriteFileModeAppliesUmask(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "renameio")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temporary directory: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
const mode = 0644
|
||||
const umask = 0007
|
||||
defer syscall.Umask(syscall.Umask(umask))
|
||||
|
||||
file := filepath.Join(dir, "testWrite")
|
||||
err = WriteFile(file, []byte("go-build"), mode)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to write file: %v", err)
|
||||
}
|
||||
|
||||
fi, err := os.Stat(file)
|
||||
if err != nil {
|
||||
t.Fatalf("Stat %q (looking for mode %#o): %s", file, mode, err)
|
||||
}
|
||||
|
||||
if fi.Mode()&os.ModePerm != 0640 {
|
||||
t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&os.ModePerm, 0640)
|
||||
}
|
||||
}
|
|
@ -0,0 +1,53 @@
|
|||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package robustio wraps I/O functions that are prone to failure on Windows,
|
||||
// transparently retrying errors up to an arbitrary timeout.
|
||||
//
|
||||
// Errors are classified heuristically and retries are bounded, so the functions
|
||||
// in this package do not completely eliminate spurious errors. However, they do
|
||||
// significantly reduce the rate of failure in practice.
|
||||
//
|
||||
// If so, the error will likely wrap one of:
|
||||
// The functions in this package do not completely eliminate spurious errors,
|
||||
// but substantially reduce their rate of occurrence in practice.
|
||||
package robustio
|
||||
|
||||
// Rename is like os.Rename, but on Windows retries errors that may occur if the
|
||||
// file is concurrently read or overwritten.
|
||||
//
|
||||
// (See golang.org/issue/31247 and golang.org/issue/32188.)
|
||||
func Rename(oldpath, newpath string) error {
|
||||
return rename(oldpath, newpath)
|
||||
}
|
||||
|
||||
// ReadFile is like ioutil.ReadFile, but on Windows retries errors that may
|
||||
// occur if the file is concurrently replaced.
|
||||
//
|
||||
// (See golang.org/issue/31247 and golang.org/issue/32188.)
|
||||
func ReadFile(filename string) ([]byte, error) {
|
||||
return readFile(filename)
|
||||
}
|
||||
|
||||
// RemoveAll is like os.RemoveAll, but on Windows retries errors that may occur
|
||||
// if an executable file in the directory has recently been executed.
|
||||
//
|
||||
// (See golang.org/issue/19491.)
|
||||
func RemoveAll(path string) error {
|
||||
return removeAll(path)
|
||||
}
|
||||
|
||||
// IsEphemeralError reports whether err is one of the errors that the functions
|
||||
// in this package attempt to mitigate.
|
||||
//
|
||||
// Errors considered ephemeral include:
|
||||
// - syscall.ERROR_ACCESS_DENIED
|
||||
// - syscall.ERROR_FILE_NOT_FOUND
|
||||
// - internal/syscall/windows.ERROR_SHARING_VIOLATION
|
||||
//
|
||||
// This set may be expanded in the future; programs must not rely on the
|
||||
// non-ephemerality of any given error.
|
||||
func IsEphemeralError(err error) bool {
|
||||
return isEphemeralError(err)
|
||||
}
|
|
@ -0,0 +1,28 @@
|
|||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !windows
|
||||
|
||||
package robustio
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
)
|
||||
|
||||
func rename(oldpath, newpath string) error {
|
||||
return os.Rename(oldpath, newpath)
|
||||
}
|
||||
|
||||
func readFile(filename string) ([]byte, error) {
|
||||
return ioutil.ReadFile(filename)
|
||||
}
|
||||
|
||||
func removeAll(path string) error {
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
|
||||
func isEphemeralError(err error) bool {
|
||||
return false
|
||||
}
|
|
@ -0,0 +1,105 @@
|
|||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package robustio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"internal/syscall/windows"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
const arbitraryTimeout = 500 * time.Millisecond
|
||||
|
||||
// retry retries ephemeral errors from f up to an arbitrary timeout
|
||||
// to work around spurious filesystem errors on Windows
|
||||
func retry(f func() (err error, mayRetry bool)) error {
|
||||
var (
|
||||
bestErr error
|
||||
lowestErrno syscall.Errno
|
||||
start time.Time
|
||||
nextSleep time.Duration = 1 * time.Millisecond
|
||||
)
|
||||
for {
|
||||
err, mayRetry := f()
|
||||
if err == nil || !mayRetry {
|
||||
return err
|
||||
}
|
||||
|
||||
var errno syscall.Errno
|
||||
if errors.As(err, &errno) && (lowestErrno == 0 || errno < lowestErrno) {
|
||||
bestErr = err
|
||||
lowestErrno = errno
|
||||
} else if bestErr == nil {
|
||||
bestErr = err
|
||||
}
|
||||
|
||||
if start.IsZero() {
|
||||
start = time.Now()
|
||||
} else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout {
|
||||
break
|
||||
}
|
||||
time.Sleep(nextSleep)
|
||||
nextSleep += time.Duration(rand.Int63n(int64(nextSleep)))
|
||||
}
|
||||
|
||||
return bestErr
|
||||
}
|
||||
|
||||
// rename is like os.Rename, but retries ephemeral errors.
|
||||
//
|
||||
// It wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with
|
||||
// MOVEFILE_REPLACE_EXISTING.
|
||||
//
|
||||
// Windows also provides a different system call, ReplaceFile,
|
||||
// that provides similar semantics, but perhaps preserves more metadata. (The
|
||||
// documentation on the differences between the two is very sparse.)
|
||||
//
|
||||
// Empirical error rates with MoveFileEx are lower under modest concurrency, so
|
||||
// for now we're sticking with what the os package already provides.
|
||||
func rename(oldpath, newpath string) (err error) {
|
||||
return retry(func() (err error, mayRetry bool) {
|
||||
err = os.Rename(oldpath, newpath)
|
||||
return err, isEphemeralError(err)
|
||||
})
|
||||
}
|
||||
|
||||
// readFile is like ioutil.ReadFile, but retries ephemeral errors.
|
||||
func readFile(filename string) ([]byte, error) {
|
||||
var b []byte
|
||||
err := retry(func() (err error, mayRetry bool) {
|
||||
b, err = ioutil.ReadFile(filename)
|
||||
|
||||
// Unlike in rename, we do not retry ERROR_FILE_NOT_FOUND here: it can occur
|
||||
// as a spurious error, but the file may also genuinely not exist, so the
|
||||
// increase in robustness is probably not worth the extra latency.
|
||||
return err, isEphemeralError(err) && !errors.Is(err, syscall.ERROR_FILE_NOT_FOUND)
|
||||
})
|
||||
return b, err
|
||||
}
|
||||
|
||||
func removeAll(path string) error {
|
||||
return retry(func() (err error, mayRetry bool) {
|
||||
err = os.RemoveAll(path)
|
||||
return err, isEphemeralError(err)
|
||||
})
|
||||
}
|
||||
|
||||
// isEphemeralError returns true if err may be resolved by waiting.
|
||||
func isEphemeralError(err error) bool {
|
||||
var errno syscall.Errno
|
||||
if errors.As(err, &errno) {
|
||||
switch errno {
|
||||
case syscall.ERROR_ACCESS_DENIED,
|
||||
syscall.ERROR_FILE_NOT_FOUND,
|
||||
windows.ERROR_SHARING_VIOLATION:
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
Loading…
Reference in New Issue