renameio.WriteFile writes files atomically by renaming temporary files

pull/118/head
kun 2019-09-20 10:00:05 +08:00
parent 389bbbec05
commit 2b89997884
7 changed files with 485 additions and 6 deletions

View File

@ -14,6 +14,8 @@ import (
"path/filepath"
"strings"
"time"
"github.com/goproxyio/goproxy/renameio"
)
const ListExpire = 5 * time.Minute
@ -66,19 +68,30 @@ func NewRouter(srv *Server, opts *RouterOptions) *Router {
if r.StatusCode == http.StatusOK {
var buf []byte
if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") {
if gr, err := gzip.NewReader(r.Body); err == nil {
defer gr.Close()
buf, _ = ioutil.ReadAll(gr)
r.Header.Del("Content-Encoding")
gr, err := gzip.NewReader(r.Body)
if err != nil {
return err
}
defer gr.Close()
buf, err = ioutil.ReadAll(gr)
if err != nil {
return err
}
r.Header.Del("Content-Encoding")
} else {
buf, _ = ioutil.ReadAll(r.Body)
buf, err = ioutil.ReadAll(r.Body)
if err != nil {
return err
}
}
r.Body = ioutil.NopCloser(bytes.NewReader(buf))
if buf != nil {
file := filepath.Join(opts.DownloadRoot, r.Request.URL.Path)
os.MkdirAll(path.Dir(file), os.ModePerm)
ioutil.WriteFile(file, buf, 0666)
err = renameio.WriteFile(file, buf, 0666)
if err != nil {
return err
}
}
}
return nil

93
renameio/renameio.go Normal file
View File

@ -0,0 +1,93 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package renameio writes files atomically by renaming temporary files.
package renameio
import (
"bytes"
"io"
"math/rand"
"os"
"path/filepath"
"strconv"
"github.com/goproxyio/goproxy/robustio"
)
const patternSuffix = ".tmp"
// Pattern returns a glob pattern that matches the unrenamed temporary files
// created when writing to filename.
func Pattern(filename string) string {
return filepath.Join(filepath.Dir(filename), filepath.Base(filename)+patternSuffix)
}
// WriteFile is like ioutil.WriteFile, but first writes data to an arbitrary
// file in the same directory as filename, then renames it atomically to the
// final name.
//
// That ensures that the final location, if it exists, is always a complete file.
func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
return WriteToFile(filename, bytes.NewReader(data), perm)
}
// WriteToFile is a variant of WriteFile that accepts the data as an io.Reader
// instead of a slice.
func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) {
f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm)
if err != nil {
return err
}
defer func() {
// Only call os.Remove on f.Name() if we failed to rename it: otherwise,
// some other process may have created a new file with the same name after
// that.
if err != nil {
f.Close()
os.Remove(f.Name())
}
}()
if _, err := io.Copy(f, data); err != nil {
return err
}
// Sync the file before renaming it: otherwise, after a crash the reader may
// observe a 0-length file instead of the actual contents.
// See https://golang.org/issue/22397#issuecomment-380831736.
if err := f.Sync(); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
return robustio.Rename(f.Name(), filename)
}
// ReadFile is like ioutil.ReadFile, but on Windows retries spurious errors that
// may occur if the file is concurrently replaced.
//
// Errors are classified heuristically and retries are bounded, so even this
// function may occasionally return a spurious error on Windows.
// If so, the error will likely wrap one of:
// - syscall.ERROR_ACCESS_DENIED
// - syscall.ERROR_FILE_NOT_FOUND
// - internal/syscall/windows.ERROR_SHARING_VIOLATION
func ReadFile(filename string) ([]byte, error) {
return robustio.ReadFile(filename)
}
// tempFile creates a new temporary file with given permission bits.
func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) {
for i := 0; i < 10000; i++ {
name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix)
f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
if os.IsExist(err) {
continue
}
break
}
return
}

145
renameio/renameio_test.go Normal file
View File

@ -0,0 +1,145 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !plan9
package renameio
import (
"encoding/binary"
"errors"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"runtime"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/goproxyio/goproxy/robustio"
)
func TestConcurrentReadsAndWrites(t *testing.T) {
dir, err := ioutil.TempDir("", "renameio")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
path := filepath.Join(dir, "blob.bin")
const chunkWords = 8 << 10
buf := make([]byte, 2*chunkWords*8)
for i := uint64(0); i < 2*chunkWords; i++ {
binary.LittleEndian.PutUint64(buf[i*8:], i)
}
var attempts int64 = 128
if !testing.Short() {
attempts *= 16
}
const parallel = 32
var sem = make(chan bool, parallel)
var (
writeSuccesses, readSuccesses int64 // atomic
writeErrnoSeen, readErrnoSeen sync.Map
)
for n := attempts; n > 0; n-- {
sem <- true
go func() {
defer func() { <-sem }()
time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond)
offset := rand.Intn(chunkWords)
chunk := buf[offset*8 : (offset+chunkWords)*8]
if err := WriteFile(path, chunk, 0666); err == nil {
atomic.AddInt64(&writeSuccesses, 1)
} else if robustio.IsEphemeralError(err) {
var (
errno syscall.Errno
dup bool
)
if errors.As(err, &errno) {
_, dup = writeErrnoSeen.LoadOrStore(errno, true)
}
if !dup {
t.Logf("ephemeral error: %v", err)
}
} else {
t.Errorf("unexpected error: %v", err)
}
time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond)
data, err := ReadFile(path)
if err == nil {
atomic.AddInt64(&readSuccesses, 1)
} else if robustio.IsEphemeralError(err) {
var (
errno syscall.Errno
dup bool
)
if errors.As(err, &errno) {
_, dup = readErrnoSeen.LoadOrStore(errno, true)
}
if !dup {
t.Logf("ephemeral error: %v", err)
}
return
} else {
t.Errorf("unexpected error: %v", err)
return
}
if len(data) != 8*chunkWords {
t.Errorf("read %d bytes, but each write is a %d-byte file", len(data), 8*chunkWords)
return
}
u := binary.LittleEndian.Uint64(data)
for i := 1; i < chunkWords; i++ {
next := binary.LittleEndian.Uint64(data[i*8:])
if next != u+1 {
t.Errorf("wrote sequential integers, but read integer out of sequence at offset %d", i)
return
}
u = next
}
}()
}
for n := parallel; n > 0; n-- {
sem <- true
}
var minWriteSuccesses int64 = attempts
if runtime.GOOS == "windows" {
// Windows produces frequent "Access is denied" errors under heavy rename load.
// As long as those are the only errors and *some* of the writes succeed, we're happy.
minWriteSuccesses = attempts / 4
}
if writeSuccesses < minWriteSuccesses {
t.Errorf("%d (of %d) writes succeeded; want ≥ %d", writeSuccesses, attempts, minWriteSuccesses)
} else {
t.Logf("%d (of %d) writes succeeded (ok: ≥ %d)", writeSuccesses, attempts, minWriteSuccesses)
}
var minReadSuccesses int64 = attempts
if runtime.GOOS == "windows" {
// Windows produces frequent "Access is denied" errors under heavy rename load.
// As long as those are the only errors and *some* of the writes succeed, we're happy.
minReadSuccesses = attempts / 4
}
if readSuccesses < minReadSuccesses {
t.Errorf("%d (of %d) reads succeeded; want ≥ %d", readSuccesses, attempts, minReadSuccesses)
} else {
t.Logf("%d (of %d) reads succeeded (ok: ≥ %d)", readSuccesses, attempts, minReadSuccesses)
}
}

42
renameio/umask_test.go Normal file
View File

@ -0,0 +1,42 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !nacl,!plan9,!windows,!js
package renameio
import (
"io/ioutil"
"os"
"path/filepath"
"syscall"
"testing"
)
func TestWriteFileModeAppliesUmask(t *testing.T) {
dir, err := ioutil.TempDir("", "renameio")
if err != nil {
t.Fatalf("Failed to create temporary directory: %v", err)
}
defer os.RemoveAll(dir)
const mode = 0644
const umask = 0007
defer syscall.Umask(syscall.Umask(umask))
file := filepath.Join(dir, "testWrite")
err = WriteFile(file, []byte("go-build"), mode)
if err != nil {
t.Fatalf("Failed to write file: %v", err)
}
fi, err := os.Stat(file)
if err != nil {
t.Fatalf("Stat %q (looking for mode %#o): %s", file, mode, err)
}
if fi.Mode()&os.ModePerm != 0640 {
t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&os.ModePerm, 0640)
}
}

53
robustio/robustio.go Normal file
View File

@ -0,0 +1,53 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package robustio wraps I/O functions that are prone to failure on Windows,
// transparently retrying errors up to an arbitrary timeout.
//
// Errors are classified heuristically and retries are bounded, so the functions
// in this package do not completely eliminate spurious errors. However, they do
// significantly reduce the rate of failure in practice.
//
// If so, the error will likely wrap one of:
// The functions in this package do not completely eliminate spurious errors,
// but substantially reduce their rate of occurrence in practice.
package robustio
// Rename is like os.Rename, but on Windows retries errors that may occur if the
// file is concurrently read or overwritten.
//
// (See golang.org/issue/31247 and golang.org/issue/32188.)
func Rename(oldpath, newpath string) error {
return rename(oldpath, newpath)
}
// ReadFile is like ioutil.ReadFile, but on Windows retries errors that may
// occur if the file is concurrently replaced.
//
// (See golang.org/issue/31247 and golang.org/issue/32188.)
func ReadFile(filename string) ([]byte, error) {
return readFile(filename)
}
// RemoveAll is like os.RemoveAll, but on Windows retries errors that may occur
// if an executable file in the directory has recently been executed.
//
// (See golang.org/issue/19491.)
func RemoveAll(path string) error {
return removeAll(path)
}
// IsEphemeralError reports whether err is one of the errors that the functions
// in this package attempt to mitigate.
//
// Errors considered ephemeral include:
// - syscall.ERROR_ACCESS_DENIED
// - syscall.ERROR_FILE_NOT_FOUND
// - internal/syscall/windows.ERROR_SHARING_VIOLATION
//
// This set may be expanded in the future; programs must not rely on the
// non-ephemerality of any given error.
func IsEphemeralError(err error) bool {
return isEphemeralError(err)
}

View File

@ -0,0 +1,28 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !windows
package robustio
import (
"io/ioutil"
"os"
)
func rename(oldpath, newpath string) error {
return os.Rename(oldpath, newpath)
}
func readFile(filename string) ([]byte, error) {
return ioutil.ReadFile(filename)
}
func removeAll(path string) error {
return os.RemoveAll(path)
}
func isEphemeralError(err error) bool {
return false
}

View File

@ -0,0 +1,105 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package robustio
import (
"errors"
"internal/syscall/windows"
"io/ioutil"
"math/rand"
"os"
"syscall"
"time"
)
const arbitraryTimeout = 500 * time.Millisecond
// retry retries ephemeral errors from f up to an arbitrary timeout
// to work around spurious filesystem errors on Windows
func retry(f func() (err error, mayRetry bool)) error {
var (
bestErr error
lowestErrno syscall.Errno
start time.Time
nextSleep time.Duration = 1 * time.Millisecond
)
for {
err, mayRetry := f()
if err == nil || !mayRetry {
return err
}
var errno syscall.Errno
if errors.As(err, &errno) && (lowestErrno == 0 || errno < lowestErrno) {
bestErr = err
lowestErrno = errno
} else if bestErr == nil {
bestErr = err
}
if start.IsZero() {
start = time.Now()
} else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout {
break
}
time.Sleep(nextSleep)
nextSleep += time.Duration(rand.Int63n(int64(nextSleep)))
}
return bestErr
}
// rename is like os.Rename, but retries ephemeral errors.
//
// It wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with
// MOVEFILE_REPLACE_EXISTING.
//
// Windows also provides a different system call, ReplaceFile,
// that provides similar semantics, but perhaps preserves more metadata. (The
// documentation on the differences between the two is very sparse.)
//
// Empirical error rates with MoveFileEx are lower under modest concurrency, so
// for now we're sticking with what the os package already provides.
func rename(oldpath, newpath string) (err error) {
return retry(func() (err error, mayRetry bool) {
err = os.Rename(oldpath, newpath)
return err, isEphemeralError(err)
})
}
// readFile is like ioutil.ReadFile, but retries ephemeral errors.
func readFile(filename string) ([]byte, error) {
var b []byte
err := retry(func() (err error, mayRetry bool) {
b, err = ioutil.ReadFile(filename)
// Unlike in rename, we do not retry ERROR_FILE_NOT_FOUND here: it can occur
// as a spurious error, but the file may also genuinely not exist, so the
// increase in robustness is probably not worth the extra latency.
return err, isEphemeralError(err) && !errors.Is(err, syscall.ERROR_FILE_NOT_FOUND)
})
return b, err
}
func removeAll(path string) error {
return retry(func() (err error, mayRetry bool) {
err = os.RemoveAll(path)
return err, isEphemeralError(err)
})
}
// isEphemeralError returns true if err may be resolved by waiting.
func isEphemeralError(err error) bool {
var errno syscall.Errno
if errors.As(err, &errno) {
switch errno {
case syscall.ERROR_ACCESS_DENIED,
syscall.ERROR_FILE_NOT_FOUND,
windows.ERROR_SHARING_VIOLATION:
return true
}
}
return false
}