From fea0684c805fd54e07bd77e8408cd0c0d19861cd Mon Sep 17 00:00:00 2001 From: Yicheng Qin Date: Thu, 16 Apr 2015 14:41:22 -0700 Subject: [PATCH] *: update go-etcd dependency We recently updated the go-etcd. It solves the following issues: 1. support both 0.4.x and 2.x cluster 2. support redirection 3. always switch to another etcd member in case of a network failure 4. fix a bug in sync cluster --- Godeps/Godeps.json | 4 +- .../github.com/coreos/go-etcd/etcd/client.go | 75 +++++++++++++------ .../coreos/go-etcd/etcd/client_test.go | 12 +++ .../github.com/coreos/go-etcd/etcd/cluster.go | 3 + .../github.com/coreos/go-etcd/etcd/member.go | 30 ++++++++ .../coreos/go-etcd/etcd/member_test.go | 71 ++++++++++++++++++ .../coreos/go-etcd/etcd/requests.go | 38 +++++++--- .../coreos/go-etcd/etcd/requests_test.go | 22 ++++++ .../github.com/coreos/go-etcd/etcd/version.go | 5 +- 9 files changed, 226 insertions(+), 34 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 8232c06955..adfa1bda05 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -74,8 +74,8 @@ }, { "ImportPath": "github.com/coreos/go-etcd/etcd", - "Comment": "v0.4.6-8-g60e12ca", - "Rev": "60e12cac3db8ffce00b576b4af0e7b0a968f1003" + "Comment": "v2.0.0-3-g0424b5f", + "Rev": "0424b5f86ef0ca57a5309c599f74bbb3e97ecd9d" }, { "ImportPath": "github.com/coreos/go-systemd/dbus", diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go index 8ecb50ee53..c6cf3341ba 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go @@ -15,8 +15,6 @@ import ( "path" "strings" "time" - - "github.com/coreos/etcd/etcdserver/etcdhttp/httptypes" ) // See SetConsistency for how to use these constants. @@ -44,10 +42,17 @@ type Config struct { Consistency string `json:"consistency"` } +type credentials struct { + username string + password string +} + type Client struct { config Config `json:"config"` cluster *Cluster `json:"cluster"` httpClient *http.Client + credentials *credentials + transport *http.Transport persistence io.Writer cURLch chan string // CheckRetry can be used to control the policy for failed requests @@ -172,17 +177,27 @@ func NewClientFromReader(reader io.Reader) (*Client, error) { // Override the Client's HTTP Transport object func (c *Client) SetTransport(tr *http.Transport) { c.httpClient.Transport = tr + c.transport = tr +} + +func (c *Client) SetCredentials(username, password string) { + c.credentials = &credentials{username, password} +} + +func (c *Client) Close() { + c.transport.DisableKeepAlives = true + c.transport.CloseIdleConnections() } // initHTTPClient initializes a HTTP client for etcd client func (c *Client) initHTTPClient() { - tr := &http.Transport{ + c.transport = &http.Transport{ Dial: c.dial, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, } - c.httpClient = &http.Client{Transport: tr} + c.httpClient = &http.Client{Transport: c.transport} } // initHTTPClient initializes a HTTPS client for etcd client @@ -305,31 +320,49 @@ func (c *Client) internalSyncCluster(machines []string) bool { continue } - b, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - // try another machine in the cluster - continue - } + if resp.StatusCode != http.StatusOK { // fall-back to old endpoint + httpPath := c.createHttpPath(machine, path.Join(version, "machines")) + resp, err := c.httpClient.Get(httpPath) + if err != nil { + // try another machine in the cluster + continue + } + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + // try another machine in the cluster + continue + } + // update Machines List + c.cluster.updateFromStr(string(b)) + } else { + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + // try another machine in the cluster + continue + } - var mCollection httptypes.MemberCollection - if err := json.Unmarshal(b, &mCollection); err != nil { - // try another machine - continue - } + var mCollection memberCollection + if err := json.Unmarshal(b, &mCollection); err != nil { + // try another machine + continue + } - urls := make([]string, 0) - for _, m := range mCollection { - urls = append(urls, m.ClientURLs...) - } + urls := make([]string, 0) + for _, m := range mCollection { + urls = append(urls, m.ClientURLs...) + } - // update Machines List - c.cluster.updateFromStr(strings.Join(urls, ",")) + // update Machines List + c.cluster.updateFromStr(strings.Join(urls, ",")) + } logger.Debug("sync.machines ", c.cluster.Machines) c.saveConfig() return true } + return false } diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go index 66d79d7332..4720d8d693 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go @@ -94,3 +94,15 @@ func TestPersistence(t *testing.T) { t.Fatalf("The two configs should be equal!") } } + +func TestClientRetry(t *testing.T) { + c := NewClient([]string{"http://strange", "http://127.0.0.1:4001"}) + // use first endpoint as the picked url + c.cluster.picked = 0 + if _, err := c.Set("foo", "bar", 5); err != nil { + t.Fatal(err) + } + if _, err := c.Delete("foo", true); err != nil { + t.Fatal(err) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go index 787cf753ba..1ad3e155be 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go @@ -30,5 +30,8 @@ func (cl *Cluster) pick() string { return cl.Machines[cl.picked] } func (cl *Cluster) updateFromStr(machines string) { cl.Machines = strings.Split(machines, ",") + for i := range cl.Machines { + cl.Machines[i] = strings.TrimSpace(cl.Machines[i]) + } cl.picked = rand.Intn(len(cl.Machines)) } diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member.go new file mode 100644 index 0000000000..5b13b28e1a --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member.go @@ -0,0 +1,30 @@ +package etcd + +import "encoding/json" + +type Member struct { + ID string `json:"id"` + Name string `json:"name"` + PeerURLs []string `json:"peerURLs"` + ClientURLs []string `json:"clientURLs"` +} + +type memberCollection []Member + +func (c *memberCollection) UnmarshalJSON(data []byte) error { + d := struct { + Members []Member + }{} + + if err := json.Unmarshal(data, &d); err != nil { + return err + } + + if d.Members == nil { + *c = make([]Member, 0) + return nil + } + + *c = d.Members + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member_test.go new file mode 100644 index 0000000000..53ebdd4bfd --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/member_test.go @@ -0,0 +1,71 @@ +package etcd + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestMemberCollectionUnmarshal(t *testing.T) { + tests := []struct { + body []byte + want memberCollection + }{ + { + body: []byte(`{"members":[]}`), + want: memberCollection([]Member{}), + }, + { + body: []byte(`{"members":[{"id":"2745e2525fce8fe","peerURLs":["http://127.0.0.1:7003"],"name":"node3","clientURLs":["http://127.0.0.1:4003"]},{"id":"42134f434382925","peerURLs":["http://127.0.0.1:2380","http://127.0.0.1:7001"],"name":"node1","clientURLs":["http://127.0.0.1:2379","http://127.0.0.1:4001"]},{"id":"94088180e21eb87b","peerURLs":["http://127.0.0.1:7002"],"name":"node2","clientURLs":["http://127.0.0.1:4002"]}]}`), + want: memberCollection( + []Member{ + { + ID: "2745e2525fce8fe", + Name: "node3", + PeerURLs: []string{ + "http://127.0.0.1:7003", + }, + ClientURLs: []string{ + "http://127.0.0.1:4003", + }, + }, + { + ID: "42134f434382925", + Name: "node1", + PeerURLs: []string{ + "http://127.0.0.1:2380", + "http://127.0.0.1:7001", + }, + ClientURLs: []string{ + "http://127.0.0.1:2379", + "http://127.0.0.1:4001", + }, + }, + { + ID: "94088180e21eb87b", + Name: "node2", + PeerURLs: []string{ + "http://127.0.0.1:7002", + }, + ClientURLs: []string{ + "http://127.0.0.1:4002", + }, + }, + }, + ), + }, + } + + for i, tt := range tests { + var got memberCollection + err := json.Unmarshal(tt.body, &got) + if err != nil { + t.Errorf("#%d: unexpected error: %v", i, err) + continue + } + + if !reflect.DeepEqual(tt.want, got) { + t.Errorf("#%d: incorrect output: want=%#v, got=%#v", i, tt.want, got) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go index 70d9db2def..c4d2267da7 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "net" "net/http" "net/url" "path" @@ -189,7 +188,10 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { logger.Debug("Connecting to etcd: attempt ", attempt+1, " for ", rr.RelativePath) - httpPath = c.getHttpPath(rr.RelativePath) + // get httpPath if not set + if httpPath == "" { + httpPath = c.getHttpPath(rr.RelativePath) + } // Return a cURL command if curlChan is set if c.cURLch != nil { @@ -197,6 +199,9 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { for key, value := range rr.Values { command += fmt.Sprintf(" -d %s=%s", key, value[0]) } + if c.credentials != nil { + command += fmt.Sprintf(" -u %s", c.credentials.username) + } c.sendCURL(command) } @@ -226,7 +231,13 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { return nil, err } + if c.credentials != nil { + req.SetBasicAuth(c.credentials.username, c.credentials.password) + } + resp, err = c.httpClient.Do(req) + // clear previous httpPath + httpPath = "" defer func() { if resp != nil { resp.Body.Close() @@ -281,6 +292,19 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { } } + if resp.StatusCode == http.StatusTemporaryRedirect { + u, err := resp.Location() + + if err != nil { + logger.Warning(err) + } else { + // set httpPath for following redirection + httpPath = u.String() + } + resp.Body.Close() + continue + } + if checkErr := checkRetry(c.cluster, numReqs, *resp, errors.New("Unexpected HTTP status code")); checkErr != nil { return nil, checkErr @@ -304,9 +328,8 @@ func DefaultCheckRetry(cluster *Cluster, numReqs int, lastResp http.Response, err error) error { if isEmptyResponse(lastResp) { - if !isConnectionError(err) { - return err - } + // always retry if it failed to get response from one machine + return nil } else if !shouldRetry(lastResp) { body := []byte("nil") if lastResp.Body != nil { @@ -333,11 +356,6 @@ func DefaultCheckRetry(cluster *Cluster, numReqs int, lastResp http.Response, func isEmptyResponse(r http.Response) bool { return r.StatusCode == 0 } -func isConnectionError(err error) bool { - _, ok := err.(*net.OpError) - return ok -} - // shouldRetry returns whether the reponse deserves retry. func shouldRetry(r http.Response) bool { // TODO: only retry when the cluster is in leader election diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests_test.go new file mode 100644 index 0000000000..7a2bd190a1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests_test.go @@ -0,0 +1,22 @@ +package etcd + +import "testing" + +func TestKeyToPath(t *testing.T) { + tests := []struct { + key string + wpath string + }{ + {"", "keys/"}, + {"foo", "keys/foo"}, + {"foo/bar", "keys/foo/bar"}, + {"%z", "keys/%25z"}, + {"/", "keys/"}, + } + for i, tt := range tests { + path := keyToPath(tt.key) + if path != tt.wpath { + t.Errorf("#%d: path = %s, want %s", i, path, tt.wpath) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go index b3d05df70b..b1e9ed2713 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go @@ -1,3 +1,6 @@ package etcd -const version = "v2" +const ( + version = "v2" + packageVersion = "v2.0.0+git" +)