Merge remote-tracking branch 'origin/master' into release/1-6

5 years ago · 206b2016a4
7 changed files with 203 additions and 19 deletions
--- a/.circleci/bash_env.sh
+++ b/.circleci/bash_env.sh
@ -0,0 +1,7 @@
+#!/bin/bash
+
+export GIT_COMMIT=$(git rev-parse --short HEAD)
+export GIT_DIRTY=$(test -n "`git status --porcelain`" && echo "+CHANGES" || true)
+export GIT_DESCRIBE=$(git describe --tags --always --match "v*")
+export GIT_IMPORT=github.com/hashicorp/consul/version
+export GOLDFLAGS="-X ${GIT_IMPORT}.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X ${GIT_IMPORT}.GitDescribe=${GIT_DESCRIBE}"
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -21,6 +21,7 @@ references:
    GIT_AUTHOR_NAME: circleci-consul
    GIT_COMMITTER_NAME: circleci-consul
    S3_ARTIFACT_BUCKET:  consul-dev-artifacts
+    BASH_ENV: .circleci/bash_env.sh

 jobs:
  # lint consul tests
@ -140,6 +141,7 @@ jobs:
    docker:
      - image: *GOLANG_IMAGE
    environment: &build-env
+      <<: *ENVIRONMENT
      GOXPARALLEL: 2 # CircleCI containers are 2 CPU x 4GB RAM
    resource_class: large
    steps:
@ -168,17 +170,42 @@ jobs:
      XC_ARCH: "amd64"

  # build all arm/arm64 architecture supported OS binaries
-  build-arm-arm64:
-    <<: *build-distros
+  build-arm:
+    docker:
+      - image: *GOLANG_IMAGE
    environment:
-      <<: *build-env
-      XC_OS: linux
-      XC_ARCH: "arm arm64"
+      <<: *ENVIRONMENT
+      CGO_ENABLED: 1
+      GOOS: linux
+    steps:
+      - checkout
+      - run: sudo apt-get update && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
+      - run:
+          environment:
+            GOARM: 5
+            CC: arm-linux-gnueabi-gcc
+            GOARCH: arm
+          command: go build -o ./pkg/bin/linux_armel/consul  -ldflags="${GOLDFLAGS}"
+      - run:
+          environment:
+            GOARM: 6
+            CC: arm-linux-gnueabihf-gcc
+            GOARCH: arm
+          command: go build -o ./pkg/bin/linux_armhf/consul  -ldflags="${GOLDFLAGS}"
+      - run:
+          environment:
+            CC: aarch64-linux-gnu-gcc
+            GOARCH: arm64
+          command: go build -o ./pkg/bin/linux_aarch64/consul  -ldflags="${GOLDFLAGS}"
+      - store_artifacts:
+          path: ./pkg/bin

  # create a development build
  dev-build:
    docker:
      - image: *GOLANG_IMAGE
+    environment:
+      <<: *ENVIRONMENT
    steps:
      - checkout
      - restore_cache:
@ -497,7 +524,7 @@ jobs:
            git fetch origin

            # Create a merge branch to run tests on
-            git_merge_branch="ci/master-merge-$(date +%Y%m%d%H%M%S)"
+            git_merge_branch="ci/master-merge-${CIRCLE_BRANCH}-$(date +%Y%m%d%H%M%S)"
            git checkout -b "${git_merge_branch}"
            latest_oss_commit="$(git rev-parse origin/master)"

@ -554,7 +581,7 @@ jobs:
                \"attachments\": [ \
                  { \
                    \"fallback\": \"Nightly Master Merge Failed!\", \
-                    \"text\": \"Nightly *master* merge into *release/1-6* failed!\n\nBuild Log: ${CIRCLE_BUILD_URL}\n\nAttempted merge from: https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/tree/${git_merge_branch}\n\nThere may be a merge conflict to resolve.\", \
+                    \"text\": \"Nightly *master* merge into *${CIRCLE_BRANCH}* failed!\n\nBuild Log: ${CIRCLE_BUILD_URL}\n\nAttempted merge from: https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/tree/${git_merge_branch}\n\nThere may be a merge conflict to resolve.\", \
                    \"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \
                    \"ts\": \"$(date +%s)\", \
                    \"color\": \"danger\" \
@ -590,7 +617,7 @@ jobs:
                \"attachments\": [ \
                  { \
                    \"fallback\": \"Nightly master merge success!\", \
-                    \"text\": \"Nightly *master* merge into *release/1-6* succeeded!\", \
+                    \"text\": \"Nightly *master* merge into *${CIRCLE_BRANCH}* succeeded!\", \
                    \"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \
                    \"ts\": \"$(date +%s)\", \
                    \"color\": \"good\" \
@ -609,7 +636,7 @@ workflows:
          filters:
            branches:
              only:
-                - release/1-6
+                - /^release\/.*$/
  go-tests:
    jobs:
      - check-vendor
@ -628,7 +655,7 @@ workflows:
    jobs:
      - build-386
      - build-amd64
-      - build-arm-arm64
+      - build-arm
  test-integrations:
    jobs:
      - dev-build
--- a/command/connect/envoy/envoy.go
+++ b/command/connect/envoy/envoy.go
@ -220,11 +220,6 @@ func (c *cmd) Run(args []string) int {
 	if c.grpcAddr == "" {
 		c.grpcAddr = os.Getenv(api.GRPCAddrEnvName)
 	}
-	if c.grpcAddr == "" {
-		// This is the dev mode default and recommended production setting if
-		// enabled.
-		c.grpcAddr = "localhost:8502"
-	}
 	if c.http.Token() == "" && c.http.TokenFile() == "" {
 		// Extra check needed since CONSUL_HTTP_TOKEN has not been consulted yet but
 		// calling SetToken with empty will force that to override the
@ -360,6 +355,21 @@ func (c *cmd) Run(args []string) int {
 		return 1
 	}

+	// See if we need to lookup grpcAddr
+	if c.grpcAddr == "" {
+		port, err := c.lookupGRPCPort()
+		if err != nil {
+			c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
+		}
+		if port <= 0 {
+			// This is the dev mode default and recommended production setting if
+			// enabled.
+			port = 8502
+			c.UI.Info(fmt.Sprintf("Defaulting to grpc port = %d", port))
+		}
+		c.grpcAddr = fmt.Sprintf("localhost:%v", port)
+	}
+
 	// Generate config
 	bootstrapJson, err := c.generateConfig()
 	if err != nil {
@ -548,6 +558,27 @@ func (c *cmd) lookupGatewayProxy() (*api.AgentService, error) {
 	return proxyCmd.LookupGatewayProxy(c.client)
 }

+func (c *cmd) lookupGRPCPort() (int, error) {
+	self, err := c.client.Agent().Self()
+	if err != nil {
+		return 0, err
+	}
+	cfg, ok := self["DebugConfig"]
+	if !ok {
+		return 0, fmt.Errorf("unexpected agent response: no debug config")
+	}
+	port, ok := cfg["GRPCPort"]
+	if !ok {
+		return 0, fmt.Errorf("agent does not have grpc port enabled")
+	}
+	portN, ok := port.(float64)
+	if !ok {
+		return 0, fmt.Errorf("invalid grpc port in agent response")
+	}
+
+	return int(portN), nil
+}
+
 func (c *cmd) Synopsis() string {
 	return synopsis
 }
--- a/command/connect/envoy/envoy_test.go
+++ b/command/connect/envoy/envoy_test.go
@ -12,6 +12,7 @@ import (
 	"strings"
 	"testing"

+	"github.com/hashicorp/consul/agent"
 	"github.com/hashicorp/consul/agent/xds"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil"
@ -67,6 +68,7 @@ func TestGenerateConfig(t *testing.T) {
 		Env         []string
 		Files       map[string]string
 		ProxyConfig map[string]interface{}
+		GRPCPort    int // only used for testing custom-configured grpc port
 		WantArgs    BootstrapTplArgs
 		WantErr     string
 	}{
@ -222,6 +224,24 @@ func TestGenerateConfig(t *testing.T) {
 				LocalAgentClusterName: xds.LocalAgentClusterName,
 			},
 		},
+		{
+			Name:     "grpc-addr-config",
+			Flags:    []string{"-proxy-id", "test-proxy"},
+			GRPCPort: 9999,
+			WantArgs: BootstrapTplArgs{
+				ProxyCluster: "test-proxy",
+				ProxyID:      "test-proxy",
+				// Should resolve IP, note this might not resolve the same way
+				// everywhere which might make this test brittle but not sure what else
+				// to do.
+				AgentAddress:          "127.0.0.1",
+				AgentPort:             "9999",
+				AdminAccessLogPath:    "/dev/null",
+				AdminBindAddress:      "127.0.0.1",
+				AdminBindPort:         "19000",
+				LocalAgentClusterName: xds.LocalAgentClusterName,
+			},
+		},
 		{
 			Name:  "access-log-path",
 			Flags: []string{"-proxy-id", "test-proxy", "-admin-access-log-path", "/some/path/access.log"},
@ -453,7 +473,7 @@ func TestGenerateConfig(t *testing.T) {

 			// Run a mock agent API that just always returns the proxy config in the
 			// test.
-			srv := httptest.NewServer(testMockAgentProxyConfig(tc.ProxyConfig))
+			srv := httptest.NewServer(testMockAgent(tc.ProxyConfig, tc.GRPCPort))
 			defer srv.Close()

 			// Set the agent HTTP address in ENV to be our mock
@ -501,6 +521,25 @@ func TestGenerateConfig(t *testing.T) {
 	}
 }

+// testMockAgent combines testMockAgentProxyConfig and testMockAgentSelf,
+// routing /agent/service/... requests to testMockAgentProxyConfig and
+// routing /agent/self requests to testMockAgentSelf.
+func testMockAgent(agentCfg map[string]interface{}, grpcPort int) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.Contains(r.URL.Path, "/agent/service") {
+			testMockAgentProxyConfig(agentCfg)(w, r)
+			return
+		}
+
+		if strings.Contains(r.URL.Path, "/agent/self") {
+			testMockAgentSelf(grpcPort)(w, r)
+			return
+		}
+
+		http.NotFound(w, r)
+	})
+}
+
 func testMockAgentProxyConfig(cfg map[string]interface{}) http.HandlerFunc {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Parse the proxy-id from the end of the URL (blindly assuming it's correct
@ -624,3 +663,23 @@ func TestEnvoyCommand_canBindInternal(t *testing.T) {
 		})
 	}
 }
+
+// testMockAgentSelf returns an empty /v1/agent/self response except GRPC
+// port is filled in to match the given wantGRPCPort argument.
+func testMockAgentSelf(wantGRPCPort int) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		resp := agent.Self{
+			DebugConfig: map[string]interface{}{
+				"GRPCPort": wantGRPCPort,
+			},
+		}
+
+		selfJSON, err := json.Marshal(resp)
+		if err != nil {
+			w.WriteHeader(500)
+			w.Write([]byte(err.Error()))
+			return
+		}
+		w.Write(selfJSON)
+	})
+}
--- a/command/connect/envoy/testdata/grpc-addr-config.golden
+++ b/command/connect/envoy/testdata/grpc-addr-config.golden
@ -0,0 +1,60 @@
+{
+  "admin": {
+    "access_log_path": "/dev/null",
+    "address": {
+      "socket_address": {
+        "address": "127.0.0.1",
+        "port_value": 19000
+      }
+    }
+  },
+  "node": {
+    "cluster": "test-proxy",
+    "id": "test-proxy"
+  },
+  "static_resources": {
+    "clusters": [
+      {
+        "name": "local_agent",
+        "connect_timeout": "1s",
+        "type": "STATIC",
+        "http2_protocol_options": {},
+        "hosts": [
+          {
+            "socket_address": {
+              "address": "127.0.0.1",
+              "port_value": 9999
+            }
+          }
+        ]
+      }
+    ]
+  },
+  "stats_config": {
+			"stats_tags": [
+				{
+			"tag_name": "local_cluster",
+			"fixed_value": "test-proxy"
+		}
+			],
+			"use_all_default_tags": true
+		},
+  "dynamic_resources": {
+    "lds_config": { "ads": {} },
+    "cds_config": { "ads": {} },
+    "ads_config": {
+      "api_type": "GRPC",
+      "grpc_services": {
+        "initial_metadata": [
+          {
+            "key": "x-consul-token",
+            "value": ""
+          }
+        ],
+        "envoy_grpc": {
+          "cluster_name": "local_agent"
+        }
+      }
+    }
+  }
+}
--- a/website/source/docs/connect/index.html.md
+++ b/website/source/docs/connect/index.html.md
@ -31,7 +31,7 @@ networks, software-defined networks, cross-cloud, and more.

 ## Observability

-One of the key benefits Consul Connect is the uniform and consistent view it can
+One of the key benefits of Consul Connect is the uniform and consistent view it can
 provide  of all the services on your network, irrespective of their different
 programming languages and frameworks. When you configure Consul Connect to use
 sidecar proxies, those proxies "see" all service-to-service traffic and can
@ -52,7 +52,7 @@ There are several ways to try Connect in different environments.
   locally without installing anything else.

 - The [Kubernetes guide](https://learn.hashicorp.com/consul/getting-started-k8s/minikube)
-   walks you though configuring Consul Connect in Kubernetes using the Helm
+   walks you through configuring Consul Connect in Kubernetes using the Helm
   chart, and using intentions. You can run the guide on Minikube or an extant
   Kubernets cluster.

--- a/website/source/docs/internals/architecture.html.md
+++ b/website/source/docs/internals/architecture.html.md
@ -41,7 +41,7 @@ This means there is a gossip pool that contains all the agents for a given datac
 a few purposes: first, there is no need to configure clients with the addresses of servers;
 discovery is done automatically. Second, the work of detecting agent failures
 is not placed on the servers but is distributed. This makes failure detection much more
-scalable than naive heartbeating schemes. It also provides failure detection for the nodes; if the agent is not reachable, than the node may have experienced a failure. Thirdly, it is used as a messaging layer to notify
+scalable than naive heartbeating schemes. It also provides failure detection for the nodes; if the agent is not reachable, then the node may have experienced a failure. Thirdly, it is used as a messaging layer to notify
 when important events such as leader election take place.

 The servers in each datacenter are all part of a single Raft peer set. This means that