diff --git a/cluster/aws/config-default.sh b/cluster/aws/config-default.sh index 8ed226a48c..dd961a6903 100644 --- a/cluster/aws/config-default.sh +++ b/cluster/aws/config-default.sh @@ -41,10 +41,10 @@ MASTER_NAME="${INSTANCE_PREFIX}-master" MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) MASTER_TAG="${INSTANCE_PREFIX}-master" MINION_TAG="${INSTANCE_PREFIX}-minion" -MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24")) MINION_SCOPES="" POLL_SLEEP_INTERVAL=3 SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET +CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" # If set to Elastic IP, master instance will be associated with this IP. # If set to auto, a new Elastic IP will be aquired diff --git a/cluster/aws/config-test.sh b/cluster/aws/config-test.sh index 87cdf8c29c..5c51fd380f 100755 --- a/cluster/aws/config-test.sh +++ b/cluster/aws/config-test.sh @@ -37,10 +37,10 @@ MASTER_NAME="${INSTANCE_PREFIX}-master" MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) MASTER_TAG="${INSTANCE_PREFIX}-master" MINION_TAG="${INSTANCE_PREFIX}-minion" -MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24")) MINION_SCOPES="" POLL_SLEEP_INTERVAL=3 SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET +CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.245.0.0/16}" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" # If set to Elastic IP, master instance will be associated with this IP. # If set to auto, a new Elastic IP will be aquired diff --git a/cluster/aws/coreos/util.sh b/cluster/aws/coreos/util.sh index 9b8557dafe..77ea538df5 100644 --- a/cluster/aws/coreos/util.sh +++ b/cluster/aws/coreos/util.sh @@ -30,8 +30,8 @@ function detect-minion-image (){ function generate-minion-user-data() { i=$1 + # TODO(bakins): Is this actually used? MINION_PRIVATE_IP=$INTERNAL_IP_BASE.1${i} - MINION_IP_RANGE=${MINION_IP_RANGES[$i]} # this is a bit of a hack. We make all of our "variables" in # our cloud config controlled by env vars from this script @@ -44,7 +44,6 @@ function generate-minion-user-data() { DNS_SERVER_IP=$(yaml-quote ${DNS_SERVER_IP:-}) DNS_DOMAIN=$(yaml-quote ${DNS_DOMAIN:-}) MASTER_IP=$(yaml-quote ${MASTER_INTERNAL_IP}) - MINION_IP_RANGE=$(yaml-quote ${MINION_IP_RANGE}) MINION_IP=$(yaml-quote ${MINION_PRIVATE_IP}) KUBELET_TOKEN=$(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN=$(yaml-quote ${KUBE_PROXY_TOKEN:-}) diff --git a/cluster/aws/templates/create-dynamic-salt-files.sh b/cluster/aws/templates/create-dynamic-salt-files.sh index afff0d38d2..428659ca2b 100644 --- a/cluster/aws/templates/create-dynamic-salt-files.sh +++ b/cluster/aws/templates/create-dynamic-salt-files.sh @@ -22,6 +22,8 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")' node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")' +cluster_cidr: '$(echo "$CLUSTER_IP_RANGE" | sed -e "s/'/''/g")' +allocate_node_cidrs: '$(echo "$ALLOCATE_NODE_CIDRS" | sed -e "s/'/''/g")' service_cluster_ip_range: '$(echo "$SERVICE_CLUSTER_IP_RANGE" | sed -e "s/'/''/g")' enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")' enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")' diff --git a/cluster/aws/templates/salt-minion.sh b/cluster/aws/templates/salt-minion.sh index 0462ab9ad7..e5e25722c1 100755 --- a/cluster/aws/templates/salt-minion.sh +++ b/cluster/aws/templates/salt-minion.sh @@ -26,7 +26,7 @@ cat </etc/salt/minion.d/grains.conf grains: roles: - kubernetes-pool - cbr-cidr: $MINION_IP_RANGE + cbr-cidr: 10.123.45.0/30 cloud: aws EOF diff --git a/cluster/aws/ubuntu/common.sh b/cluster/aws/ubuntu/common.sh index cb8282a6fc..126c316671 100644 --- a/cluster/aws/ubuntu/common.sh +++ b/cluster/aws/ubuntu/common.sh @@ -29,7 +29,6 @@ function generate-minion-user-data { # We pipe this to the ami as a startup script in the user-data field. Requires a compatible ami echo "#! /bin/bash" echo "SALT_MASTER='${MASTER_INTERNAL_IP}'" - echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'" echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS:-}'" echo "readonly DOCKER_STORAGE='${DOCKER_STORAGE:-}'" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh" diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index d2c5c32d2d..02da313c4c 100644 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -22,6 +22,8 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/cluster/aws/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" +ALLOCATE_NODE_CIDRS=true + case "${KUBE_OS_DISTRIBUTION}" in ubuntu|wheezy|coreos) source "${KUBE_ROOT}/cluster/aws/${KUBE_OS_DISTRIBUTION}/util.sh" @@ -695,6 +697,8 @@ function kube-up { echo "readonly SALT_MASTER='${MASTER_INTERNAL_IP}'" echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'" echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'" + echo "readonly CLUSTER_IP_RANGE='${CLUSTER_IP_RANGE}'" + echo "readonly ALLOCATE_NODE_CIDRS='${ALLOCATE_NODE_CIDRS}'" echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'" echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" echo "readonly ZONE='${ZONE}'" @@ -854,7 +858,8 @@ function kube-up { MINION_IDS[$i]=$minion_id done - # Add routes to minions + # Configure minion networking + # TODO(justinsb): Check if we can change source-dest-check before instance fully running for (( i=0; i<${#MINION_NAMES[@]}; i++)); do # We are not able to add a route to the instance until that instance is in "running" state. # This is quite an ugly solution to this problem. In Bash 4 we could use assoc. arrays to do this for @@ -864,7 +869,6 @@ function kube-up { echo "Minion ${MINION_NAMES[$i]} running" sleep 10 $AWS_CMD modify-instance-attribute --instance-id $minion_id --source-dest-check '{"Value": false}' > $LOG - $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block ${MINION_IP_RANGES[$i]} --instance-id $minion_id > $LOG done FAIL=0 diff --git a/pkg/cloudprovider/aws/aws.go b/pkg/cloudprovider/aws/aws.go index 54d3b7dbd3..4aa8e7e564 100644 --- a/pkg/cloudprovider/aws/aws.go +++ b/pkg/cloudprovider/aws/aws.go @@ -86,6 +86,10 @@ type EC2 interface { DescribeSubnets(*ec2.DescribeSubnetsInput) ([]*ec2.Subnet, error) CreateTags(*ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) + + DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) + CreateRoute(request *ec2.CreateRouteInput) (*ec2.CreateRouteOutput, error) + DeleteRoute(request *ec2.DeleteRouteInput) (*ec2.DeleteRouteOutput, error) } // This is a simple pass-through of the ELB client interface, which allows for testing @@ -393,6 +397,23 @@ func (s *awsSdkEC2) CreateTags(request *ec2.CreateTagsInput) (*ec2.CreateTagsOut return s.ec2.CreateTags(request) } +func (s *awsSdkEC2) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) { + // Not paged + response, err := s.ec2.DescribeRouteTables(request) + if err != nil { + return nil, fmt.Errorf("error listing AWS route tables: %v", err) + } + return response.RouteTables, nil +} + +func (s *awsSdkEC2) CreateRoute(request *ec2.CreateRouteInput) (*ec2.CreateRouteOutput, error) { + return s.ec2.CreateRoute(request) +} + +func (s *awsSdkEC2) DeleteRoute(request *ec2.DeleteRouteInput) (*ec2.DeleteRouteOutput, error) { + return s.ec2.DeleteRoute(request) +} + func init() { cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { creds := credentials.NewChainCredentials( @@ -550,7 +571,7 @@ func (aws *AWSCloud) Zones() (cloudprovider.Zones, bool) { // Routes returns an implementation of Routes for Amazon Web Services. func (aws *AWSCloud) Routes() (cloudprovider.Routes, bool) { - return nil, false + return aws, true } // NodeAddresses is an implementation of Instances.NodeAddresses. diff --git a/pkg/cloudprovider/aws/aws_routes.go b/pkg/cloudprovider/aws/aws_routes.go new file mode 100644 index 0000000000..ba9b4d9da1 --- /dev/null +++ b/pkg/cloudprovider/aws/aws_routes.go @@ -0,0 +1,114 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws_cloud + +import ( + "fmt" + "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" +) + +func (s *AWSCloud) findRouteTable(clusterName string) (*ec2.RouteTable, error) { + request := &ec2.DescribeRouteTablesInput{} + filters := []*ec2.Filter{} + // This should be unnecessary (we already filter on TagNameKubernetesCluster, + // and something is broken if cluster name doesn't match, but anyway... + // TODO: All clouds should be cluster-aware by default + filters = append(filters, newEc2Filter("tag:"+TagNameKubernetesCluster, clusterName)) + request.Filters = s.addFilters(filters) + + tables, err := s.ec2.DescribeRouteTables(request) + if err != nil { + return nil, err + } + + if len(tables) == 0 { + return nil, fmt.Errorf("unable to find route table for AWS cluster: %s", clusterName) + } + + if len(tables) != 1 { + return nil, fmt.Errorf("found multiple matching AWS route tables for AWS cluster: %s", clusterName) + } + return tables[0], nil +} + +// ListRoutes implements Routes.ListRoutes +// List all routes that match the filter +func (s *AWSCloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) { + table, err := s.findRouteTable(clusterName) + if err != nil { + return nil, err + } + + var routes []*cloudprovider.Route + for _, r := range table.Routes { + instanceID := orEmpty(r.InstanceID) + destinationCIDR := orEmpty(r.DestinationCIDRBlock) + + if instanceID == "" || destinationCIDR == "" { + continue + } + + routeName := clusterName + "-" + destinationCIDR + routes = append(routes, &cloudprovider.Route{routeName, instanceID, destinationCIDR}) + } + + return routes, nil +} + +// CreateRoute implements Routes.CreateRoute +// Create the described route +func (s *AWSCloud) CreateRoute(clusterName string, nameHint string, route *cloudprovider.Route) error { + table, err := s.findRouteTable(clusterName) + if err != nil { + return err + } + + request := &ec2.CreateRouteInput{} + // TODO: use ClientToken for idempotency? + request.DestinationCIDRBlock = aws.String(route.DestinationCIDR) + request.InstanceID = aws.String(route.TargetInstance) + request.RouteTableID = table.RouteTableID + + _, err = s.ec2.CreateRoute(request) + if err != nil { + return fmt.Errorf("error creating AWS route (%s): %v", route.DestinationCIDR, err) + } + + return nil +} + +// DeleteRoute implements Routes.DeleteRoute +// Delete the specified route +func (s *AWSCloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error { + table, err := s.findRouteTable(clusterName) + if err != nil { + return err + } + + request := &ec2.DeleteRouteInput{} + request.DestinationCIDRBlock = aws.String(route.DestinationCIDR) + request.RouteTableID = table.RouteTableID + + _, err = s.ec2.DeleteRoute(request) + if err != nil { + return fmt.Errorf("error deleting AWS route (%s): %v", route.DestinationCIDR, err) + } + + return nil +} diff --git a/pkg/cloudprovider/aws/aws_test.go b/pkg/cloudprovider/aws/aws_test.go index 60462a7415..f45537abd8 100644 --- a/pkg/cloudprovider/aws/aws_test.go +++ b/pkg/cloudprovider/aws/aws_test.go @@ -360,6 +360,18 @@ func (ec2 *FakeEC2) CreateTags(*ec2.CreateTagsInput) (*ec2.CreateTagsOutput, err panic("Not implemented") } +func (s *FakeEC2) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) { + panic("Not implemented") +} + +func (s *FakeEC2) CreateRoute(request *ec2.CreateRouteInput) (*ec2.CreateRouteOutput, error) { + panic("Not implemented") +} + +func (s *FakeEC2) DeleteRoute(request *ec2.DeleteRouteInput) (*ec2.DeleteRouteOutput, error) { + panic("Not implemented") +} + type FakeELB struct { aws *FakeAWSServices }