github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #286 from lavalamp/exampleEtcd 

Separate scheduler from registry
pull/6/head
brendandburns 2014-06-30 20:17:07 -07:00
parent 2b8ccbf9e9 011ce9d871
commit 65a62278b1
15 changed files with 450 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/integration/integration.go
View File

@ -38,6 +38,7 @@ import (
func main() {
runtime.GOMAXPROCS(4)
util.ReallyCrash = true
util.InitLogs()
defer util.FlushLogs()

5
pkg/master/master.go
View File

@ -25,6 +25,7 @@ import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
"github.com/GoogleCloudPlatform/kubernetes/pkg/registry"
"github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
"github.com/coreos/go-etcd/etcd"
"github.com/golang/glog"
@ -37,6 +38,7 @@ type Master struct {
serviceRegistry registry.ServiceRegistry
minionRegistry registry.MinionRegistry
// TODO: don't reuse non-threadsafe objects.
random *rand.Rand
storage map[string]apiserver.RESTStorage
}
@ -86,8 +88,9 @@ func (m *Master) init(cloud cloudprovider.Interface) {
m.random = rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
podCache := NewPodCache(containerInfo, m.podRegistry, time.Second*30)
go podCache.Loop()
s := scheduler.MakeFirstFitScheduler(m.podRegistry, m.random)
m.storage = map[string]apiserver.RESTStorage{
"pods": registry.MakePodRegistryStorage(m.podRegistry, containerInfo, registry.MakeFirstFitScheduler(m.minionRegistry, m.podRegistry, m.random), cloud, podCache),
"pods": registry.MakePodRegistryStorage(m.podRegistry, containerInfo, s, m.minionRegistry, cloud, podCache),
"replicationControllers": registry.MakeControllerRegistryStorage(m.controllerRegistry, m.podRegistry),
"services": registry.MakeServiceRegistryStorage(m.serviceRegistry, cloud, m.minionRegistry),
"minions": registry.MakeMinionRegistryStorage(m.minionRegistry),

25
pkg/registry/pod_registry.go
View File

@ -25,6 +25,7 @@ import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
"github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler"
"github.com/golang/glog"
)
@ -33,22 +34,30 @@ type PodRegistryStorage struct {
registry PodRegistry
containerInfo client.ContainerInfo
podCache client.ContainerInfo
scheduler Scheduler
scheduler scheduler.Scheduler
minionLister scheduler.MinionLister
cloud cloudprovider.Interface
}
// MakePodRegistryStorage makes a RESTStorage object for a pod registry.
// Parameters:
// registry The pod registry
// containerInfo Source of fresh container info
// scheduler The scheduler for assigning pods to machines
// cloud Interface to a cloud provider (may be null)
// podCache Source of cached container info
func MakePodRegistryStorage(registry PodRegistry, containerInfo client.ContainerInfo, scheduler Scheduler, cloud cloudprovider.Interface, podCache client.ContainerInfo) apiserver.RESTStorage {
// registry: The pod registry
// containerInfo: Source of fresh container info
// scheduler: The scheduler for assigning pods to machines
// minionLister: Object which can list available minions for the scheduler
// cloud: Interface to a cloud provider (may be null)
// podCache: Source of cached container info
func MakePodRegistryStorage(registry PodRegistry,
containerInfo client.ContainerInfo,
scheduler scheduler.Scheduler,
minionLister scheduler.MinionLister,
cloud cloudprovider.Interface,
podCache client.ContainerInfo) apiserver.RESTStorage {
return &PodRegistryStorage{
registry: registry,
containerInfo: containerInfo,
scheduler: scheduler,
minionLister: minionLister,
cloud: cloud,
podCache: podCache,
}
@ -150,7 +159,7 @@ func (storage *PodRegistryStorage) Create(obj interface{}) (<-chan interface{},
return apiserver.MakeAsync(func() (interface{}, error) {
// TODO(lavalamp): Separate scheduler more cleanly.
machine, err := storage.scheduler.Schedule(pod)
machine, err := storage.scheduler.Schedule(pod, storage.minionLister)
if err != nil {
return nil, err
}

115
pkg/registry/scheduler_test.go
View File

@ -1,115 +0,0 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"math/rand"
"testing"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)
func expectSchedule(scheduler Scheduler, pod api.Pod, expected string, t *testing.T) {
actual, err := scheduler.Schedule(pod)
expectNoError(t, err)
if actual != expected {
t.Errorf("Unexpected scheduling value: %v, expected %v", actual, expected)
}
}
func TestRoundRobinScheduler(t *testing.T) {
scheduler := MakeRoundRobinScheduler(MakeMinionRegistry([]string{"m1", "m2", "m3", "m4"}))
expectSchedule(scheduler, api.Pod{}, "m1", t)
expectSchedule(scheduler, api.Pod{}, "m2", t)
expectSchedule(scheduler, api.Pod{}, "m3", t)
expectSchedule(scheduler, api.Pod{}, "m4", t)
}
func TestRandomScheduler(t *testing.T) {
random := rand.New(rand.NewSource(0))
scheduler := MakeRandomScheduler(MakeMinionRegistry([]string{"m1", "m2", "m3", "m4"}), *random)
_, err := scheduler.Schedule(api.Pod{})
expectNoError(t, err)
}
func TestFirstFitSchedulerNothingScheduled(t *testing.T) {
mockRegistry := MockPodRegistry{}
r := rand.New(rand.NewSource(0))
scheduler := MakeFirstFitScheduler(MakeMinionRegistry([]string{"m1", "m2", "m3"}), &mockRegistry, r)
expectSchedule(scheduler, api.Pod{}, "m3", t)
}
func makePod(host string, hostPorts ...int) api.Pod {
networkPorts := []api.Port{}
for _, port := range hostPorts {
networkPorts = append(networkPorts, api.Port{HostPort: port})
}
return api.Pod{
CurrentState: api.PodState{
Host: host,
},
DesiredState: api.PodState{
Manifest: api.ContainerManifest{
Containers: []api.Container{
{
Ports: networkPorts,
},
},
},
},
}
}
func TestFirstFitSchedulerFirstScheduled(t *testing.T) {
mockRegistry := MockPodRegistry{
pods: []api.Pod{
makePod("m1", 8080),
},
}
r := rand.New(rand.NewSource(0))
scheduler := MakeFirstFitScheduler(MakeMinionRegistry([]string{"m1", "m2", "m3"}), &mockRegistry, r)
expectSchedule(scheduler, makePod("", 8080), "m3", t)
}
func TestFirstFitSchedulerFirstScheduledComplicated(t *testing.T) {
mockRegistry := MockPodRegistry{
pods: []api.Pod{
makePod("m1", 80, 8080),
makePod("m2", 8081, 8082, 8083),
makePod("m3", 80, 443, 8085),
},
}
r := rand.New(rand.NewSource(0))
scheduler := MakeFirstFitScheduler(MakeMinionRegistry([]string{"m1", "m2", "m3"}), &mockRegistry, r)
expectSchedule(scheduler, makePod("", 8080, 8081), "m3", t)
}
func TestFirstFitSchedulerFirstScheduledImpossible(t *testing.T) {
mockRegistry := MockPodRegistry{
pods: []api.Pod{
makePod("m1", 8080),
makePod("m2", 8081),
makePod("m3", 8080),
},
}
r := rand.New(rand.NewSource(0))
scheduler := MakeFirstFitScheduler(MakeMinionRegistry([]string{"m1", "m2", "m3"}), &mockRegistry, r)
_, err := scheduler.Schedule(makePod("", 8080, 8081))
if err == nil {
t.Error("Unexpected non-error.")
}
}

19
pkg/scheduler/doc.go Normal file
View File

@ -0,0 +1,19 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package scheduler contains a generic Scheduler interface and several
// implementations.
package scheduler

70
pkg/registry/scheduler.go → pkg/scheduler/firstfit.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
package scheduler
import (
"fmt"
@ -24,66 +24,16 @@ import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
)
// Scheduler is an interface implemented by things that know how to schedule pods onto machines.
type Scheduler interface {
Schedule(api.Pod) (string, error)
}
// RandomScheduler choses machines uniformly at random.
type RandomScheduler struct {
machines MinionRegistry
random rand.Rand
}
func MakeRandomScheduler(machines MinionRegistry, random rand.Rand) Scheduler {
return &RandomScheduler{
machines: machines,
random: random,
}
}
func (s *RandomScheduler) Schedule(pod api.Pod) (string, error) {
machines, err := s.machines.List()
if err != nil {
return "", err
}
return machines[s.random.Int()%len(machines)], nil
}
// RoundRobinScheduler chooses machines in order.
type RoundRobinScheduler struct {
machines MinionRegistry
currentIndex int
}
func MakeRoundRobinScheduler(machines MinionRegistry) Scheduler {
return &RoundRobinScheduler{
machines: machines,
currentIndex: -1,
}
}
func (s *RoundRobinScheduler) Schedule(pod api.Pod) (string, error) {
machines, err := s.machines.List()
if err != nil {
return "", err
}
s.currentIndex = (s.currentIndex + 1) % len(machines)
result := machines[s.currentIndex]
return result, nil
}
type FirstFitScheduler struct {
machines MinionRegistry
registry PodRegistry
random *rand.Rand
podLister PodLister
// TODO: *rand.Rand is *not* threadsafe
random *rand.Rand
}
func MakeFirstFitScheduler(machines MinionRegistry, registry PodRegistry, random *rand.Rand) Scheduler {
func MakeFirstFitScheduler(podLister PodLister, random *rand.Rand) Scheduler {
return &FirstFitScheduler{
machines: machines,
registry: registry,
random: random,
podLister: podLister,
random: random,
}
}
@ -98,13 +48,13 @@ func (s *FirstFitScheduler) containsPort(pod api.Pod, port api.Port) bool {
return false
}
func (s *FirstFitScheduler) Schedule(pod api.Pod) (string, error) {
machines, err := s.machines.List()
func (s *FirstFitScheduler) Schedule(pod api.Pod, minionLister MinionLister) (string, error) {
machines, err := minionLister.List()
if err != nil {
return "", err
}
machineToPods := map[string][]api.Pod{}
pods, err := s.registry.ListPods(labels.Everything())
pods, err := s.podLister.ListPods(labels.Everything())
if err != nil {
return "", err
}

78
pkg/scheduler/firstfit_test.go Normal file
View File

@ -0,0 +1,78 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheduler
import (
"math/rand"
"testing"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)
func TestFirstFitSchedulerNothingScheduled(t *testing.T) {
fakeRegistry := FakePodLister{}
r := rand.New(rand.NewSource(0))
st := schedulerTester{
t: t,
scheduler: MakeFirstFitScheduler(&fakeRegistry, r),
minionLister: FakeMinionLister{"m1", "m2", "m3"},
}
st.expectSchedule(api.Pod{}, "m3")
}
func TestFirstFitSchedulerFirstScheduled(t *testing.T) {
fakeRegistry := FakePodLister{
makePod("m1", 8080),
}
r := rand.New(rand.NewSource(0))
st := schedulerTester{
t: t,
scheduler: MakeFirstFitScheduler(fakeRegistry, r),
minionLister: FakeMinionLister{"m1", "m2", "m3"},
}
st.expectSchedule(makePod("", 8080), "m3")
}
func TestFirstFitSchedulerFirstScheduledComplicated(t *testing.T) {
fakeRegistry := FakePodLister{
makePod("m1", 80, 8080),
makePod("m2", 8081, 8082, 8083),
makePod("m3", 80, 443, 8085),
}
r := rand.New(rand.NewSource(0))
st := schedulerTester{
t: t,
scheduler: MakeFirstFitScheduler(fakeRegistry, r),
minionLister: FakeMinionLister{"m1", "m2", "m3"},
}
st.expectSchedule(makePod("", 8080, 8081), "m3")
}
func TestFirstFitSchedulerFirstScheduledImpossible(t *testing.T) {
fakeRegistry := FakePodLister{
makePod("m1", 8080),
makePod("m2", 8081),
makePod("m3", 8080),
}
r := rand.New(rand.NewSource(0))
st := schedulerTester{
t: t,
scheduler: MakeFirstFitScheduler(fakeRegistry, r),
minionLister: FakeMinionLister{"m1", "m2", "m3"},
}
st.expectFailure(makePod("", 8080, 8081))
}

52
pkg/scheduler/listers.go Normal file
View File

@ -0,0 +1,52 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheduler
import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
)
// Anything that can list minions for a scheduler.
type MinionLister interface {
List() (machines []string, err error)
}
// Make a MinionLister from a []string
type FakeMinionLister []string
// Returns minions as a []string
func (f FakeMinionLister) List() ([]string, error) {
return []string(f), nil
}
// Anything that can list pods for a scheduler
type PodLister interface {
ListPods(labels.Selector) ([]api.Pod, error)
}
// Make a MinionLister from an []api.Pods
type FakePodLister []api.Pod
func (f FakePodLister) ListPods(s labels.Selector) (selected []api.Pod, err error) {
for _, pod := range f {
if s.Matches(labels.Set(pod.Labels)) {
selected = append(selected, pod)
}
}
return selected, nil
}

43
pkg/scheduler/random.go Normal file
View File

@ -0,0 +1,43 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheduler
import (
"math/rand"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)
// RandomScheduler choses machines uniformly at random.
type RandomScheduler struct {
// TODO: rand.Rand is *NOT* thread safe.
random *rand.Rand
}
func MakeRandomScheduler(random *rand.Rand) Scheduler {
return &RandomScheduler{
random: random,
}
}
func (s *RandomScheduler) Schedule(pod api.Pod, minionLister MinionLister) (string, error) {
machines, err := minionLister.List()
if err != nil {
return "", err
}
return machines[s.random.Int()%len(machines)], nil
}

34
pkg/scheduler/random_test.go Normal file
View File

@ -0,0 +1,34 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheduler
import (
"math/rand"
"testing"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)
func TestRandomScheduler(t *testing.T) {
random := rand.New(rand.NewSource(0))
st := schedulerTester{
t: t,
scheduler: MakeRandomScheduler(random),
minionLister: FakeMinionLister{"m1", "m2", "m3", "m4"},
}
st.expectSuccess(api.Pod{})
}

42
pkg/scheduler/roundrobin.go Normal file
View File

@ -0,0 +1,42 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheduler
import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)
// RoundRobinScheduler chooses machines in order.
type RoundRobinScheduler struct {
currentIndex int
}
func MakeRoundRobinScheduler() Scheduler {
return &RoundRobinScheduler{
currentIndex: -1,
}
}
func (s *RoundRobinScheduler) Schedule(pod api.Pod, minionLister MinionLister) (string, error) {
machines, err := minionLister.List()
if err != nil {
return "", err
}
s.currentIndex = (s.currentIndex + 1) % len(machines)
result := machines[s.currentIndex]
return result, nil
}

35
pkg/scheduler/roundrobin_test.go Normal file
View File

@ -0,0 +1,35 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheduler
import (
"testing"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)
func TestRoundRobinScheduler(t *testing.T) {
st := schedulerTester{
t: t,
scheduler: MakeRoundRobinScheduler(),
minionLister: FakeMinionLister{"m1", "m2", "m3", "m4"},
}
st.expectSchedule(api.Pod{}, "m1")
st.expectSchedule(api.Pod{}, "m2")
st.expectSchedule(api.Pod{}, "m3")
st.expectSchedule(api.Pod{}, "m4")
}

27
pkg/scheduler/scheduler.go Normal file
View File

@ -0,0 +1,27 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheduler
import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)
// Scheduler is an interface implemented by things that know how to schedule pods
// onto machines.
type Scheduler interface {
Schedule(api.Pod, MinionLister) (selectedMachine string, err error)
}

81
pkg/scheduler/scheduler_test.go Normal file
View File

@ -0,0 +1,81 @@
/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package scheduler
import (
"testing"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)
// Some functions used by multiple scheduler tests.
type schedulerTester struct {
t *testing.T
scheduler Scheduler
minionLister MinionLister
}
// Call if you know exactly where pod should get scheduled.
func (st *schedulerTester) expectSchedule(pod api.Pod, expected string) {
actual, err := st.scheduler.Schedule(pod, st.minionLister)
if err != nil {
st.t.Errorf("Unexpected error %v\nTried to scheduler: %#v", err, pod)
return
}
if actual != expected {
st.t.Errorf("Unexpected scheduling value: %v, expected %v", actual, expected)
}
}
// Call if you can't predict where pod will be scheduled.
func (st *schedulerTester) expectSuccess(pod api.Pod) {
_, err := st.scheduler.Schedule(pod, st.minionLister)
if err != nil {
st.t.Errorf("Unexpected error %v\nTried to scheduler: %#v", err, pod)
return
}
}
// Call if pod should *not* schedule.
func (st *schedulerTester) expectFailure(pod api.Pod) {
_, err := st.scheduler.Schedule(pod, st.minionLister)
if err == nil {
st.t.Error("Unexpected non-error")
}
}
func makePod(host string, hostPorts ...int) api.Pod {
networkPorts := []api.Port{}
for _, port := range hostPorts {
networkPorts = append(networkPorts, api.Port{HostPort: port})
}
return api.Pod{
CurrentState: api.PodState{
Host: host,
},
DesiredState: api.PodState{
Manifest: api.ContainerManifest{
Containers: []api.Container{
{
Ports: networkPorts,
},
},
},
},
}
}

7
pkg/util/util.go
View File

@ -25,8 +25,15 @@ import (
"github.com/golang/glog"
)
// For testing, bypass HandleCrash.
var ReallyCrash bool
// Simply catches a crash and logs an error. Meant to be called via defer.
func HandleCrash() {
if ReallyCrash {
return
}
r := recover()
if r != nil {
callers := ""