mirror of https://github.com/k3s-io/k3s
Merge pull request #62856 from liggitt/node-authorizer-contention-benchmark
Automatic merge from submit-queue (batch tested with PRs 62409, 62856). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Add node authorizer contention benchmark * Makes the node authorization benchmark run in parallel * Runs the tests a second time with a background goroutine pushing graph modifications at a rate of 100x per second (to test authorization performance with contention on the graph lock). Graph modifications come from the informers watching objects relevant to node authorization, and only fire when a relevant change is made (for example, most node updates do not trigger a graph modification, only ones which change the node's config source configmap reference; most pod updates do not trigger a graph modification, only ones that set the pod's nodeName or uid) The results do not indicate bottlenecks in the authorizer, even under higher-than-expected write contention. ``` $ go test ./plugin/pkg/auth/authorizer/node/ -run foo -bench 'Authorization' -benchmem -v goos: darwin goarch: amd64 pkg: k8s.io/kubernetes/plugin/pkg/auth/authorizer/node BenchmarkAuthorization/allowed_node_configmap-8 596 ns/op 529 B/op 11 allocs/op 3000000 BenchmarkAuthorization/allowed_configmap-8 609 ns/op 529 B/op 11 allocs/op 3000000 BenchmarkAuthorization/allowed_secret_via_pod-8 586 ns/op 529 B/op 11 allocs/op 3000000 BenchmarkAuthorization/allowed_shared_secret_via_pod-8 18202 ns/op 542 B/op 11 allocs/op 100000 BenchmarkAuthorization/disallowed_node_configmap-8 900 ns/op 691 B/op 17 allocs/op 2000000 BenchmarkAuthorization/disallowed_configmap-8 868 ns/op 693 B/op 17 allocs/op 2000000 BenchmarkAuthorization/disallowed_secret_via_pod-8 875 ns/op 693 B/op 17 allocs/op 2000000 BenchmarkAuthorization/disallowed_shared_secret_via_pvc-8 1215 ns/op 948 B/op 22 allocs/op 1000000 BenchmarkAuthorization/disallowed_pvc-8 912 ns/op 693 B/op 17 allocs/op 2000000 BenchmarkAuthorization/disallowed_pv-8 1137 ns/op 834 B/op 19 allocs/op 2000000 BenchmarkAuthorization/disallowed_attachment_-_no_relationship-8 892 ns/op 677 B/op 16 allocs/op 2000000 BenchmarkAuthorization/disallowed_attachment_-_feature_disabled-8 236 ns/op 208 B/op 2 allocs/op 10000000 BenchmarkAuthorization/allowed_attachment_-_feature_enabled-8 723 ns/op 593 B/op 12 allocs/op 2000000 BenchmarkAuthorization/contentious_allowed_node_configmap-8 726 ns/op 529 B/op 11 allocs/op 2000000 BenchmarkAuthorization/contentious_allowed_configmap-8 698 ns/op 529 B/op 11 allocs/op 2000000 BenchmarkAuthorization/contentious_allowed_secret_via_pod-8 778 ns/op 529 B/op 11 allocs/op 2000000 BenchmarkAuthorization/contentious_allowed_shared_secret_via_pod-8 21406 ns/op 638 B/op 13 allocs/op 100000 BenchmarkAuthorization/contentious_disallowed_node_configmap-8 1135 ns/op 692 B/op 17 allocs/op 1000000 BenchmarkAuthorization/contentious_disallowed_configmap-8 1239 ns/op 691 B/op 17 allocs/op 1000000 BenchmarkAuthorization/contentious_disallowed_secret_via_pod-8 1043 ns/op 692 B/op 17 allocs/op 1000000 BenchmarkAuthorization/contentious_disallowed_shared_secret_via_pvc-8 1404 ns/op 950 B/op 22 allocs/op 1000000 BenchmarkAuthorization/contentious_disallowed_pvc-8 1177 ns/op 693 B/op 17 allocs/op 1000000 BenchmarkAuthorization/contentious_disallowed_pv-8 1295 ns/op 834 B/op 19 allocs/op 1000000 BenchmarkAuthorization/contentious_disallowed_attachment_-_no_relationship-8 1170 ns/op 676 B/op 16 allocs/op 1000000 BenchmarkAuthorization/contentious_disallowed_attachment_-_feature_disabled-8 262 ns/op 208 B/op 2 allocs/op 10000000 BenchmarkAuthorization/contentious_allowed_attachment_-_feature_enabled-8 790 ns/op 593 B/op 12 allocs/op 2000000 --- BENCH: BenchmarkAuthorization node_authorizer_test.go:592: graph modifications during non-contention test: 0 node_authorizer_test.go:589: graph modifications during contention test: 6301 node_authorizer_test.go:590: <1ms=5507, <10ms=128, <25ms=43, <50ms=65, <100ms=135, <250ms=328, <500ms=93, <1000ms=2, >1000ms=0 PASS ok k8s.io/kubernetes/plugin/pkg/auth/authorizer/node 112.616s ``` ```release-note NONE ```pull/8/head
Kubernetes Submit Queue
GitHub
commit
f0b207df2d
|
|
@ -20,7 +20,9 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"runtime"
|
"runtime"
|
||||||
"runtime/pprof"
|
"runtime/pprof"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
|
@ -405,6 +407,10 @@ func BenchmarkAuthorization(b *testing.B) {
|
||||||
g := NewGraph()
|
g := NewGraph()
|
||||||
|
|
||||||
opts := sampleDataOpts{
|
opts := sampleDataOpts{
|
||||||
|
// To simulate high replication in a small number of namespaces:
|
||||||
|
// nodes: 5000,
|
||||||
|
// namespaces: 10,
|
||||||
|
// podsPerNode: 10,
|
||||||
nodes: 500,
|
nodes: 500,
|
||||||
namespaces: 200,
|
namespaces: 200,
|
||||||
podsPerNode: 200,
|
podsPerNode: 200,
|
||||||
|
|
@ -502,20 +508,93 @@ func BenchmarkAuthorization(b *testing.B) {
|
||||||
}
|
}
|
||||||
|
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
for _, tc := range tests {
|
for _, testWriteContention := range []bool{false, true} {
|
||||||
if tc.features == nil {
|
|
||||||
authz.features = utilfeature.DefaultFeatureGate
|
shouldWrite := int32(1)
|
||||||
} else {
|
writes := int64(0)
|
||||||
authz.features = tc.features
|
_1ms := int64(0)
|
||||||
}
|
_10ms := int64(0)
|
||||||
b.Run(tc.name, func(b *testing.B) {
|
_25ms := int64(0)
|
||||||
for i := 0; i < b.N; i++ {
|
_50ms := int64(0)
|
||||||
decision, _, _ := authz.Authorize(tc.attrs)
|
_100ms := int64(0)
|
||||||
if decision != tc.expect {
|
_250ms := int64(0)
|
||||||
b.Errorf("expected %v, got %v", tc.expect, decision)
|
_500ms := int64(0)
|
||||||
|
_1000ms := int64(0)
|
||||||
|
_1s := int64(0)
|
||||||
|
|
||||||
|
contentionPrefix := ""
|
||||||
|
if testWriteContention {
|
||||||
|
contentionPrefix = "contentious "
|
||||||
|
// Start a writer pushing graph modifications 100x a second
|
||||||
|
go func() {
|
||||||
|
for shouldWrite == 1 {
|
||||||
|
go func() {
|
||||||
|
start := time.Now()
|
||||||
|
authz.graph.AddPod(&api.Pod{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Name: "testwrite", Namespace: "ns0"},
|
||||||
|
Spec: api.PodSpec{
|
||||||
|
NodeName: "node0",
|
||||||
|
ServiceAccountName: "default",
|
||||||
|
Volumes: []api.Volume{
|
||||||
|
{Name: "token", VolumeSource: api.VolumeSource{Secret: &api.SecretVolumeSource{SecretName: "secret0-shared"}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
diff := time.Now().Sub(start)
|
||||||
|
atomic.AddInt64(&writes, 1)
|
||||||
|
switch {
|
||||||
|
case diff < time.Millisecond:
|
||||||
|
atomic.AddInt64(&_1ms, 1)
|
||||||
|
case diff < 10*time.Millisecond:
|
||||||
|
atomic.AddInt64(&_10ms, 1)
|
||||||
|
case diff < 25*time.Millisecond:
|
||||||
|
atomic.AddInt64(&_25ms, 1)
|
||||||
|
case diff < 50*time.Millisecond:
|
||||||
|
atomic.AddInt64(&_50ms, 1)
|
||||||
|
case diff < 100*time.Millisecond:
|
||||||
|
atomic.AddInt64(&_100ms, 1)
|
||||||
|
case diff < 250*time.Millisecond:
|
||||||
|
atomic.AddInt64(&_250ms, 1)
|
||||||
|
case diff < 500*time.Millisecond:
|
||||||
|
atomic.AddInt64(&_500ms, 1)
|
||||||
|
case diff < 1000*time.Millisecond:
|
||||||
|
atomic.AddInt64(&_1000ms, 1)
|
||||||
|
default:
|
||||||
|
atomic.AddInt64(&_1s, 1)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
}
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
if tc.features == nil {
|
||||||
|
authz.features = utilfeature.DefaultFeatureGate
|
||||||
|
} else {
|
||||||
|
authz.features = tc.features
|
||||||
}
|
}
|
||||||
})
|
b.Run(contentionPrefix+tc.name, func(b *testing.B) {
|
||||||
|
// Run authorization checks in parallel
|
||||||
|
b.SetParallelism(5000)
|
||||||
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
for pb.Next() {
|
||||||
|
decision, _, _ := authz.Authorize(tc.attrs)
|
||||||
|
if decision != tc.expect {
|
||||||
|
b.Errorf("expected %v, got %v", tc.expect, decision)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
atomic.StoreInt32(&shouldWrite, 0)
|
||||||
|
if testWriteContention {
|
||||||
|
b.Logf("graph modifications during contention test: %d", writes)
|
||||||
|
b.Logf("<1ms=%d, <10ms=%d, <25ms=%d, <50ms=%d, <100ms=%d, <250ms=%d, <500ms=%d, <1000ms=%d, >1000ms=%d", _1ms, _10ms, _25ms, _50ms, _100ms, _250ms, _500ms, _1000ms, _1s)
|
||||||
|
} else {
|
||||||
|
b.Logf("graph modifications during non-contention test: %d", writes)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue