github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #31326 from feiskyer/kuberuntime-gc 

Automatic merge from submit-queue

Kubelet: add garbage collection for new runtime API

This PR adds garbage collection for new runtime API.

Note that this PR also adds `CreatedAt` and `PodSandboxId` to `ListContainers()` result.

CC @yujuhong @Random-Liu  @kubernetes/sig-node @kubernetes/sig-rktnetes
pull/6/head
Kubernetes Submit Queue 2016-09-15 02:28:05 -07:00 committed by GitHub
parent e0345a1b1a f774a68d52
commit 265746af18
11 changed files with 707 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/kubelet/api/testing/fake_runtime_service.go
View File

@ -313,12 +313,14 @@ func (r *FakeRuntimeService) ListContainers(filter *runtimeApi.ContainerFilter)
} }
result = append(result, &runtimeApi.Container{ result = append(result, &runtimeApi.Container{
Id: s.Id, Id: s.Id,
Metadata: s.Metadata, CreatedAt: s.CreatedAt,
State: s.State, PodSandboxId: &s.SandboxID,
Image: s.Image, Metadata: s.Metadata,
ImageRef: s.ImageRef, State: s.State,
Labels: s.Labels, Image: s.Image,
ImageRef: s.ImageRef,
Labels: s.Labels,
}) })
} }

385
pkg/kubelet/api/v1alpha1/runtime/api.pb.go
View File

@ -1722,20 +1722,24 @@ type Container struct {
// The ID of the container, used by the container runtime to identify // The ID of the container, used by the container runtime to identify
// a container. // a container.
Id *string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` Id *string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
// The id of the sandbox which this container belongs to.
PodSandboxId *string `protobuf:"bytes,2,opt,name=pod_sandbox_id,json=podSandboxId" json:"pod_sandbox_id,omitempty"`
// The metadata of the container. // The metadata of the container.
Metadata *ContainerMetadata `protobuf:"bytes,2,opt,name=metadata" json:"metadata,omitempty"` Metadata *ContainerMetadata `protobuf:"bytes,3,opt,name=metadata" json:"metadata,omitempty"`
// The spec of the image // The spec of the image
Image *ImageSpec `protobuf:"bytes,3,opt,name=image" json:"image,omitempty"` Image *ImageSpec `protobuf:"bytes,4,opt,name=image" json:"image,omitempty"`
// Reference to the image in use. For most runtimes, this should be an // Reference to the image in use. For most runtimes, this should be an
// image ID. // image ID.
ImageRef *string `protobuf:"bytes,4,opt,name=image_ref,json=imageRef" json:"image_ref,omitempty"` ImageRef *string `protobuf:"bytes,5,opt,name=image_ref,json=imageRef" json:"image_ref,omitempty"`
// State is the state of the container. // State is the state of the container.
State *ContainerState `protobuf:"varint,5,opt,name=state,enum=runtime.ContainerState" json:"state,omitempty"` State *ContainerState `protobuf:"varint,6,opt,name=state,enum=runtime.ContainerState" json:"state,omitempty"`
// Creation time of the container.
CreatedAt *int64 `protobuf:"varint,7,opt,name=created_at,json=createdAt" json:"created_at,omitempty"`
// Labels are key value pairs that may be used to scope and select individual resources. // Labels are key value pairs that may be used to scope and select individual resources.
Labels map[string]string `protobuf:"bytes,6,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` Labels map[string]string `protobuf:"bytes,8,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
// Annotations is an unstructured key value map that may be set by external // Annotations is an unstructured key value map that may be set by external
// tools to store and retrieve arbitrary metadata. // tools to store and retrieve arbitrary metadata.
Annotations map[string]string `protobuf:"bytes,7,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` Annotations map[string]string `protobuf:"bytes,9,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
XXX_unrecognized []byte `json:"-"` XXX_unrecognized []byte `json:"-"`
} }
@ -1751,6 +1755,13 @@ func (m *Container) GetId() string {
return "" return ""
} }
func (m *Container) GetPodSandboxId() string {
if m != nil && m.PodSandboxId != nil {
return *m.PodSandboxId
}
return ""
}
func (m *Container) GetMetadata() *ContainerMetadata { func (m *Container) GetMetadata() *ContainerMetadata {
if m != nil { if m != nil {
return m.Metadata return m.Metadata
@ -1779,6 +1790,13 @@ func (m *Container) GetState() ContainerState {
return ContainerState_CREATED return ContainerState_CREATED
} }
func (m *Container) GetCreatedAt() int64 {
if m != nil && m.CreatedAt != nil {
return *m.CreatedAt
}
return 0
}
func (m *Container) GetLabels() map[string]string { func (m *Container) GetLabels() map[string]string {
if m != nil { if m != nil {
return m.Labels return m.Labels
@ -3097,182 +3115,183 @@ var _ImageService_serviceDesc = grpc.ServiceDesc{
} }
var fileDescriptorApi = []byte{ var fileDescriptorApi = []byte{
// 2828 bytes of a gzipped FileDescriptorProto // 2842 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x72, 0x1b, 0xc7, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x72, 0x1b, 0xc7,
0xf1, 0x17, 0x08, 0x82, 0x04, 0x1a, 0x04, 0x08, 0x8e, 0x28, 0x12, 0x82, 0x64, 0x89, 0x5a, 0xdb, 0x11, 0x16, 0x00, 0x82, 0x04, 0x1a, 0x04, 0x08, 0x8e, 0x28, 0x12, 0x82, 0x64, 0x89, 0x5a, 0xdb,
0xff, 0xbf, 0x44, 0x5b, 0x2c, 0x85, 0x49, 0x49, 0x91, 0x6c, 0xcb, 0xa6, 0x49, 0x46, 0x45, 0x4b, 0x89, 0x44, 0x5b, 0x2c, 0x85, 0x49, 0x49, 0x91, 0x6c, 0xcb, 0xa6, 0x49, 0x46, 0x45, 0x4b, 0xa2,
0xa2, 0x98, 0x85, 0xa4, 0x58, 0x27, 0xd4, 0x0a, 0x3b, 0x24, 0x57, 0x02, 0x76, 0xd7, 0xbb, 0x0b, 0x98, 0x85, 0xa4, 0x58, 0x27, 0xd4, 0x0a, 0x3b, 0x24, 0x57, 0x02, 0x76, 0xd7, 0xbb, 0x0b, 0x46,
0x46, 0xcc, 0x35, 0xd7, 0x5c, 0x72, 0xcb, 0x03, 0xb8, 0x2a, 0xa7, 0x5c, 0x52, 0x95, 0x4b, 0x6e, 0xcc, 0x35, 0x97, 0x1c, 0x72, 0xc9, 0x2d, 0x0f, 0xe0, 0xaa, 0x9c, 0x72, 0x49, 0x55, 0x2e, 0xb9,
0xa9, 0x1c, 0x53, 0x79, 0x90, 0x3c, 0x41, 0x72, 0x4c, 0xcf, 0xc7, 0xce, 0xce, 0x7e, 0x51, 0xa4, 0xe5, 0x9c, 0xca, 0x83, 0xe4, 0x09, 0x92, 0x63, 0x7a, 0x7e, 0x76, 0x76, 0xf6, 0x07, 0x34, 0x28,
0x5c, 0x15, 0xeb, 0xb6, 0xd3, 0xdd, 0x33, 0xd3, 0xd3, 0xdd, 0xd3, 0xd3, 0xbf, 0x99, 0x85, 0x86, 0x57, 0xd9, 0xba, 0xed, 0x74, 0xf7, 0xcc, 0xf4, 0x74, 0xf7, 0xf4, 0xf4, 0x37, 0xb3, 0x50, 0xb7,
0xe5, 0x3b, 0x6b, 0x7e, 0xe0, 0x45, 0x1e, 0x99, 0x0d, 0x26, 0x6e, 0xe4, 0x8c, 0xa9, 0xb1, 0x0a, 0x7c, 0x67, 0xdd, 0x0f, 0xbc, 0xc8, 0x23, 0x73, 0xc1, 0xd8, 0x8d, 0x9c, 0x11, 0x35, 0xd6, 0xa0,
0xed, 0xe7, 0x34, 0x08, 0x1d, 0xcf, 0x35, 0xe9, 0x77, 0x13, 0x1a, 0x46, 0xa4, 0x0b, 0xb3, 0x47, 0xf5, 0x9c, 0x06, 0xa1, 0xe3, 0xb9, 0x26, 0xfd, 0x66, 0x4c, 0xc3, 0x88, 0x74, 0x60, 0xee, 0x58,
0x82, 0xd2, 0xad, 0xac, 0x54, 0xae, 0x37, 0xcc, 0xb8, 0x69, 0xfc, 0xb1, 0x02, 0xf3, 0x4a, 0x38, 0x50, 0x3a, 0xa5, 0xd5, 0xd2, 0xf5, 0xba, 0x19, 0x37, 0x8d, 0xbf, 0x94, 0x60, 0x41, 0x09, 0x87,
0xf4, 0x3d, 0x37, 0xa4, 0xe5, 0xd2, 0xe4, 0x1a, 0xcc, 0xc9, 0x49, 0x06, 0xae, 0x35, 0xa6, 0xdd, 0xbe, 0xe7, 0x86, 0x74, 0xb2, 0x34, 0xb9, 0x06, 0xf3, 0x72, 0x92, 0xbe, 0x6b, 0x8d, 0x68, 0xa7,
0x29, 0xce, 0x6e, 0x4a, 0xda, 0x2e, 0x92, 0xc8, 0xff, 0xc3, 0x7c, 0x2c, 0x12, 0x0f, 0x52, 0xe5, 0xcc, 0xd9, 0x0d, 0x49, 0xdb, 0x43, 0x12, 0xf9, 0x29, 0x2c, 0xc4, 0x22, 0xf1, 0x20, 0x15, 0x2e,
0x52, 0x6d, 0x49, 0x96, 0xb3, 0x91, 0x35, 0x38, 0x1f, 0x0b, 0xe2, 0x1a, 0x94, 0xf0, 0x34, 0x17, 0xd5, 0x92, 0x64, 0x39, 0x1b, 0x59, 0x87, 0xf3, 0xb1, 0x20, 0xae, 0x41, 0x09, 0xcf, 0x70, 0xe1,
0x5e, 0x90, 0xac, 0x0d, 0xdf, 0x91, 0xf2, 0xc6, 0x06, 0x34, 0xb6, 0x76, 0xfb, 0x4f, 0xfc, 0x88, 0x45, 0xc9, 0xda, 0xf4, 0x1d, 0x29, 0x6f, 0x6c, 0x42, 0x7d, 0x7b, 0xaf, 0xf7, 0xc4, 0x8f, 0x58,
0x75, 0x46, 0x15, 0x43, 0x1a, 0xb0, 0x3e, 0xa8, 0x62, 0x95, 0xa9, 0x28, 0x9b, 0xa4, 0x07, 0xf5, 0x67, 0x54, 0x31, 0xa4, 0x01, 0xeb, 0x83, 0x2a, 0x56, 0x98, 0x8a, 0xb2, 0x49, 0xba, 0x50, 0x0b,
0x90, 0x5a, 0xc1, 0xf0, 0x90, 0x86, 0xa8, 0x1e, 0x63, 0xa9, 0xb6, 0xf1, 0xa7, 0x0a, 0x34, 0xf7, 0xa9, 0x15, 0x0c, 0x8e, 0x68, 0x88, 0xea, 0x31, 0x96, 0x6a, 0x1b, 0x7f, 0x2d, 0x41, 0x63, 0xdf,
0xbc, 0x20, 0x7a, 0x6c, 0xf9, 0xbe, 0xe3, 0x1e, 0x10, 0x02, 0xd3, 0x7c, 0x19, 0x62, 0x95, 0xfc, 0x0b, 0xa2, 0xc7, 0x96, 0xef, 0x3b, 0xee, 0x21, 0x21, 0x30, 0xc3, 0x97, 0x21, 0x56, 0xc9, 0xbf,
0x9b, 0xdc, 0x84, 0x3a, 0x37, 0xe7, 0xd0, 0x1b, 0xf1, 0xe5, 0xb5, 0xd7, 0x17, 0xd6, 0xa4, 0x32, 0xc9, 0x4d, 0xa8, 0x71, 0x73, 0x0e, 0xbc, 0x21, 0x5f, 0x5e, 0x6b, 0x63, 0x71, 0x5d, 0x2a, 0xb3,
0x6b, 0x7b, 0x92, 0x61, 0x2a, 0x11, 0xf2, 0x31, 0xb4, 0x87, 0x9e, 0x1b, 0x59, 0x8e, 0x4b, 0x83, 0xbe, 0x2f, 0x19, 0xa6, 0x12, 0x21, 0x1f, 0x42, 0x6b, 0xe0, 0xb9, 0x91, 0xe5, 0xb8, 0x34, 0xe8,
0x81, 0x8f, 0x63, 0xf3, 0xd5, 0xd6, 0xcc, 0x96, 0xa2, 0xb2, 0x09, 0xc9, 0x25, 0x68, 0x1c, 0x7a, 0xfb, 0x38, 0x36, 0x5f, 0x6d, 0xd5, 0x6c, 0x2a, 0x2a, 0x9b, 0x90, 0x5c, 0x82, 0xfa, 0x91, 0x17,
0x61, 0x24, 0x24, 0xa6, 0xb9, 0x44, 0x9d, 0x11, 0x38, 0x73, 0x19, 0x66, 0x39, 0xd3, 0xf1, 0xbb, 0x46, 0x42, 0x62, 0x86, 0x4b, 0xd4, 0x18, 0x81, 0x33, 0x57, 0x60, 0x8e, 0x33, 0x1d, 0xbf, 0x53,
0x35, 0xae, 0xc9, 0x0c, 0x6b, 0xee, 0xf8, 0xc6, 0xf7, 0x15, 0xa8, 0x3d, 0xf6, 0x70, 0xf2, 0x42, 0xe5, 0x9a, 0xcc, 0xb2, 0xe6, 0xae, 0x6f, 0x7c, 0x5b, 0x82, 0xea, 0x63, 0x0f, 0x27, 0x2f, 0xd4,
0x4d, 0xd3, 0x53, 0x5b, 0xd1, 0xa1, 0x74, 0x87, 0x36, 0x35, 0x12, 0x93, 0xa9, 0x99, 0x84, 0x70, 0x34, 0x3d, 0xb5, 0x15, 0x1d, 0x49, 0x77, 0x68, 0x53, 0x23, 0x31, 0x99, 0x9a, 0x49, 0x08, 0x57,
0x85, 0x98, 0x9a, 0x31, 0xd1, 0x5a, 0x01, 0xb5, 0x6c, 0xcf, 0x1d, 0x1d, 0x73, 0xb5, 0xea, 0xa6, 0x88, 0xa9, 0x19, 0x13, 0xad, 0x15, 0x50, 0xcb, 0xf6, 0xdc, 0xe1, 0x09, 0x57, 0xab, 0x66, 0xaa,
0x6a, 0x33, 0x4f, 0x86, 0x74, 0xe4, 0xb8, 0x93, 0x37, 0x83, 0x80, 0x8e, 0xac, 0x97, 0x74, 0xc4, 0x36, 0xf3, 0x64, 0x48, 0x87, 0x8e, 0x3b, 0x7e, 0xd3, 0x0f, 0xe8, 0xd0, 0x7a, 0x49, 0x87, 0x5c,
0xd5, 0xab, 0x9b, 0x6d, 0x49, 0x36, 0x05, 0xd5, 0x78, 0x05, 0xf3, 0xcc, 0xf5, 0xa1, 0x6f, 0x0d, 0xbd, 0x9a, 0xd9, 0x92, 0x64, 0x53, 0x50, 0x8d, 0x57, 0xb0, 0xc0, 0x5c, 0x1f, 0xfa, 0xd6, 0x80,
0xa9, 0xf4, 0x0f, 0x06, 0x0a, 0x9f, 0xd4, 0xa5, 0xd1, 0xaf, 0xbd, 0xe0, 0x35, 0xd7, 0xbb, 0x6e, 0x4a, 0xff, 0x60, 0xa0, 0xf0, 0x49, 0x5d, 0x1a, 0xfd, 0xd6, 0x0b, 0x5e, 0x73, 0xbd, 0x6b, 0x66,
0x36, 0x19, 0x6d, 0x57, 0x90, 0xc8, 0x45, 0xa8, 0x0b, 0xbd, 0x1c, 0x9b, 0x2b, 0x5e, 0x37, 0xb9, 0x83, 0xd1, 0xf6, 0x04, 0x89, 0x5c, 0x84, 0x9a, 0xd0, 0xcb, 0xb1, 0xb9, 0xe2, 0x35, 0x93, 0x5b,
0x15, 0xf6, 0x1c, 0x5b, 0xb1, 0x1c, 0x7f, 0xc8, 0x35, 0x96, 0xac, 0x1d, 0x7f, 0x68, 0xfc, 0xb6, 0x61, 0xdf, 0xb1, 0x15, 0xcb, 0xf1, 0x07, 0x5c, 0x63, 0xc9, 0xda, 0xf5, 0x07, 0xc6, 0xef, 0x4b,
0x02, 0x17, 0x1e, 0xb1, 0xc9, 0xf7, 0x3c, 0xbb, 0x6f, 0xb9, 0xf6, 0x4b, 0xef, 0xcd, 0xa6, 0xe7, 0x70, 0xe1, 0x11, 0x9b, 0x7c, 0xdf, 0xb3, 0x7b, 0x96, 0x6b, 0xbf, 0xf4, 0xde, 0x6c, 0x79, 0xee,
0xee, 0x3b, 0x07, 0xe4, 0x43, 0x68, 0x0d, 0x0f, 0x02, 0x6f, 0xe2, 0xe3, 0x4a, 0x03, 0xea, 0x46, 0x81, 0x73, 0x48, 0xde, 0x87, 0xe6, 0xe0, 0x30, 0xf0, 0xc6, 0x3e, 0xae, 0x34, 0xa0, 0x6e, 0x24,
0xd2, 0x56, 0x73, 0x82, 0xb8, 0xc7, 0x69, 0x64, 0x1b, 0x16, 0xdc, 0x58, 0xd5, 0x81, 0xc7, 0x75, 0x6d, 0x35, 0x2f, 0x88, 0xfb, 0x9c, 0x46, 0x76, 0x60, 0xd1, 0x8d, 0x55, 0xed, 0x7b, 0x5c, 0xd7,
0x0d, 0xf9, 0xec, 0xcd, 0xf5, 0xae, 0x72, 0x73, 0x66, 0x31, 0x66, 0xc7, 0x4d, 0x13, 0x42, 0x23, 0x90, 0xcf, 0xde, 0xd8, 0xe8, 0x28, 0x37, 0x67, 0x16, 0x63, 0xb6, 0xdd, 0x34, 0x21, 0x34, 0x02,
0x00, 0x92, 0xcc, 0xff, 0x98, 0x46, 0x96, 0x6d, 0x45, 0x56, 0xa1, 0x93, 0x3a, 0x50, 0x9d, 0xc8, 0x20, 0xc9, 0xfc, 0x8f, 0x69, 0x64, 0xd9, 0x56, 0x64, 0x15, 0x3a, 0xa9, 0x0d, 0x95, 0xb1, 0x5c,
0x05, 0x36, 0x4c, 0xf6, 0x49, 0x2e, 0x43, 0x43, 0x8d, 0x27, 0xfd, 0x91, 0x10, 0x58, 0x60, 0x5b, 0x60, 0xdd, 0x64, 0x9f, 0xe4, 0x32, 0xd4, 0xd5, 0x78, 0xd2, 0x1f, 0x09, 0x81, 0x05, 0xb6, 0x15,
0x51, 0x44, 0xc7, 0xbe, 0x08, 0x93, 0x96, 0x19, 0x37, 0x8d, 0xbf, 0x4e, 0x43, 0x27, 0xb7, 0xe8, 0x45, 0x74, 0xe4, 0x8b, 0x30, 0x69, 0x9a, 0x71, 0xd3, 0xf8, 0xc7, 0x0c, 0xb4, 0x73, 0x8b, 0xbe,
0x3b, 0x50, 0x1f, 0xcb, 0xe9, 0xf9, 0xb4, 0xcd, 0xf5, 0x4b, 0x49, 0xb4, 0xe6, 0x34, 0x34, 0x95, 0x03, 0xb5, 0x91, 0x9c, 0x9e, 0x4f, 0xdb, 0xd8, 0xb8, 0x94, 0x44, 0x6b, 0x4e, 0x43, 0x53, 0x09,
0x30, 0x73, 0x3c, 0x33, 0xa9, 0xb6, 0x8b, 0x55, 0x9b, 0x59, 0x72, 0xe4, 0x1d, 0x0c, 0x6c, 0x27, 0x33, 0xc7, 0x33, 0x93, 0x6a, 0xbb, 0x58, 0xb5, 0x99, 0x25, 0x87, 0xde, 0x61, 0xdf, 0x76, 0x02,
0xa0, 0xc3, 0xc8, 0x0b, 0x8e, 0xa5, 0x96, 0x73, 0x48, 0xdc, 0x8a, 0x69, 0xe4, 0xa7, 0xd0, 0xb4, 0x3a, 0x88, 0xbc, 0xe0, 0x44, 0x6a, 0x39, 0x8f, 0xc4, 0xed, 0x98, 0x46, 0x7e, 0x0e, 0x0d, 0xdb,
0xdd, 0x50, 0xd9, 0x70, 0x9a, 0x4f, 0x4e, 0xd4, 0xe4, 0x6a, 0xab, 0x9a, 0x80, 0x62, 0xd2, 0x6e, 0x0d, 0x95, 0x0d, 0x67, 0xf8, 0xe4, 0x44, 0x4d, 0xae, 0xb6, 0xaa, 0x09, 0x28, 0x26, 0xed, 0x46,
0xe4, 0x2e, 0xb4, 0xd8, 0x0e, 0x18, 0x8c, 0xc5, 0x06, 0x0c, 0x31, 0xa0, 0xaa, 0xd8, 0x6d, 0x51, 0xee, 0x42, 0x93, 0xed, 0x80, 0xfe, 0x48, 0x6c, 0xc0, 0x10, 0x03, 0xaa, 0x82, 0xdd, 0x96, 0x34,
0xd3, 0x59, 0xed, 0x4e, 0x73, 0xce, 0x4f, 0x1a, 0x21, 0xf9, 0x02, 0x66, 0x78, 0xb4, 0x85, 0xdd, 0x9d, 0xd5, 0xee, 0x34, 0xe7, 0xfd, 0xa4, 0x11, 0x92, 0xcf, 0x60, 0x96, 0x47, 0x5b, 0xd8, 0x99,
0x19, 0xde, 0xe7, 0xe3, 0x82, 0x75, 0x0a, 0xa3, 0xac, 0x3d, 0xe2, 0x72, 0xdb, 0x6e, 0x14, 0x1c, 0xe5, 0x7d, 0x3e, 0x2c, 0x58, 0xa7, 0x30, 0xca, 0xfa, 0x23, 0x2e, 0xb7, 0xe3, 0x46, 0xc1, 0x89,
0x9b, 0xb2, 0x13, 0x79, 0x04, 0x4d, 0xcb, 0x75, 0xbd, 0xc8, 0x12, 0xea, 0xce, 0xf2, 0x31, 0x56, 0x29, 0x3b, 0x91, 0x47, 0xd0, 0xb0, 0x5c, 0xd7, 0x8b, 0x2c, 0xa1, 0xee, 0x1c, 0x1f, 0x63, 0x6d,
0xcb, 0xc7, 0xd8, 0x48, 0x84, 0xc5, 0x40, 0x7a, 0x77, 0xf2, 0x33, 0xa8, 0xf1, 0x1d, 0xd0, 0xad, 0xf2, 0x18, 0x9b, 0x89, 0xb0, 0x18, 0x48, 0xef, 0x4e, 0x7e, 0x01, 0x55, 0xbe, 0x03, 0x3a, 0x35,
0xf3, 0x65, 0x5f, 0x51, 0xe3, 0x14, 0x86, 0xa6, 0x29, 0x84, 0x7b, 0x77, 0xa1, 0xa9, 0xa9, 0xc6, 0xbe, 0xec, 0x2b, 0x6a, 0x9c, 0xc2, 0xd0, 0x34, 0x85, 0x70, 0xf7, 0x2e, 0x34, 0x34, 0xd5, 0x58,
0x42, 0xe3, 0x35, 0x3d, 0x96, 0xd1, 0xc2, 0x3e, 0xc9, 0x22, 0xd4, 0x8e, 0xac, 0xd1, 0x24, 0xf6, 0x68, 0xbc, 0xa6, 0x27, 0x32, 0x5a, 0xd8, 0x27, 0x59, 0x82, 0xea, 0xb1, 0x35, 0x1c, 0xc7, 0x1e,
0x88, 0x68, 0xdc, 0x9b, 0xfa, 0x79, 0xa5, 0x77, 0x1f, 0x3a, 0x59, 0x8d, 0xce, 0xd2, 0xdf, 0xd8, 0x11, 0x8d, 0x7b, 0xe5, 0x5f, 0x96, 0xba, 0xf7, 0xa1, 0x9d, 0xd5, 0xe8, 0x2c, 0xfd, 0x8d, 0x5d,
0x81, 0x45, 0x73, 0xe2, 0x26, 0x8a, 0xc5, 0x07, 0xc3, 0x4f, 0x60, 0x66, 0xc8, 0x75, 0x94, 0xd1, 0x58, 0x32, 0xc7, 0x6e, 0xa2, 0x58, 0x7c, 0x30, 0xfc, 0x0c, 0x66, 0x07, 0x5c, 0x47, 0x19, 0x3d,
0x73, 0xb1, 0xd4, 0x22, 0xa6, 0x14, 0x34, 0xbe, 0x80, 0x0b, 0x99, 0xa1, 0xe4, 0xb1, 0xf1, 0x11, 0x17, 0x27, 0x5a, 0xc4, 0x94, 0x82, 0xc6, 0x67, 0x70, 0x21, 0x33, 0x94, 0x3c, 0x36, 0x3e, 0x80,
0xb4, 0x7d, 0xcf, 0x1e, 0x84, 0x82, 0x3c, 0xc0, 0xa8, 0x97, 0x3b, 0xd0, 0x57, 0xb2, 0x3b, 0x36, 0x96, 0xef, 0xd9, 0xfd, 0x50, 0x90, 0xfb, 0x18, 0xf5, 0x72, 0x07, 0xfa, 0x4a, 0x76, 0xd7, 0x66,
0xeb, 0xde, 0x8f, 0x3c, 0x3f, 0xaf, 0xca, 0xe9, 0xba, 0x77, 0x61, 0x29, 0xdb, 0x5d, 0x4c, 0x6f, 0xdd, 0x7b, 0x91, 0xe7, 0xe7, 0x55, 0x99, 0xae, 0x7b, 0x07, 0x96, 0xb3, 0xdd, 0xc5, 0xf4, 0xc6,
0x7c, 0x09, 0xcb, 0x26, 0x1d, 0x7b, 0x47, 0xf4, 0x5d, 0x87, 0xee, 0x41, 0x37, 0x3f, 0x40, 0x32, 0xe7, 0xb0, 0x62, 0xd2, 0x91, 0x77, 0x4c, 0xdf, 0x76, 0xe8, 0x2e, 0x74, 0xf2, 0x03, 0x24, 0x83,
0x78, 0x42, 0xed, 0xa3, 0x1b, 0x26, 0xe1, 0xd9, 0x06, 0xbf, 0xa1, 0x0f, 0x20, 0x53, 0xa0, 0x18, 0x27, 0xd4, 0x1e, 0xba, 0x61, 0x1c, 0x9e, 0x6d, 0xf0, 0x1b, 0xfa, 0x00, 0x32, 0x05, 0x8a, 0x71,
0x87, 0xb4, 0x61, 0x0a, 0x33, 0xbf, 0xe8, 0x84, 0x5f, 0xc6, 0x0b, 0x68, 0xec, 0xea, 0xf9, 0x40, 0x48, 0x0b, 0xca, 0x98, 0xf9, 0x45, 0x27, 0xfc, 0x32, 0x5e, 0x40, 0x7d, 0x4f, 0xcf, 0x07, 0x7a,
0xcf, 0xa1, 0x78, 0xd0, 0xc9, 0x26, 0x59, 0x87, 0xd9, 0xd3, 0x26, 0xb0, 0x58, 0xd0, 0x78, 0x98, 0x0e, 0xc5, 0x83, 0x4e, 0x36, 0xc9, 0x06, 0xcc, 0x4d, 0x9b, 0xc0, 0x62, 0x41, 0xe3, 0x61, 0x2e,
0x4b, 0x9e, 0x52, 0x87, 0x75, 0x00, 0x95, 0x83, 0x42, 0x19, 0x0b, 0x24, 0x3f, 0x9e, 0xa9, 0x49, 0x79, 0x4a, 0x1d, 0x36, 0x00, 0x54, 0x0e, 0x0a, 0x65, 0x2c, 0x90, 0xfc, 0x78, 0xa6, 0x26, 0x65,
0x19, 0xdf, 0xa7, 0x12, 0x92, 0xb6, 0x18, 0x5b, 0x2d, 0xc6, 0x4e, 0x25, 0xa8, 0xa9, 0xb3, 0x24, 0x7c, 0x9b, 0x4a, 0x48, 0xda, 0x62, 0x6c, 0xb5, 0x18, 0x3b, 0x95, 0xa0, 0xca, 0x67, 0x49, 0x50,
0xa8, 0x35, 0xa8, 0x85, 0x38, 0xa4, 0x48, 0x91, 0x6d, 0x6d, 0x71, 0xb2, 0xd7, 0xd7, 0x62, 0x4a, 0xeb, 0x50, 0x0d, 0x71, 0x48, 0x91, 0x22, 0x5b, 0xda, 0xe2, 0x64, 0xaf, 0x2f, 0xc5, 0x94, 0xd4,
0x6a, 0x0a, 0x31, 0xf2, 0x01, 0xc0, 0x10, 0x8f, 0xae, 0x88, 0xda, 0x03, 0x4b, 0xe4, 0xce, 0xaa, 0x14, 0x62, 0xe4, 0x3d, 0x80, 0x01, 0x1e, 0x5d, 0x11, 0xb5, 0xfb, 0x96, 0xc8, 0x9d, 0x15, 0xb3,
0xd9, 0x90, 0x94, 0x8d, 0x88, 0xdc, 0x4b, 0xec, 0x58, 0xe3, 0x6a, 0xac, 0x14, 0xa8, 0x91, 0xf2, 0x2e, 0x29, 0x9b, 0x11, 0xb9, 0x97, 0xd8, 0xb1, 0xca, 0xd5, 0x58, 0x2d, 0x50, 0x23, 0xe5, 0x97,
0x4b, 0x62, 0x69, 0xb5, 0xdb, 0x67, 0x4e, 0xde, 0xed, 0xb2, 0x9f, 0x10, 0xd6, 0x12, 0xd6, 0x6c, 0xc4, 0xd2, 0x6a, 0xb7, 0xcf, 0x9e, 0xbe, 0xdb, 0x65, 0x3f, 0x21, 0xac, 0x25, 0xac, 0xb9, 0x89,
0x69, 0xc2, 0x12, 0x3d, 0x4e, 0x93, 0xb0, 0xea, 0xa5, 0x09, 0x4b, 0x8e, 0x71, 0x62, 0xc2, 0xfa, 0x09, 0x4b, 0xf4, 0x98, 0x26, 0x61, 0xd5, 0x26, 0x26, 0x2c, 0x39, 0xc6, 0xa9, 0x09, 0xeb, 0xc7,
0x31, 0x53, 0xcf, 0x63, 0xe8, 0xe6, 0xb7, 0x8e, 0x4c, 0x19, 0x98, 0x7e, 0x42, 0x4e, 0x39, 0x21, 0x4c, 0x3d, 0x8f, 0xa1, 0x93, 0xdf, 0x3a, 0x32, 0x65, 0x60, 0xfa, 0x09, 0x39, 0xe5, 0x94, 0xf4,
0xfd, 0xc8, 0x2e, 0x52, 0xd0, 0xf8, 0x57, 0x45, 0x8f, 0xba, 0x5f, 0x38, 0xa3, 0x88, 0x06, 0xb9, 0x23, 0xbb, 0x48, 0x41, 0xe3, 0x3f, 0x25, 0x3d, 0xea, 0x7e, 0xe5, 0x0c, 0x23, 0x1a, 0xe4, 0xa2,
0xa8, 0x53, 0xc1, 0x33, 0x75, 0xba, 0xe0, 0xe9, 0x43, 0x9b, 0x9b, 0x7d, 0x80, 0x95, 0x0d, 0x3f, 0x4e, 0x05, 0x4f, 0x79, 0xba, 0xe0, 0xe9, 0x41, 0x8b, 0x9b, 0xbd, 0x8f, 0x95, 0x0d, 0x3f, 0xdf,
0xdf, 0x30, 0xea, 0x98, 0xbd, 0x3f, 0x2d, 0xd0, 0x47, 0x4c, 0x29, 0x7c, 0xd6, 0x97, 0xe2, 0xc2, 0x30, 0xea, 0x98, 0xbd, 0x3f, 0x2e, 0xd0, 0x47, 0x4c, 0x29, 0x7c, 0xd6, 0x93, 0xe2, 0xc2, 0xe2,
0xe2, 0xad, 0x91, 0x4e, 0xeb, 0x7d, 0x05, 0x24, 0x2f, 0x74, 0x26, 0xd3, 0x7d, 0xc3, 0xb6, 0x2b, 0xcd, 0xa1, 0x4e, 0xeb, 0x7e, 0x01, 0x24, 0x2f, 0x74, 0x26, 0xd3, 0x7d, 0xc5, 0xb6, 0x2b, 0x2b,
0x2b, 0x12, 0x0b, 0xd2, 0xf6, 0x3e, 0x57, 0xe3, 0x04, 0xbb, 0x09, 0x3d, 0x4d, 0x29, 0x68, 0xfc, 0x12, 0x0b, 0xd2, 0xf6, 0x01, 0x57, 0xe3, 0x14, 0xbb, 0x09, 0x3d, 0x4d, 0x29, 0x68, 0xfc, 0xb9,
0x61, 0x0a, 0x20, 0x61, 0xbe, 0xb7, 0xfb, 0xf4, 0x8e, 0xda, 0x35, 0xa2, 0x34, 0xb8, 0x5a, 0xa0, 0x0c, 0x90, 0x30, 0xdf, 0xd9, 0x7d, 0x7a, 0x47, 0xed, 0x1a, 0x51, 0x1a, 0x5c, 0x2d, 0xd0, 0xa2,
0x45, 0xd1, 0x7e, 0xf9, 0x01, 0x11, 0x6e, 0x6c, 0xc2, 0x52, 0xd6, 0xcc, 0x32, 0x3e, 0x6f, 0x40, 0x68, 0xbf, 0x7c, 0x8f, 0x08, 0x37, 0xb6, 0x60, 0x39, 0x6b, 0x66, 0x19, 0x9f, 0x37, 0xa0, 0xea,
0xcd, 0xc1, 0xea, 0x4b, 0x80, 0x8c, 0xe6, 0xfa, 0xf9, 0x02, 0x65, 0x4c, 0x21, 0x61, 0x5c, 0x83, 0x60, 0xf5, 0x25, 0x40, 0x46, 0x63, 0xe3, 0x7c, 0x81, 0x32, 0xa6, 0x90, 0x30, 0xae, 0x41, 0x7d,
0xc6, 0xce, 0xd8, 0x3a, 0xa0, 0x7d, 0x9f, 0x0e, 0xd9, 0x5c, 0x0e, 0x6b, 0xc8, 0xf9, 0x45, 0xc3, 0x77, 0x64, 0x1d, 0xd2, 0x9e, 0x4f, 0x07, 0x6c, 0x2e, 0x87, 0x35, 0xe4, 0xfc, 0xa2, 0x61, 0x6c,
0x58, 0x87, 0xfa, 0x43, 0x7a, 0xfc, 0x9c, 0xcd, 0x7b, 0x5a, 0xfd, 0x8c, 0x7f, 0x54, 0x60, 0x99, 0x40, 0xed, 0x21, 0x3d, 0x79, 0xce, 0xe6, 0x9d, 0x56, 0x3f, 0xe3, 0x5f, 0x25, 0x58, 0xe1, 0x69,
0xa7, 0x99, 0xcd, 0xb8, 0xa8, 0x47, 0xe5, 0xbc, 0x49, 0x80, 0x09, 0x98, 0x9b, 0xd2, 0x9f, 0x0c, 0x66, 0x2b, 0x2e, 0xea, 0x51, 0x39, 0x6f, 0x1c, 0x60, 0x02, 0xe6, 0xa6, 0xf4, 0xc7, 0x7d, 0x9f,
0x7c, 0x1a, 0x38, 0x9e, 0xf0, 0x25, 0x33, 0xa5, 0x3f, 0xd9, 0xe3, 0x04, 0x56, 0xf8, 0x33, 0xf6, 0x06, 0x8e, 0x27, 0x7c, 0xc9, 0x4c, 0xe9, 0x8f, 0xf7, 0x39, 0x81, 0x15, 0xfe, 0x8c, 0xfd, 0xcd,
0x77, 0x13, 0x4f, 0xfa, 0xb4, 0x6a, 0xd6, 0x91, 0xf0, 0x4b, 0xd6, 0x8e, 0xfb, 0x86, 0x87, 0x58, 0xd8, 0x93, 0x3e, 0xad, 0x98, 0x35, 0x24, 0xfc, 0x9a, 0xb5, 0xe3, 0xbe, 0xe1, 0x11, 0xd6, 0xc5,
0x17, 0x87, 0xdc, 0x77, 0xa2, 0x6f, 0x9f, 0x13, 0x30, 0xc0, 0x2e, 0x8c, 0xf1, 0x2c, 0x0c, 0x8e, 0x21, 0xf7, 0x9d, 0xe8, 0xdb, 0xe3, 0x04, 0x0c, 0xb0, 0x0b, 0x23, 0x3c, 0x0b, 0x83, 0x93, 0xfe,
0x07, 0x23, 0x67, 0xec, 0x60, 0x25, 0xee, 0x0e, 0x5e, 0x1e, 0x47, 0x34, 0x94, 0x0e, 0x23, 0x82, 0xd0, 0x19, 0x39, 0x58, 0x89, 0xbb, 0xfd, 0x97, 0x27, 0x11, 0x0d, 0xa5, 0xc3, 0x88, 0x60, 0x3e,
0xf9, 0x88, 0xf1, 0x76, 0xdc, 0xaf, 0x19, 0x87, 0x18, 0xd0, 0xf2, 0xbc, 0xf1, 0x20, 0x1c, 0x7a, 0x62, 0xbc, 0x5d, 0xf7, 0x4b, 0xc6, 0x21, 0x06, 0x34, 0x3d, 0x6f, 0xd4, 0x0f, 0x07, 0x5e, 0x80,
0x01, 0x22, 0x3a, 0xfb, 0x15, 0xcf, 0xb3, 0x55, 0xb3, 0x89, 0xc4, 0x3e, 0xa3, 0x6d, 0xd8, 0xaf, 0x88, 0xce, 0x7e, 0xc5, 0xf3, 0x6c, 0xc5, 0x6c, 0x20, 0xb1, 0xc7, 0x68, 0x9b, 0xf6, 0x2b, 0xc3,
0x0c, 0x0b, 0x5a, 0xfd, 0x6d, 0xbe, 0x1c, 0x89, 0x13, 0xb0, 0x64, 0x9e, 0x84, 0x32, 0x8c, 0xb1, 0x82, 0x66, 0x6f, 0x87, 0x2f, 0x47, 0xe2, 0x04, 0x2c, 0x99, 0xc7, 0xa1, 0x0c, 0x63, 0x2c, 0x99,
0x64, 0x66, 0xdf, 0x8c, 0x16, 0x78, 0xa3, 0xd8, 0x0e, 0xfc, 0x9b, 0xd1, 0xa2, 0x63, 0x3f, 0xae, 0xd9, 0x37, 0xa3, 0x05, 0xde, 0x30, 0xb6, 0x03, 0xff, 0x66, 0xb4, 0xe8, 0xc4, 0x8f, 0xeb, 0x65,
0x97, 0xf9, 0x37, 0x33, 0xd8, 0x88, 0x1e, 0x21, 0x2a, 0x11, 0x90, 0x51, 0x34, 0x0c, 0x1b, 0x60, 0xfe, 0xcd, 0x0c, 0x36, 0xa4, 0xc7, 0x88, 0x4a, 0x04, 0x64, 0x14, 0x0d, 0xc3, 0x06, 0xd8, 0xb2,
0xd3, 0xf2, 0xad, 0x97, 0xce, 0xc8, 0x89, 0x8e, 0xd1, 0x81, 0x1d, 0xcb, 0xb6, 0x07, 0xc3, 0x98, 0x7c, 0xeb, 0xa5, 0x33, 0x74, 0xa2, 0x13, 0x74, 0x60, 0xdb, 0xb2, 0xed, 0xfe, 0x20, 0xa6, 0x38,
0xe2, 0xd0, 0x18, 0x30, 0xce, 0x23, 0x7d, 0x53, 0x23, 0x93, 0x4f, 0x60, 0xc1, 0x0e, 0x3c, 0x3f, 0x34, 0x06, 0x8c, 0x0b, 0x48, 0xdf, 0xd2, 0xc8, 0xe4, 0x23, 0x58, 0xb4, 0x03, 0xcf, 0x4f, 0xcb,
0x2d, 0x2b, 0x10, 0x64, 0x87, 0x31, 0x74, 0x61, 0xe3, 0x3f, 0x15, 0x58, 0x4c, 0xbb, 0x45, 0x16, 0x0a, 0x04, 0xd9, 0x66, 0x0c, 0x5d, 0xd8, 0xf8, 0x5f, 0x09, 0x96, 0xd2, 0x6e, 0x91, 0x05, 0xf9,
0xe4, 0xf7, 0xa1, 0x11, 0xc4, 0x0e, 0x92, 0x9b, 0x73, 0x25, 0x7d, 0x5e, 0xe4, 0x1d, 0x69, 0x26, 0x7d, 0xa8, 0x07, 0xb1, 0x83, 0xe4, 0xe6, 0x5c, 0x4d, 0x9f, 0x17, 0x79, 0x47, 0x9a, 0x49, 0x17,
0x5d, 0x30, 0xfe, 0xe7, 0x32, 0x0a, 0x54, 0x52, 0x81, 0x97, 0xac, 0xcd, 0x4c, 0x09, 0x92, 0x2f, 0x8c, 0xff, 0xf9, 0x8c, 0x02, 0xa5, 0x54, 0xe0, 0x25, 0x6b, 0x33, 0x53, 0x82, 0xe4, 0xf3, 0x04,
0x13, 0xb4, 0x16, 0x97, 0x05, 0x55, 0xde, 0x77, 0x49, 0xf5, 0x4d, 0x99, 0x5e, 0xa1, 0xb8, 0xb8, 0xad, 0xc5, 0x65, 0x41, 0x85, 0xf7, 0x5d, 0x56, 0x7d, 0x53, 0xa6, 0x57, 0x28, 0x2e, 0xae, 0xcd,
0x36, 0xff, 0x3f, 0xe9, 0x8a, 0x6c, 0x25, 0xcf, 0xfb, 0x3c, 0x43, 0x8e, 0x70, 0x8f, 0xf1, 0x2d, 0x7f, 0x22, 0x5d, 0x91, 0xad, 0xe4, 0x79, 0x9f, 0x67, 0xc8, 0x11, 0xee, 0x31, 0xbe, 0x86, 0xba,
0x34, 0x14, 0x29, 0x86, 0x37, 0x22, 0xf6, 0x38, 0xbc, 0x41, 0xca, 0x81, 0x04, 0x3c, 0x48, 0xc1, 0x22, 0xc5, 0xf0, 0x46, 0xc4, 0x1e, 0x87, 0x37, 0x48, 0x39, 0x94, 0x80, 0x07, 0x29, 0xf8, 0xc9,
0x4f, 0x86, 0x23, 0xd1, 0xd6, 0x0e, 0x9b, 0xc5, 0x1a, 0x0d, 0x90, 0x12, 0xf2, 0xec, 0x5a, 0x35, 0x70, 0x24, 0xda, 0xda, 0x61, 0xb3, 0x58, 0xc3, 0x3e, 0x52, 0x42, 0x9e, 0x5d, 0x2b, 0x66, 0x2b,
0xdb, 0x09, 0xf9, 0x01, 0x52, 0x11, 0xe1, 0x2f, 0x28, 0xe3, 0x9c, 0x08, 0xaa, 0x34, 0x90, 0x34, 0x21, 0x3f, 0x40, 0x2a, 0x22, 0xfc, 0x45, 0x65, 0x9c, 0x53, 0x41, 0x95, 0x06, 0x92, 0xca, 0x69,
0x95, 0x06, 0x49, 0xff, 0xae, 0xc1, 0x7c, 0xd6, 0x25, 0xb7, 0x73, 0x18, 0xa9, 0x97, 0x98, 0x33, 0x90, 0xf4, 0xdf, 0x2a, 0x2c, 0x64, 0x5d, 0x72, 0x3b, 0x87, 0x91, 0xba, 0x89, 0x39, 0xb3, 0xf3,
0x3b, 0x9f, 0x96, 0xd9, 0xae, 0xc7, 0x9b, 0x78, 0x2a, 0x63, 0x11, 0xb5, 0xcf, 0xe5, 0xc6, 0x66, 0x69, 0x99, 0xed, 0x7a, 0xbc, 0x89, 0xcb, 0x19, 0x8b, 0xa8, 0x7d, 0x2e, 0x37, 0x36, 0xd3, 0x67,
0xfa, 0x0c, 0xbd, 0xf1, 0x18, 0x13, 0x02, 0x5f, 0x19, 0x16, 0x69, 0xb2, 0xc9, 0xb4, 0xb7, 0x82, 0xe0, 0x8d, 0x46, 0x98, 0x10, 0xf8, 0xca, 0xb0, 0x48, 0x93, 0x4d, 0xa6, 0xbd, 0x15, 0x1c, 0xb2,
0x03, 0xb6, 0x6d, 0x18, 0x99, 0x7f, 0x93, 0xab, 0xd0, 0x64, 0x65, 0x05, 0xa2, 0x1a, 0x06, 0xb1, 0x6d, 0xc3, 0xc8, 0xfc, 0x9b, 0x5c, 0x85, 0x06, 0x2b, 0x2b, 0x10, 0xd5, 0x30, 0x88, 0x25, 0x21,
0x24, 0xe4, 0x07, 0x49, 0x42, 0x80, 0x85, 0xc0, 0x7e, 0x9a, 0xba, 0x47, 0x31, 0xd0, 0x49, 0xae, 0x3f, 0x48, 0x12, 0x02, 0x2c, 0x04, 0xf6, 0x33, 0xd4, 0x3d, 0x8e, 0x81, 0x4e, 0x72, 0xfd, 0x10,
0x1f, 0xe2, 0xe4, 0x61, 0x72, 0x36, 0x3a, 0x6c, 0x66, 0xcc, 0x2e, 0x07, 0xe2, 0x02, 0xa3, 0xad, 0x27, 0x0f, 0x93, 0xb3, 0xd1, 0x61, 0xb3, 0x23, 0x76, 0x39, 0x10, 0x17, 0x18, 0x2d, 0x25, 0xc8,
0x04, 0xf9, 0x9d, 0x81, 0x29, 0xb9, 0xe4, 0x73, 0x95, 0x52, 0x45, 0x11, 0xf1, 0x51, 0x7e, 0xf5, 0xef, 0x0c, 0x4c, 0xc9, 0x25, 0x9f, 0xaa, 0x94, 0x2a, 0x8a, 0x88, 0x0f, 0xf2, 0xab, 0x3f, 0x05,
0x27, 0x00, 0xa7, 0x87, 0xe9, 0x3a, 0xa4, 0xc1, 0x87, 0xb8, 0x51, 0x3a, 0xc4, 0xc9, 0xb8, 0xe9, 0x38, 0x3d, 0x4c, 0xd7, 0x21, 0x75, 0x3e, 0xc4, 0x8d, 0x89, 0x43, 0x9c, 0x8e, 0x9b, 0xae, 0x00,
0x0a, 0x80, 0x1f, 0x38, 0x47, 0xce, 0x88, 0x1e, 0x50, 0xbb, 0x0b, 0x1c, 0xda, 0x6b, 0x14, 0x7e, 0xf8, 0x81, 0x73, 0xec, 0x0c, 0xe9, 0x21, 0xb5, 0x3b, 0xc0, 0xa1, 0xbd, 0x46, 0xe1, 0x97, 0x47,
0x79, 0x24, 0xaf, 0x1f, 0x06, 0x81, 0xe7, 0x45, 0xfb, 0x61, 0xb7, 0x29, 0xae, 0x1c, 0x62, 0xb2, 0xf2, 0xfa, 0xa1, 0x1f, 0x78, 0x5e, 0x74, 0x10, 0x76, 0x1a, 0xe2, 0xca, 0x21, 0x26, 0x9b, 0x9c,
0xc9, 0xa9, 0xec, 0x86, 0x80, 0x41, 0x54, 0x7e, 0xa7, 0x31, 0x27, 0xea, 0x62, 0x6c, 0xf3, 0x2b, 0xca, 0x6e, 0x08, 0x18, 0x44, 0xe5, 0x77, 0x1a, 0xf3, 0xa2, 0x2e, 0xc6, 0x36, 0xbf, 0xd2, 0x58,
0x8d, 0x45, 0x76, 0x20, 0xd9, 0x8e, 0xdb, 0x6d, 0xf1, 0x9e, 0xa2, 0xc1, 0xf2, 0x1d, 0xff, 0x18, 0x62, 0x07, 0x92, 0xed, 0xb8, 0x9d, 0x26, 0xef, 0x29, 0x1a, 0x2c, 0xdf, 0xf1, 0x8f, 0xbe, 0xe7,
0x78, 0x2e, 0xc2, 0xee, 0x36, 0x67, 0x35, 0x38, 0xe5, 0x09, 0x12, 0x58, 0xd4, 0x46, 0xd1, 0x71, 0x22, 0xec, 0x6e, 0x71, 0x56, 0x9d, 0x53, 0x9e, 0x20, 0x81, 0x45, 0x6d, 0x14, 0x9d, 0x74, 0x16,
0x77, 0x9e, 0xd3, 0xd9, 0x27, 0xe2, 0x5b, 0x59, 0xf4, 0x75, 0xb8, 0xf7, 0x3f, 0x28, 0xd9, 0xc4, 0x38, 0x9d, 0x7d, 0x22, 0xbe, 0x95, 0x45, 0x5f, 0x9b, 0x7b, 0xff, 0xbd, 0x09, 0x9b, 0xf8, 0x9d,
0xef, 0x0d, 0xc2, 0xfb, 0x73, 0x05, 0x96, 0x36, 0xf9, 0x31, 0xaa, 0x25, 0x98, 0x33, 0x20, 0x14, 0x41, 0x78, 0x7f, 0x2b, 0xc1, 0xf2, 0x16, 0x3f, 0x46, 0xb5, 0x04, 0x73, 0x06, 0x84, 0x42, 0x6e,
0x72, 0x4b, 0x41, 0xc1, 0x2c, 0x9c, 0xc8, 0x2e, 0x56, 0xca, 0x91, 0xaf, 0xa0, 0x1d, 0x8f, 0x29, 0x29, 0x28, 0x98, 0x85, 0x13, 0xd9, 0xc5, 0x4a, 0x39, 0xf2, 0x05, 0xb4, 0xe2, 0x31, 0x65, 0xcf,
0x7b, 0x56, 0xdf, 0x06, 0x22, 0x5b, 0xa1, 0xde, 0x34, 0x3e, 0x87, 0xe5, 0x9c, 0xce, 0xf2, 0xe8, 0xca, 0x77, 0x81, 0xc8, 0x66, 0xa8, 0x37, 0x8d, 0x4f, 0x61, 0x25, 0xa7, 0xb3, 0x3c, 0x7a, 0xaf,
0xbd, 0x86, 0x89, 0x50, 0xdd, 0x6e, 0x29, 0x95, 0x9b, 0x8a, 0x86, 0x98, 0xea, 0x1e, 0x83, 0x92, 0x61, 0x22, 0x54, 0xb7, 0x5b, 0x4a, 0xe5, 0x86, 0xa2, 0x21, 0xa6, 0xba, 0xc7, 0xa0, 0xa4, 0x15,
0x56, 0x10, 0xe5, 0x16, 0x7c, 0x8a, 0xbe, 0x1c, 0x47, 0xa6, 0xfb, 0x4a, 0xa8, 0xd7, 0x87, 0x45, 0x44, 0xb9, 0x05, 0x4f, 0xd1, 0x97, 0xe3, 0xc8, 0x74, 0x5f, 0x09, 0xf5, 0x7a, 0xb0, 0xc4, 0x10,
0x86, 0x30, 0xdf, 0x61, 0x50, 0x96, 0x07, 0xd8, 0xb2, 0xbd, 0x49, 0x24, 0xf3, 0x5f, 0xdc, 0x34, 0xe6, 0x5b, 0x0c, 0xca, 0xf2, 0x00, 0x5b, 0xb6, 0x37, 0x8e, 0x64, 0xfe, 0x8b, 0x9b, 0xc6, 0x8a,
0x96, 0x05, 0xea, 0xcd, 0xcf, 0xf6, 0x19, 0x2c, 0x09, 0xd0, 0xf9, 0x2e, 0x8b, 0xb8, 0x18, 0x43, 0x40, 0xbd, 0xf9, 0xd9, 0x3e, 0x81, 0x65, 0x01, 0x3a, 0xdf, 0x66, 0x11, 0x17, 0x63, 0xc8, 0x9b,
0xde, 0xfc, 0xb8, 0xbf, 0x9b, 0xd2, 0x12, 0x61, 0x49, 0x95, 0x7c, 0x33, 0x5d, 0x25, 0x2f, 0xe7, 0x1f, 0xf7, 0x8f, 0x65, 0x2d, 0x11, 0x4e, 0xa8, 0x92, 0x6f, 0xa6, 0xab, 0xe4, 0x95, 0xbc, 0xc3,
0x1d, 0x9e, 0xaa, 0xdc, 0xf2, 0x61, 0x54, 0x2d, 0x08, 0x23, 0x33, 0x57, 0x4a, 0x4f, 0xf3, 0x94, 0x53, 0x95, 0x5b, 0x3e, 0x8c, 0x2a, 0x05, 0x61, 0x64, 0xe6, 0x4a, 0xe9, 0x19, 0x9e, 0x32, 0x3e,
0xf1, 0x49, 0x7e, 0xf4, 0xff, 0x61, 0x25, 0xbd, 0x23, 0x2a, 0x69, 0x35, 0xb5, 0x42, 0xef, 0xb7, 0xca, 0x8f, 0xfe, 0x03, 0x56, 0xd2, 0xbb, 0xa2, 0x92, 0x56, 0x53, 0x2b, 0xf4, 0x7e, 0x2b, 0x53,
0x32, 0x95, 0x74, 0xb7, 0x4c, 0x4d, 0x55, 0x48, 0xff, 0xbd, 0x0a, 0x0d, 0xc5, 0xcb, 0xd9, 0xf4, 0x49, 0x77, 0x26, 0xa9, 0xa9, 0x0a, 0xe9, 0x3f, 0xcc, 0x40, 0x5d, 0xf1, 0x72, 0x36, 0xcd, 0x1b,
0x76, 0xae, 0x8e, 0x3e, 0xe3, 0x61, 0x53, 0x7d, 0xdb, 0x61, 0x83, 0x65, 0x1d, 0xff, 0x18, 0x04, 0xa9, 0x5c, 0x60, 0x24, 0xfd, 0x48, 0xaa, 0xbc, 0xcd, 0x91, 0x34, 0xf3, 0x5d, 0x47, 0x12, 0x16,
0x74, 0x5f, 0x96, 0x3e, 0x75, 0x4e, 0x30, 0xe9, 0x7e, 0xe2, 0xd2, 0xda, 0xa9, 0x5c, 0x7a, 0x3b, 0x7f, 0xfc, 0xa3, 0x1f, 0xd0, 0x03, 0x79, 0xc4, 0xd4, 0x38, 0xc1, 0xa4, 0x07, 0x89, 0xe3, 0x67,
0x73, 0xa9, 0x76, 0x25, 0x2f, 0x5f, 0x78, 0x28, 0x6c, 0x17, 0xdd, 0xa6, 0x7d, 0x58, 0xd0, 0xf9, 0xa7, 0x72, 0x7c, 0xba, 0x64, 0x9f, 0xcb, 0x96, 0xec, 0xb7, 0x33, 0xe7, 0xcb, 0x95, 0xfc, 0x70,
0xbd, 0x45, 0xa5, 0x8f, 0x44, 0xcd, 0xaf, 0x07, 0x84, 0x4c, 0x3c, 0xeb, 0x58, 0x19, 0x2b, 0xaa, 0x85, 0x27, 0xcb, 0x4e, 0xd1, 0xc9, 0xf2, 0x7e, 0x41, 0xe7, 0x77, 0x16, 0xda, 0x3e, 0x12, 0xc0,
0x2c, 0xfc, 0x49, 0x7e, 0x69, 0xa6, 0x26, 0xc5, 0x76, 0x71, 0xca, 0xc0, 0xc9, 0xed, 0xd0, 0x29, 0x41, 0x8f, 0x2a, 0x99, 0xbd, 0x36, 0xd0, 0x64, 0x8a, 0x2a, 0xd1, 0x03, 0xc9, 0x2f, 0xcd, 0xd4,
0x76, 0xf1, 0xef, 0xf5, 0x9a, 0xa5, 0xe4, 0x1a, 0xe5, 0x5d, 0xc3, 0xea, 0x66, 0x1a, 0x9d, 0xbd, 0xa4, 0x58, 0x2a, 0x48, 0xd9, 0x3f, 0xb9, 0x62, 0x9a, 0x22, 0x15, 0xfc, 0x49, 0x2f, 0x7c, 0x26,
0x2d, 0x1e, 0xde, 0x02, 0xce, 0xf8, 0x21, 0x8a, 0x49, 0x53, 0xb0, 0x45, 0x7d, 0xdf, 0x90, 0x14, 0xdc, 0xc5, 0xdc, 0xce, 0x61, 0xbc, 0xe9, 0xa2, 0xee, 0x66, 0x1a, 0xe2, 0x9d, 0x2d, 0x5c, 0x72,
0x64, 0x63, 0x61, 0xb3, 0xef, 0xb8, 0x4e, 0x78, 0x28, 0xf8, 0x33, 0x9c, 0x0f, 0x31, 0x69, 0x83, 0x08, 0x8f, 0x9f, 0xc4, 0x98, 0x79, 0x05, 0x5b, 0x80, 0x84, 0xba, 0xa4, 0x20, 0x1b, 0xab, 0xa3,
0xbf, 0x82, 0xd0, 0x37, 0x88, 0x26, 0x86, 0x9e, 0x4d, 0x31, 0x68, 0xf8, 0x2b, 0x08, 0x23, 0x6c, 0x03, 0xc7, 0x75, 0xc2, 0x23, 0xc1, 0x9f, 0xe5, 0x7c, 0x88, 0x49, 0x9b, 0xfc, 0x29, 0x85, 0xbe,
0x62, 0x3b, 0xd9, 0x01, 0xf5, 0x33, 0xed, 0x80, 0x46, 0x66, 0x07, 0x2c, 0xc1, 0x0c, 0xea, 0x1b, 0x41, 0x48, 0x32, 0xf0, 0x6c, 0xca, 0x83, 0xb1, 0x6a, 0xd6, 0x18, 0x61, 0x0b, 0xdb, 0xc9, 0x06,
0x7a, 0x2e, 0x2f, 0x2f, 0x1a, 0xa6, 0x6c, 0x69, 0x55, 0x50, 0xb3, 0xac, 0x0a, 0x3a, 0xe1, 0x36, 0xa9, 0x9d, 0x69, 0x83, 0xd4, 0x33, 0x1b, 0x64, 0x19, 0x66, 0x51, 0xdf, 0xd0, 0x73, 0x79, 0x8d,
0x26, 0x53, 0x05, 0xcd, 0x95, 0x55, 0x41, 0xa7, 0xb9, 0x8c, 0xd1, 0x0a, 0xb7, 0xd6, 0x49, 0x85, 0x52, 0x37, 0x65, 0x4b, 0x2b, 0xa5, 0x1a, 0x93, 0x4a, 0xa9, 0x53, 0xae, 0x74, 0x32, 0xa5, 0xd4,
0xdb, 0x8f, 0xb9, 0x3d, 0x1e, 0xe2, 0xc1, 0x9c, 0x0d, 0x68, 0xb9, 0x3f, 0x6e, 0x65, 0xee, 0x6c, 0xfc, 0xa4, 0x52, 0x6a, 0x9a, 0x1b, 0x1d, 0xad, 0xfa, 0x6b, 0x9e, 0x56, 0xfd, 0xfd, 0x98, 0xdb,
0xba, 0x65, 0x56, 0x50, 0x57, 0x36, 0xaf, 0xa0, 0xb9, 0xfd, 0x06, 0xdd, 0x77, 0xfa, 0x83, 0x14, 0xe3, 0x21, 0x9e, 0xee, 0xd9, 0x80, 0x96, 0xfb, 0xe3, 0x56, 0xe6, 0xe2, 0xa7, 0x33, 0xc9, 0x0a,
0x55, 0x1d, 0x8e, 0x6d, 0x89, 0xbe, 0xd8, 0x67, 0x5c, 0xa0, 0x55, 0x93, 0x02, 0x4d, 0xd5, 0x79, 0xea, 0xde, 0xe7, 0x15, 0x34, 0x76, 0xde, 0xa0, 0xfb, 0xa6, 0x3f, 0x8d, 0x51, 0xd5, 0xc1, 0xc8,
0x2c, 0x4c, 0xe7, 0x64, 0x9d, 0x67, 0xdc, 0x87, 0x39, 0x31, 0x97, 0xd4, 0x76, 0x89, 0x69, 0x6b, 0x96, 0x10, 0x8e, 0x7d, 0xc6, 0x55, 0x5e, 0x25, 0xa9, 0xf2, 0x54, 0xb1, 0xc8, 0xc2, 0x74, 0x5e,
0xb3, 0x13, 0xb9, 0xc2, 0xc5, 0x64, 0x4b, 0xd2, 0x69, 0x10, 0xf0, 0xb5, 0x0b, 0x3a, 0xb6, 0x8c, 0x16, 0x8b, 0xc6, 0x7d, 0x98, 0x17, 0x73, 0x49, 0x6d, 0x97, 0x99, 0xb6, 0x36, 0x3b, 0xd6, 0x4b,
0x3b, 0xd0, 0xe4, 0xf1, 0x26, 0x8f, 0xcc, 0xeb, 0x3a, 0x90, 0x3f, 0x29, 0x28, 0x19, 0x78, 0x61, 0x5c, 0x4c, 0xb6, 0x24, 0x9d, 0x06, 0x01, 0x5f, 0xbb, 0xa0, 0x63, 0xcb, 0xb8, 0x03, 0x0d, 0x1e,
0x09, 0x85, 0xd3, 0xd5, 0xee, 0xff, 0x34, 0x73, 0xba, 0x2c, 0xa6, 0xfb, 0x67, 0x4e, 0x96, 0xd7, 0x6f, 0xf2, 0xdc, 0xbd, 0xae, 0xdf, 0x06, 0x9c, 0x16, 0x94, 0x0c, 0x01, 0xb1, 0x84, 0xc2, 0xe9,
0x50, 0xe3, 0xe4, 0xdc, 0xee, 0xbf, 0xc4, 0x40, 0xa5, 0xef, 0x0d, 0x22, 0xeb, 0x40, 0x3d, 0x6a, 0x6a, 0xf7, 0x7f, 0x9c, 0x39, 0xa2, 0x96, 0xd2, 0xfd, 0x33, 0xc7, 0xd3, 0x6b, 0xa8, 0x72, 0x72,
0x32, 0xc2, 0x53, 0x6c, 0xf3, 0x37, 0x59, 0xc6, 0xb4, 0x1d, 0x9c, 0x38, 0x0a, 0x25, 0x02, 0x69, 0x6e, 0xf7, 0x5f, 0x62, 0xc8, 0xd4, 0xf7, 0xfa, 0x91, 0x75, 0xa8, 0x5e, 0x46, 0x19, 0xe1, 0x29,
0x32, 0xda, 0x96, 0x20, 0x31, 0x14, 0x12, 0x3a, 0xbf, 0xa1, 0xdc, 0x52, 0xd3, 0x26, 0xff, 0xc6, 0xb6, 0xf9, 0xc3, 0x2e, 0x63, 0xda, 0x0e, 0x4e, 0x1c, 0x85, 0x12, 0xc6, 0x34, 0x18, 0x6d, 0x5b,
0xd2, 0x8b, 0xe8, 0xfa, 0x4a, 0x73, 0x61, 0x68, 0xf2, 0xe5, 0xc4, 0x89, 0xaf, 0x9d, 0x56, 0xd8, 0x90, 0x18, 0x94, 0x09, 0x9d, 0xdf, 0x89, 0xa3, 0x67, 0xc6, 0xe4, 0xdf, 0x58, 0xbf, 0x11, 0x5d,
0x94, 0x5c, 0x34, 0x33, 0x11, 0x16, 0x48, 0x25, 0xbb, 0xd3, 0x5b, 0xeb, 0x33, 0x38, 0x9f, 0xea, 0x5f, 0x69, 0x2e, 0x0c, 0x4d, 0xbe, 0x9c, 0x38, 0xf1, 0xb5, 0xd2, 0x0a, 0x9b, 0x92, 0x8b, 0x66,
0xaf, 0x9e, 0x10, 0x52, 0x03, 0x64, 0x67, 0x97, 0x9d, 0xff, 0x59, 0x01, 0xd8, 0x98, 0x44, 0x87, 0x26, 0xc2, 0x02, 0xa9, 0x64, 0x37, 0xbd, 0xb5, 0x3e, 0x81, 0xf3, 0xa9, 0xfe, 0xea, 0x1d, 0x22,
0x12, 0xdf, 0xf5, 0xa0, 0xce, 0x80, 0xa9, 0x86, 0x12, 0x55, 0x9b, 0xf1, 0x7c, 0x2b, 0x0c, 0x11, 0x35, 0x40, 0x76, 0x76, 0xd9, 0xf9, 0xdf, 0x25, 0x80, 0xcd, 0x71, 0x74, 0x24, 0x41, 0x62, 0x17,
0x5c, 0xc5, 0x6f, 0x70, 0xaa, 0xcd, 0xb1, 0xd9, 0x44, 0xbd, 0x89, 0xf2, 0x6f, 0xf6, 0xa6, 0x2a, 0x6a, 0x0c, 0xdd, 0x6a, 0x50, 0x53, 0xb5, 0x19, 0xcf, 0xb7, 0xc2, 0x10, 0x11, 0x5a, 0x7c, 0x9a,
0x1e, 0x92, 0x07, 0x88, 0x4d, 0x11, 0x95, 0x87, 0xf2, 0x84, 0x6d, 0x09, 0xea, 0x86, 0x20, 0x32, 0xab, 0x36, 0x07, 0x78, 0x63, 0xf5, 0xb0, 0xca, 0xbf, 0xd9, 0xc3, 0xac, 0x78, 0x8d, 0xee, 0x23,
0x31, 0xc7, 0xa6, 0xa8, 0x5a, 0x74, 0x3c, 0x88, 0xbc, 0xd7, 0xd4, 0x95, 0x28, 0xae, 0x15, 0x53, 0xc0, 0x45, 0x68, 0x1f, 0xca, 0x1b, 0x8a, 0xa6, 0xa0, 0x6e, 0x0a, 0x22, 0x13, 0x73, 0x6c, 0x8a,
0x9f, 0x32, 0x22, 0x13, 0x0b, 0xe8, 0x01, 0x5a, 0x39, 0x88, 0xc5, 0x66, 0x84, 0x58, 0x4c, 0xe5, 0xaa, 0x45, 0x27, 0xfd, 0xc8, 0x7b, 0x4d, 0x5d, 0x79, 0x4e, 0x37, 0x63, 0xea, 0x53, 0x46, 0x64,
0x62, 0xec, 0x0d, 0xbe, 0xb3, 0x37, 0x19, 0x8d, 0xc4, 0x22, 0xcf, 0x6a, 0x4b, 0x04, 0x4d, 0x62, 0x62, 0x01, 0x3d, 0x44, 0x2b, 0x07, 0xb1, 0xd8, 0xac, 0x10, 0x8b, 0xa9, 0x5c, 0x8c, 0x3d, 0xe4,
0x1d, 0xd9, 0xab, 0x82, 0xc4, 0x44, 0x72, 0x71, 0x3f, 0xbc, 0x5e, 0x3f, 0x0f, 0x0b, 0x9a, 0xa2, 0xb7, 0xf7, 0xc7, 0xc3, 0xa1, 0x58, 0xe4, 0x59, 0x6d, 0x89, 0xc8, 0x4b, 0xac, 0x23, 0x7b, 0xdf,
0xb2, 0xd4, 0xc4, 0x58, 0x10, 0x55, 0xe8, 0xbb, 0xe9, 0x6f, 0x5c, 0x80, 0xf3, 0xa9, 0xfe, 0x62, 0x90, 0x98, 0x48, 0x2e, 0xee, 0xfb, 0x17, 0xfd, 0xe7, 0x61, 0x51, 0x53, 0x54, 0xd6, 0xab, 0x18,
0xd8, 0xd5, 0xcb, 0x50, 0x8f, 0xdf, 0xdb, 0xc9, 0x2c, 0x54, 0x9f, 0x6e, 0xee, 0x75, 0xce, 0xb1, 0x0b, 0xa2, 0x94, 0x7d, 0x3b, 0xfd, 0x8d, 0x0b, 0x70, 0x3e, 0xd5, 0x5f, 0x0c, 0xbb, 0x76, 0x19,
0x8f, 0x67, 0x5b, 0x7b, 0x9d, 0xca, 0xea, 0x2a, 0xcc, 0x67, 0x2e, 0x18, 0x49, 0x03, 0x6a, 0xe6, 0x6a, 0xf1, 0xa3, 0x3d, 0x99, 0x83, 0xca, 0xd3, 0xad, 0xfd, 0xf6, 0x39, 0xf6, 0xf1, 0x6c, 0x7b,
0xf6, 0xc6, 0xd6, 0x0b, 0x14, 0x9b, 0x83, 0xfa, 0xee, 0x93, 0xa7, 0xa2, 0x55, 0x59, 0xdd, 0x84, 0xbf, 0x5d, 0x5a, 0x5b, 0x83, 0x85, 0xcc, 0x2d, 0x25, 0xa9, 0x43, 0xd5, 0xdc, 0xd9, 0xdc, 0x7e,
0x76, 0xfa, 0xb8, 0x23, 0x4d, 0x98, 0xdd, 0x44, 0xee, 0xd3, 0xed, 0x2d, 0x14, 0xc6, 0x86, 0xf9, 0x81, 0x62, 0xf3, 0x50, 0xdb, 0x7b, 0xf2, 0x54, 0xb4, 0x4a, 0x6b, 0x5b, 0xd0, 0x4a, 0x1f, 0x77,
0x6c, 0x77, 0x77, 0x67, 0xf7, 0x41, 0xa7, 0x42, 0x00, 0x66, 0xb6, 0xbf, 0xdd, 0x61, 0x8c, 0x29, 0xa4, 0x01, 0x73, 0x5b, 0xc8, 0x7d, 0xba, 0xb3, 0x8d, 0xc2, 0xd8, 0x30, 0x9f, 0xed, 0xed, 0xed,
0xc6, 0x78, 0xb6, 0xfb, 0x70, 0xf7, 0xc9, 0xaf, 0x76, 0x3b, 0xd5, 0xf5, 0xbf, 0xd5, 0xa1, 0x6d, 0xee, 0x3d, 0x68, 0x97, 0x08, 0xc0, 0xec, 0xce, 0xd7, 0xbb, 0x8c, 0x51, 0x66, 0x8c, 0x67, 0x7b,
0x8a, 0x25, 0xf4, 0x31, 0x16, 0x1c, 0x04, 0x8d, 0xf7, 0x61, 0x36, 0xfe, 0x99, 0x21, 0x39, 0x58, 0x0f, 0xf7, 0x9e, 0xfc, 0x66, 0xaf, 0x5d, 0xd9, 0xf8, 0x67, 0x0d, 0x5a, 0xa6, 0x58, 0x42, 0x0f,
0xd3, 0x7f, 0x5e, 0xf4, 0xba, 0x79, 0x86, 0x34, 0xdb, 0x39, 0xb2, 0x07, 0xad, 0xd4, 0x4b, 0x1a, 0x63, 0xc1, 0x41, 0xe4, 0x79, 0x1f, 0xe6, 0xe2, 0x3f, 0x22, 0x92, 0x83, 0x35, 0xfd, 0xfb, 0x46,
0x49, 0x40, 0x66, 0xd1, 0x63, 0x5d, 0xef, 0x4a, 0x19, 0x5b, 0x8d, 0xd8, 0x87, 0x76, 0xfa, 0x75, 0xb7, 0x93, 0x67, 0x48, 0xb3, 0x9d, 0x23, 0xfb, 0xd0, 0x4c, 0x3d, 0xc7, 0x91, 0x04, 0xa9, 0x16,
0x8c, 0x24, 0x7d, 0x0a, 0x5f, 0xdd, 0x7a, 0x57, 0x4b, 0xf9, 0x6a, 0xd0, 0x17, 0xd0, 0xc9, 0xbe, 0xbd, 0xf8, 0x75, 0xaf, 0x4c, 0x62, 0xab, 0x11, 0x7b, 0xd0, 0x4a, 0x3f, 0xb1, 0x91, 0xa4, 0x4f,
0x8b, 0x91, 0xe4, 0x4e, 0xab, 0xe4, 0xcd, 0xad, 0x77, 0xed, 0x04, 0x09, 0x7d, 0xe8, 0xdc, 0x0b, 0xe1, 0xd3, 0x5d, 0xf7, 0xea, 0x44, 0xbe, 0x1a, 0xf4, 0x05, 0xb4, 0xb3, 0x8f, 0x6b, 0x24, 0xb9,
0xd2, 0x4a, 0xf9, 0x1b, 0x40, 0x6e, 0xe8, 0xb2, 0x87, 0x05, 0x61, 0x8a, 0xf4, 0xa5, 0x2e, 0xd1, 0x18, 0x9b, 0xf0, 0x70, 0xd7, 0xbd, 0x76, 0x8a, 0x84, 0x3e, 0x74, 0xee, 0x19, 0x6a, 0x75, 0xf2,
0xdf, 0x6d, 0x0a, 0x2e, 0xd5, 0x35, 0x53, 0x14, 0xdf, 0x06, 0xe3, 0xa0, 0xcf, 0xb1, 0x52, 0x4b, 0x43, 0x42, 0x6e, 0xe8, 0x49, 0xaf, 0x13, 0xc2, 0x14, 0xe9, 0x9b, 0x61, 0xa2, 0x3f, 0xfe, 0x14,
0xe3, 0x55, 0x92, 0xf4, 0x2a, 0x46, 0xdf, 0xbd, 0x95, 0x72, 0x81, 0xb4, 0xdf, 0x74, 0x34, 0x9a, 0xdc, 0xcc, 0x6b, 0xa6, 0x28, 0xbe, 0x52, 0xc6, 0x41, 0x9f, 0x63, 0xa5, 0x96, 0x06, 0xbd, 0x24,
0xf2, 0x5b, 0x01, 0xc4, 0x4d, 0xf9, 0xad, 0x10, 0xc6, 0xf2, 0xf0, 0x4a, 0x61, 0x4e, 0x2d, 0xbc, 0xe9, 0x55, 0x0c, 0xe1, 0xbb, 0xab, 0x93, 0x05, 0xd2, 0x7e, 0xd3, 0x21, 0x6d, 0xca, 0x6f, 0x05,
0x8a, 0x00, 0x6e, 0xef, 0x4a, 0x19, 0x5b, 0x5f, 0x7e, 0x06, 0x6f, 0x6a, 0xcb, 0x2f, 0x86, 0xb1, 0x38, 0x39, 0xe5, 0xb7, 0x42, 0x2c, 0xcc, 0xc3, 0x2b, 0x05, 0x5c, 0xb5, 0xf0, 0x2a, 0x42, 0xc9,
0xbd, 0x95, 0x72, 0x81, 0xac, 0xaf, 0x92, 0x62, 0x3c, 0xe3, 0xab, 0x1c, 0x6c, 0xcb, 0xf8, 0x2a, 0xdd, 0x2b, 0x93, 0xd8, 0xfa, 0xf2, 0x33, 0xa0, 0x55, 0x5b, 0x7e, 0x31, 0x16, 0xee, 0xae, 0x4e,
0x5f, 0xc5, 0x4b, 0x5f, 0x65, 0xaa, 0xea, 0xab, 0xa5, 0xa5, 0x4a, 0xde, 0x57, 0xc5, 0xd5, 0x0f, 0x16, 0xc8, 0xfa, 0x2a, 0x29, 0xc6, 0x33, 0xbe, 0xca, 0x61, 0xbf, 0x8c, 0xaf, 0xf2, 0x55, 0xbc,
0x8e, 0x7b, 0x17, 0xa6, 0x59, 0x85, 0x41, 0x92, 0xb3, 0x5c, 0x2b, 0x6e, 0x7a, 0x17, 0x32, 0xd4, 0xf4, 0x55, 0xa6, 0xaa, 0xbe, 0x3a, 0xb1, 0x54, 0xc9, 0xfb, 0xaa, 0xb8, 0xfa, 0xc1, 0x71, 0xef,
0xb8, 0xdb, 0xf5, 0xca, 0xad, 0xca, 0xfa, 0x5f, 0xa6, 0x60, 0x4e, 0xa4, 0x3f, 0x99, 0x41, 0x1e, 0xc2, 0x0c, 0xab, 0x30, 0x48, 0x72, 0x96, 0x6b, 0xc5, 0x4d, 0xf7, 0x42, 0x86, 0x1a, 0x77, 0xbb,
0x00, 0x24, 0x87, 0x30, 0xe9, 0xa5, 0x16, 0x95, 0xaa, 0x24, 0x7a, 0x97, 0x0a, 0x79, 0x4a, 0xa9, 0x5e, 0xba, 0x55, 0xda, 0xf8, 0x7b, 0x19, 0xe6, 0x45, 0xfa, 0x93, 0x19, 0xe4, 0x01, 0x40, 0x72,
0x6f, 0x64, 0xd9, 0x22, 0x17, 0x7a, 0x29, 0x93, 0x6d, 0x53, 0x8b, 0xbc, 0x5c, 0xcc, 0x54, 0x63, 0x08, 0x93, 0x6e, 0x6a, 0x51, 0xa9, 0x4a, 0xa2, 0x7b, 0xa9, 0x90, 0xa7, 0x94, 0xfa, 0x4a, 0x96,
0x6d, 0x41, 0x43, 0x25, 0x79, 0xa2, 0x9d, 0x0d, 0x99, 0x13, 0xaa, 0xd7, 0x2b, 0x62, 0xe9, 0x1a, 0x2d, 0x72, 0xa1, 0x97, 0x32, 0xd9, 0x36, 0xb5, 0xc8, 0xcb, 0xc5, 0x4c, 0x35, 0xd6, 0x36, 0xd4,
0x69, 0x59, 0x5d, 0xd3, 0x28, 0x7f, 0x56, 0x68, 0x1a, 0x15, 0x1c, 0x04, 0xc6, 0xb9, 0xff, 0x06, 0x55, 0x92, 0x27, 0xda, 0xd9, 0x90, 0x39, 0xa1, 0xba, 0xdd, 0x22, 0x96, 0xae, 0x91, 0x96, 0xd5,
0x00, 0x00, 0xff, 0xff, 0xe6, 0xbd, 0x3a, 0x14, 0xe5, 0x26, 0x00, 0x00, 0x35, 0x8d, 0xf2, 0x67, 0x85, 0xa6, 0x51, 0xc1, 0x41, 0x60, 0x9c, 0xfb, 0x7f, 0x00, 0x00, 0x00,
0xff, 0xff, 0x22, 0x65, 0x58, 0x85, 0x2a, 0x27, 0x00, 0x00,
} }

16
pkg/kubelet/api/v1alpha1/runtime/api.proto
View File

@ -515,20 +515,24 @@ message Container {
// The ID of the container, used by the container runtime to identify // The ID of the container, used by the container runtime to identify
// a container. // a container.
optional string id = 1; optional string id = 1;
// The id of the sandbox which this container belongs to.
optional string pod_sandbox_id = 2;
// The metadata of the container. // The metadata of the container.
optional ContainerMetadata metadata = 2; optional ContainerMetadata metadata = 3;
// The spec of the image // The spec of the image
optional ImageSpec image = 3; optional ImageSpec image = 4;
// Reference to the image in use. For most runtimes, this should be an // Reference to the image in use. For most runtimes, this should be an
// image ID. // image ID.
optional string image_ref = 4; optional string image_ref = 5;
// State is the state of the container. // State is the state of the container.
optional ContainerState state = 5; optional ContainerState state = 6;
// Creation time of the container.
optional int64 created_at = 7;
// Labels are key value pairs that may be used to scope and select individual resources. // Labels are key value pairs that may be used to scope and select individual resources.
map<string, string> labels = 6; map<string, string> labels = 8;
// Annotations is an unstructured key value map that may be set by external // Annotations is an unstructured key value map that may be set by external
// tools to store and retrieve arbitrary metadata. // tools to store and retrieve arbitrary metadata.
map<string, string> annotations = 7; map<string, string> annotations = 9;
} }
message ListContainersResponse { message ListContainersResponse {

1
pkg/kubelet/kubelet.go
View File

@ -619,6 +619,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
kubecontainer.FilterEventRecorder(kubeDeps.Recorder), kubecontainer.FilterEventRecorder(kubeDeps.Recorder),
klet.livenessManager, klet.livenessManager,
containerRefManager, containerRefManager,
klet.podManager,
kubeDeps.OSInterface, kubeDeps.OSInterface,
klet.networkPlugin, klet.networkPlugin,
klet, klet,

2
pkg/kubelet/kuberuntime/doc.go
View File

@ -15,5 +15,5 @@ limitations under the License.
*/ */
// Package kuberuntime contains an implementation of kubecontainer.Runtime using // Package kuberuntime contains an implementation of kubecontainer.Runtime using
// the interface in pkg/kubelet/api/services.go. // the interface in pkg/kubelet/api.
package kuberuntime package kuberuntime

15
pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go
View File

@ -30,6 +30,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/lifecycle"
"k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/network"
proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results" proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results"
"k8s.io/kubernetes/pkg/types"
kubetypes "k8s.io/kubernetes/pkg/types" kubetypes "k8s.io/kubernetes/pkg/types"
"k8s.io/kubernetes/pkg/util/flowcontrol" "k8s.io/kubernetes/pkg/util/flowcontrol"
) )
@ -76,6 +77,19 @@ func (f *fakeRuntimeHelper) GetExtraSupplementalGroupsForPod(pod *api.Pod) []int
return nil return nil
} }
type fakePodGetter struct {
pods map[types.UID]*api.Pod
}
func newFakePodGetter() *fakePodGetter {
return &fakePodGetter{make(map[types.UID]*api.Pod)}
}
func (f *fakePodGetter) GetPodByUID(uid types.UID) (*api.Pod, bool) {
pod, found := f.pods[uid]
return pod, found
}
func NewFakeKubeRuntimeManager(runtimeService internalApi.RuntimeService, imageService internalApi.ImageManagerService, networkPlugin network.NetworkPlugin, osInterface kubecontainer.OSInterface) (*kubeGenericRuntimeManager, error) { func NewFakeKubeRuntimeManager(runtimeService internalApi.RuntimeService, imageService internalApi.ImageManagerService, networkPlugin network.NetworkPlugin, osInterface kubecontainer.OSInterface) (*kubeGenericRuntimeManager, error) {
recorder := &record.FakeRecorder{} recorder := &record.FakeRecorder{}
kubeRuntimeManager := &kubeGenericRuntimeManager{ kubeRuntimeManager := &kubeGenericRuntimeManager{
@ -96,6 +110,7 @@ func NewFakeKubeRuntimeManager(runtimeService internalApi.RuntimeService, imageS
return nil, err return nil, err
} }
kubeRuntimeManager.containerGC = NewContainerGC(runtimeService, newFakePodGetter(), kubeRuntimeManager)
kubeRuntimeManager.runtimeName = typedVersion.GetRuntimeName() kubeRuntimeManager.runtimeName = typedVersion.GetRuntimeName()
kubeRuntimeManager.imagePuller = images.NewImageManager( kubeRuntimeManager.imagePuller = images.NewImageManager(
kubecontainer.FilterEventRecorder(recorder), kubecontainer.FilterEventRecorder(recorder),

43
pkg/kubelet/kuberuntime/kuberuntime_container.go
View File

@ -239,6 +239,31 @@ func makeUID() string {
return fmt.Sprintf("%08x", rand.Uint32()) return fmt.Sprintf("%08x", rand.Uint32())
} }
// getTerminationMessage gets termination message of the container.
func getTerminationMessage(status *runtimeApi.ContainerStatus, kubeStatus *kubecontainer.ContainerStatus, terminationMessagePath string) string {
message := ""
if !kubeStatus.FinishedAt.IsZero() || kubeStatus.ExitCode != 0 {
if terminationMessagePath == "" {
return ""
}
for _, mount := range status.Mounts {
if mount.GetContainerPath() == terminationMessagePath {
path := mount.GetHostPath()
if data, err := ioutil.ReadFile(path); err != nil {
message = fmt.Sprintf("Error on reading termination-log %s: %v", path, err)
} else {
message = string(data)
}
break
}
}
}
return message
}
// getKubeletContainerStatuses gets all containers' status for the pod sandbox. // getKubeletContainerStatuses gets all containers' status for the pod sandbox.
func (m *kubeGenericRuntimeManager) getKubeletContainerStatuses(podSandboxID string) ([]*kubecontainer.ContainerStatus, error) { func (m *kubeGenericRuntimeManager) getKubeletContainerStatuses(podSandboxID string) ([]*kubecontainer.ContainerStatus, error) {
containers, err := m.runtimeService.ListContainers(&runtimeApi.ContainerFilter{ containers, err := m.runtimeService.ListContainers(&runtimeApi.ContainerFilter{
@ -282,23 +307,7 @@ func (m *kubeGenericRuntimeManager) getKubeletContainerStatuses(podSandboxID str
cStatus.FinishedAt = time.Unix(status.GetFinishedAt(), 0) cStatus.FinishedAt = time.Unix(status.GetFinishedAt(), 0)
} }
message := "" cStatus.Message = getTerminationMessage(status, cStatus, annotatedInfo.TerminationMessagePath)
if !cStatus.FinishedAt.IsZero() || cStatus.ExitCode != 0 {
if annotatedInfo.TerminationMessagePath != "" {
for _, mount := range status.Mounts {
if mount.GetContainerPath() == annotatedInfo.TerminationMessagePath {
path := mount.GetHostPath()
if data, err := ioutil.ReadFile(path); err != nil {
message = fmt.Sprintf("Error on reading termination-log %s: %v", path, err)
} else {
message = string(data)
}
break
}
}
}
}
cStatus.Message = message
statuses[i] = cStatus statuses[i] = cStatus
} }

280
pkg/kubelet/kuberuntime/kuberuntime_gc.go Normal file
View File

@ -0,0 +1,280 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kuberuntime
import (
"sort"
"time"
"github.com/golang/glog"
internalApi "k8s.io/kubernetes/pkg/kubelet/api"
runtimeApi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/types"
)
// containerGC is the manager of garbage collection.
type containerGC struct {
client internalApi.RuntimeService
manager *kubeGenericRuntimeManager
podGetter podGetter
}
// NewContainerGC creates a new containerGC.
func NewContainerGC(client internalApi.RuntimeService, podGetter podGetter, manager *kubeGenericRuntimeManager) *containerGC {
return &containerGC{
client: client,
manager: manager,
podGetter: podGetter,
}
}
// containerGCInfo is the internal information kept for containers being considered for GC.
type containerGCInfo struct {
// The ID of the container.
id string
// The name of the container.
name string
// The sandbox ID which this container belongs to
sandboxID string
// Creation time for the container.
createTime time.Time
}
// evictUnit is considered for eviction as units of (UID, container name) pair.
type evictUnit struct {
// UID of the pod.
uid types.UID
// Name of the container in the pod.
name string
}
type containersByEvictUnit map[evictUnit][]containerGCInfo
// NumContainers returns the number of containers in this map.
func (cu containersByEvictUnit) NumContainers() int {
num := 0
for key := range cu {
num += len(cu[key])
}
return num
}
// NumEvictUnits returns the number of pod in this map.
func (cu containersByEvictUnit) NumEvictUnits() int {
return len(cu)
}
// Newest first.
type byCreated []containerGCInfo
func (a byCreated) Len() int { return len(a) }
func (a byCreated) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byCreated) Less(i, j int) bool { return a[i].createTime.After(a[j].createTime) }
// enforceMaxContainersPerEvictUnit enforces MaxPerPodContainer for each evictUnit.
func (cgc *containerGC) enforceMaxContainersPerEvictUnit(evictUnits containersByEvictUnit, MaxContainers int) {
for key := range evictUnits {
toRemove := len(evictUnits[key]) - MaxContainers
if toRemove > 0 {
evictUnits[key] = cgc.removeOldestN(evictUnits[key], toRemove)
}
}
}
// removeOldestN removes the oldest toRemove containers and returns the resulting slice.
func (cgc *containerGC) removeOldestN(containers []containerGCInfo, toRemove int) []containerGCInfo {
// Remove from oldest to newest (last to first).
numToKeep := len(containers) - toRemove
for i := numToKeep; i < len(containers); i++ {
cgc.removeContainer(containers[i].id, containers[i].name)
}
// Assume we removed the containers so that we're not too aggressive.
return containers[:numToKeep]
}
// removeContainer removes the container by containerID.
func (cgc *containerGC) removeContainer(containerID, containerName string) {
glog.V(4).Infof("Removing container %q name %q", containerID, containerName)
if err := cgc.client.RemoveContainer(containerID); err != nil {
glog.Warningf("Failed to remove container %q: %v", containerID, err)
}
}
// removeSandbox removes the sandbox by sandboxID.
func (cgc *containerGC) removeSandbox(sandboxID string) {
glog.V(4).Infof("Removing sandbox %q", sandboxID)
if err := cgc.client.RemovePodSandbox(sandboxID); err != nil {
glog.Warningf("Failed to remove sandbox %q: %v", sandboxID, err)
}
}
// evictableContainers gets all containers that are evictable. Evictable containers are: not running
// and created more than MinAge ago.
func (cgc *containerGC) evictableContainers(minAge time.Duration) (containersByEvictUnit, error) {
containers, err := cgc.manager.getKubeletContainers(true)
if err != nil {
return containersByEvictUnit{}, err
}
evictUnits := make(containersByEvictUnit)
newestGCTime := time.Now().Add(-minAge)
for _, container := range containers {
// Prune out running containers.
if container.GetState() == runtimeApi.ContainerState_RUNNING {
continue
}
createdAt := time.Unix(container.GetCreatedAt(), 0)
if newestGCTime.Before(createdAt) {
continue
}
labeledInfo := getContainerInfoFromLabels(container.Labels)
containerInfo := containerGCInfo{
id: container.GetId(),
name: container.Metadata.GetName(),
createTime: createdAt,
sandboxID: container.GetPodSandboxId(),
}
key := evictUnit{
uid: labeledInfo.PodUID,
name: containerInfo.name,
}
evictUnits[key] = append(evictUnits[key], containerInfo)
}
// Sort the containers by age.
for uid := range evictUnits {
sort.Sort(byCreated(evictUnits[uid]))
}
return evictUnits, nil
}
// evictableSandboxes gets all sandboxes that are evictable. Evictable sandboxes are: not running
// and contains no containers at all.
func (cgc *containerGC) evictableSandboxes() ([]string, error) {
containers, err := cgc.manager.getKubeletContainers(true)
if err != nil {
return nil, err
}
sandboxes, err := cgc.manager.getKubeletSandboxes(true)
if err != nil {
return nil, err
}
evictSandboxes := make([]string, 0)
for _, sandbox := range sandboxes {
// Prune out ready sandboxes.
if sandbox.GetState() == runtimeApi.PodSandBoxState_READY {
continue
}
// Prune out sandboxes that still have containers.
found := false
sandboxID := sandbox.GetId()
for _, container := range containers {
if container.GetPodSandboxId() == sandboxID {
found = true
break
}
}
if found {
continue
}
evictSandboxes = append(evictSandboxes, sandboxID)
}
return evictSandboxes, nil
}
// isPodDeleted returns true if the pod is already deleted.
func (cgc *containerGC) isPodDeleted(podUID types.UID) bool {
_, found := cgc.podGetter.GetPodByUID(podUID)
return !found
}
// GarbageCollect removes dead containers using the specified container gc policy.
// Note that gc policy is not applied to sandboxes. Sandboxes are only removed when they are
// not ready and containing no containers.
//
// GarbageCollect consists of the following steps:
// * gets evictable containers which are not active and created more than gcPolicy.MinAge ago.
// * removes oldest dead containers for each pod by enforcing gcPolicy.MaxPerPodContainer.
// * removes oldest dead containers by enforcing gcPolicy.MaxContainers.
// * gets evictable sandboxes which are not ready and contains no containers.
// * removes evictable sandboxes.
func (cgc *containerGC) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool) error {
// Separate containers by evict units.
evictUnits, err := cgc.evictableContainers(gcPolicy.MinAge)
if err != nil {
return err
}
// Remove deleted pod containers if all sources are ready.
if allSourcesReady {
for key, unit := range evictUnits {
if cgc.isPodDeleted(key.uid) {
cgc.removeOldestN(unit, len(unit)) // Remove all.
delete(evictUnits, key)
}
}
}
// Enforce max containers per evict unit.
if gcPolicy.MaxPerPodContainer >= 0 {
cgc.enforceMaxContainersPerEvictUnit(evictUnits, gcPolicy.MaxPerPodContainer)
}
// Enforce max total number of containers.
if gcPolicy.MaxContainers >= 0 && evictUnits.NumContainers() > gcPolicy.MaxContainers {
// Leave an equal number of containers per evict unit (min: 1).
numContainersPerEvictUnit := gcPolicy.MaxContainers / evictUnits.NumEvictUnits()
if numContainersPerEvictUnit < 1 {
numContainersPerEvictUnit = 1
}
cgc.enforceMaxContainersPerEvictUnit(evictUnits, numContainersPerEvictUnit)
// If we still need to evict, evict oldest first.
numContainers := evictUnits.NumContainers()
if numContainers > gcPolicy.MaxContainers {
flattened := make([]containerGCInfo, 0, numContainers)
for key := range evictUnits {
flattened = append(flattened, evictUnits[key]...)
}
sort.Sort(byCreated(flattened))
cgc.removeOldestN(flattened, numContainers-gcPolicy.MaxContainers)
}
}
// Remove sandboxes with zero containers
evictSandboxes, err := cgc.evictableSandboxes()
if err != nil {
return err
}
for _, sandbox := range evictSandboxes {
cgc.removeSandbox(sandbox)
}
return nil
}

141
pkg/kubelet/kuberuntime/kuberuntime_gc_test.go Normal file
View File

@ -0,0 +1,141 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kuberuntime
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"k8s.io/kubernetes/pkg/api"
apitest "k8s.io/kubernetes/pkg/kubelet/api/testing"
runtimeApi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/types"
)
type apiPodWithCreatedAt struct {
apiPod *api.Pod
createdAt int64
}
func makeAndSetFakeEvictablePod(m *kubeGenericRuntimeManager, fakeRuntime *apitest.FakeRuntimeService, pods []*apiPodWithCreatedAt) error {
sandboxes := make([]*apitest.FakePodSandbox, 0)
containers := make([]*apitest.FakeContainer, 0)
for _, pod := range pods {
fakePodSandbox, err := makeFakePodSandbox(m, pod.apiPod, pod.createdAt)
if err != nil {
return err
}
fakeContainers, err := makeFakeContainers(m, pod.apiPod, pod.apiPod.Spec.Containers, pod.createdAt)
if err != nil {
return err
}
// Set sandbox to not ready state
sandboxNotReady := runtimeApi.PodSandBoxState_NOTREADY
fakePodSandbox.State = &sandboxNotReady
sandboxes = append(sandboxes, fakePodSandbox)
// Set containers to exited state
containerExited := runtimeApi.ContainerState_EXITED
for _, c := range fakeContainers {
c.State = &containerExited
containers = append(containers, c)
}
}
fakeRuntime.SetFakeSandboxes(sandboxes)
fakeRuntime.SetFakeContainers(containers)
return nil
}
func makeTestContainer(name, image string) api.Container {
return api.Container{
Name: name,
Image: image,
}
}
func makeTestPod(podName, podNamespace, podUID string, containers []api.Container, createdAt int64) *apiPodWithCreatedAt {
return &apiPodWithCreatedAt{
createdAt: createdAt,
apiPod: &api.Pod{
ObjectMeta: api.ObjectMeta{
UID: types.UID(podUID),
Name: podName,
Namespace: podNamespace,
},
Spec: api.PodSpec{
Containers: containers,
},
},
}
}
func TestGarbageCollect(t *testing.T) {
fakeRuntime, _, m, err := createTestRuntimeManager()
assert.NoError(t, err)
pods := []*apiPodWithCreatedAt{
makeTestPod("123456", "foo1", "new", []api.Container{
makeTestContainer("foo1", "busybox"),
makeTestContainer("foo2", "busybox"),
makeTestContainer("foo3", "busybox"),
}, 1),
makeTestPod("1234567", "foo2", "new", []api.Container{
makeTestContainer("foo4", "busybox"),
makeTestContainer("foo5", "busybox"),
}, 2),
makeTestPod("12345678", "foo3", "new", []api.Container{
makeTestContainer("foo6", "busybox"),
}, 3),
}
err = makeAndSetFakeEvictablePod(m, fakeRuntime, pods)
assert.NoError(t, err)
assert.NoError(t, m.GarbageCollect(kubecontainer.ContainerGCPolicy{
MinAge: time.Second,
MaxPerPodContainer: 1,
MaxContainers: 3,
}, false))
assert.Equal(t, 3, len(fakeRuntime.Containers))
assert.Equal(t, 2, len(fakeRuntime.Sandboxes))
// no containers should be removed.
err = makeAndSetFakeEvictablePod(m, fakeRuntime, pods)
assert.NoError(t, err)
assert.NoError(t, m.GarbageCollect(kubecontainer.ContainerGCPolicy{
MinAge: time.Second,
MaxPerPodContainer: 10,
MaxContainers: 100,
}, false))
assert.Equal(t, 6, len(fakeRuntime.Containers))
assert.Equal(t, 3, len(fakeRuntime.Sandboxes))
// all containers should be removed.
err = makeAndSetFakeEvictablePod(m, fakeRuntime, pods)
assert.NoError(t, err)
assert.NoError(t, m.GarbageCollect(kubecontainer.ContainerGCPolicy{
MinAge: time.Second,
MaxPerPodContainer: 0,
MaxContainers: 0,
}, false))
assert.Equal(t, 0, len(fakeRuntime.Containers))
assert.Equal(t, 0, len(fakeRuntime.Sandboxes))
}

20
pkg/kubelet/kuberuntime/kuberuntime_manager.go
View File

@ -55,12 +55,20 @@ var (
ErrVersionNotSupported = errors.New("Runtime api version is not supported") ErrVersionNotSupported = errors.New("Runtime api version is not supported")
) )
// A subset of the pod.Manager interface extracted for garbage collection purposes.
type podGetter interface {
GetPodByUID(kubetypes.UID) (*api.Pod, bool)
}
type kubeGenericRuntimeManager struct { type kubeGenericRuntimeManager struct {
runtimeName string runtimeName string
recorder record.EventRecorder recorder record.EventRecorder
osInterface kubecontainer.OSInterface osInterface kubecontainer.OSInterface
containerRefManager *kubecontainer.RefManager containerRefManager *kubecontainer.RefManager
// Container GC manager
containerGC *containerGC
// Keyring for pulling images // Keyring for pulling images
keyring credentialprovider.DockerKeyring keyring credentialprovider.DockerKeyring
@ -92,6 +100,7 @@ func NewKubeGenericRuntimeManager(
recorder record.EventRecorder, recorder record.EventRecorder,
livenessManager proberesults.Manager, livenessManager proberesults.Manager,
containerRefManager *kubecontainer.RefManager, containerRefManager *kubecontainer.RefManager,
podGetter podGetter,
osInterface kubecontainer.OSInterface, osInterface kubecontainer.OSInterface,
networkPlugin network.NetworkPlugin, networkPlugin network.NetworkPlugin,
runtimeHelper kubecontainer.RuntimeHelper, runtimeHelper kubecontainer.RuntimeHelper,
@ -151,6 +160,7 @@ func NewKubeGenericRuntimeManager(
imageBackOff, imageBackOff,
serializeImagePulls) serializeImagePulls)
kubeRuntimeManager.runner = lifecycle.NewHandlerRunner(httpClient, kubeRuntimeManager, kubeRuntimeManager) kubeRuntimeManager.runner = lifecycle.NewHandlerRunner(httpClient, kubeRuntimeManager, kubeRuntimeManager)
kubeRuntimeManager.containerGC = NewContainerGC(runtimeService, podGetter, kubeRuntimeManager)
return kubeRuntimeManager, nil return kubeRuntimeManager, nil
} }
@ -475,6 +485,11 @@ func (m *kubeGenericRuntimeManager) GetNetNS(sandboxID kubecontainer.ContainerID
return "", fmt.Errorf("not supported") return "", fmt.Errorf("not supported")
} }
// GarbageCollect removes dead containers using the specified container gc policy.
func (m *kubeGenericRuntimeManager) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool) error {
return m.containerGC.GarbageCollect(gcPolicy, allSourcesReady)
}
// GetPodContainerID gets pod sandbox ID // GetPodContainerID gets pod sandbox ID
func (m *kubeGenericRuntimeManager) GetPodContainerID(pod *kubecontainer.Pod) (kubecontainer.ContainerID, error) { func (m *kubeGenericRuntimeManager) GetPodContainerID(pod *kubecontainer.Pod) (kubecontainer.ContainerID, error) {
podFullName := kubecontainer.BuildPodFullName(pod.Name, pod.Namespace) podFullName := kubecontainer.BuildPodFullName(pod.Name, pod.Namespace)
@ -491,8 +506,3 @@ func (m *kubeGenericRuntimeManager) GetPodContainerID(pod *kubecontainer.Pod) (k
func (m *kubeGenericRuntimeManager) PortForward(pod *kubecontainer.Pod, port uint16, stream io.ReadWriteCloser) error { func (m *kubeGenericRuntimeManager) PortForward(pod *kubecontainer.Pod, port uint16, stream io.ReadWriteCloser) error {
return fmt.Errorf("not implemented") return fmt.Errorf("not implemented")
} }
// GarbageCollect removes dead containers using the specified container gc policy
func (m *kubeGenericRuntimeManager) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool) error {
return fmt.Errorf("not implemented")
}

16
pkg/kubelet/kuberuntime/kuberuntime_manager_test.go
View File

@ -54,12 +54,12 @@ func createTestRuntimeManager() (*apitest.FakeRuntimeService, *apitest.FakeImage
} }
func makeAndSetFakePod(m *kubeGenericRuntimeManager, fakeRuntime *apitest.FakeRuntimeService, pod *api.Pod) (*apitest.FakePodSandbox, []*apitest.FakeContainer, error) { func makeAndSetFakePod(m *kubeGenericRuntimeManager, fakeRuntime *apitest.FakeRuntimeService, pod *api.Pod) (*apitest.FakePodSandbox, []*apitest.FakeContainer, error) {
fakePodSandbox, err := makeFakePodSandbox(m, pod) fakePodSandbox, err := makeFakePodSandbox(m, pod, fakeCreatedAt)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
fakeContainers, err := makeFakeContainers(m, pod, pod.Spec.Containers) fakeContainers, err := makeFakeContainers(m, pod, pod.Spec.Containers, fakeCreatedAt)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -69,7 +69,7 @@ func makeAndSetFakePod(m *kubeGenericRuntimeManager, fakeRuntime *apitest.FakeRu
return fakePodSandbox, fakeContainers, nil return fakePodSandbox, fakeContainers, nil
} }
func makeFakePodSandbox(m *kubeGenericRuntimeManager, pod *api.Pod) (*apitest.FakePodSandbox, error) { func makeFakePodSandbox(m *kubeGenericRuntimeManager, pod *api.Pod, createdAt int64) (*apitest.FakePodSandbox, error) {
config, err := m.generatePodSandboxConfig(pod, "", 0) config, err := m.generatePodSandboxConfig(pod, "", 0)
if err != nil { if err != nil {
return nil, err return nil, err
@ -82,13 +82,13 @@ func makeFakePodSandbox(m *kubeGenericRuntimeManager, pod *api.Pod) (*apitest.Fa
Id: &podSandboxID, Id: &podSandboxID,
Metadata: config.Metadata, Metadata: config.Metadata,
State: &readyState, State: &readyState,
CreatedAt: &fakeCreatedAt, CreatedAt: &createdAt,
Labels: config.Labels, Labels: config.Labels,
}, },
}, nil }, nil
} }
func makeFakeContainer(m *kubeGenericRuntimeManager, pod *api.Pod, container api.Container, sandboxConfig *runtimeApi.PodSandboxConfig) (*apitest.FakeContainer, error) { func makeFakeContainer(m *kubeGenericRuntimeManager, pod *api.Pod, container api.Container, sandboxConfig *runtimeApi.PodSandboxConfig, createdAt int64) (*apitest.FakeContainer, error) {
containerConfig, err := m.generateContainerConfig(&container, pod, 0, "") containerConfig, err := m.generateContainerConfig(&container, pod, 0, "")
if err != nil { if err != nil {
return nil, err return nil, err
@ -104,7 +104,7 @@ func makeFakeContainer(m *kubeGenericRuntimeManager, pod *api.Pod, container api
Metadata: containerConfig.Metadata, Metadata: containerConfig.Metadata,
Image: containerConfig.Image, Image: containerConfig.Image,
ImageRef: &imageRef, ImageRef: &imageRef,
CreatedAt: &fakeCreatedAt, CreatedAt: &createdAt,
State: &runningState, State: &runningState,
Labels: containerConfig.Labels, Labels: containerConfig.Labels,
Annotations: containerConfig.Annotations, Annotations: containerConfig.Annotations,
@ -113,7 +113,7 @@ func makeFakeContainer(m *kubeGenericRuntimeManager, pod *api.Pod, container api
}, nil }, nil
} }
func makeFakeContainers(m *kubeGenericRuntimeManager, pod *api.Pod, containers []api.Container) ([]*apitest.FakeContainer, error) { func makeFakeContainers(m *kubeGenericRuntimeManager, pod *api.Pod, containers []api.Container, createdAt int64) ([]*apitest.FakeContainer, error) {
sandboxConfig, err := m.generatePodSandboxConfig(pod, "", 0) sandboxConfig, err := m.generatePodSandboxConfig(pod, "", 0)
if err != nil { if err != nil {
return nil, err return nil, err
@ -121,7 +121,7 @@ func makeFakeContainers(m *kubeGenericRuntimeManager, pod *api.Pod, containers [
result := make([]*apitest.FakeContainer, len(containers)) result := make([]*apitest.FakeContainer, len(containers))
for idx, c := range containers { for idx, c := range containers {
containerWithState, err := makeFakeContainer(m, pod, c, sandboxConfig) containerWithState, err := makeFakeContainer(m, pod, c, sandboxConfig, createdAt)
if err != nil { if err != nil {
return nil, err return nil, err
} }