Merge pull request #58172 from NVIDIA/annotations

Automatic merge from submit-queue (batch tested with PRs 58184, 59307, 58172). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add annotations to the device plugin API

**What this PR does / why we need it**:

**Which issue(s) this PR fixes** : Related to #56649 but does not fix it

This adds the ability for the device plugins to annotate containers.
Product wise, this allows the NVIDIA device plugin to support CRI-O (which allows hooks through container annotations).

**Special notes for your reviewer**:
/area hw-accelerators
/cc @vishh @jiayingz @vikaschoudhary16 

I'm wondering if it would make sense to fire a blank call to `newContainerAnnotations` at the start of the deviceplugin to get Annotations that are forbidden.
Current behavior is that any Annotations that conflicts with Kubelet will be overwritten by Kubelet.

**Release note**:
```release-note
NONE
```

pkg/kubelet/apis/deviceplugin/v1alpha/api.pb.go

@@ -191,6 +191,8 @@ type AllocateResponse struct {
 	Mounts []*Mount `protobuf:"bytes,2,rep,name=mounts" json:"mounts,omitempty"`
 	// Devices for the container.
 	Devices []*DeviceSpec `protobuf:"bytes,3,rep,name=devices" json:"devices,omitempty"`
+	// Container annotations to pass to the container runtime
+	Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (m *AllocateResponse) Reset()                    { *m = AllocateResponse{} }
@@ -218,6 +220,13 @@ func (m *AllocateResponse) GetDevices() []*DeviceSpec {
 	return nil
 }
 
+func (m *AllocateResponse) GetAnnotations() map[string]string {
+	if m != nil {
+		return m.Annotations
+	}
+	return nil
+}
+
 // Mount specifies a host volume to mount into a container.
 // where device library or tools are installed on host and container
 type Mount struct {
@@ -715,6 +724,23 @@ func (m *AllocateResponse) MarshalTo(dAtA []byte) (int, error) {
 			i += n
 		}
 	}
+	if len(m.Annotations) > 0 {
+		for k := range m.Annotations {
+			dAtA[i] = 0x22
+			i++
+			v := m.Annotations[k]
+			mapSize := 1 + len(k) + sovApi(uint64(len(k))) + 1 + len(v) + sovApi(uint64(len(v)))
+			i = encodeVarintApi(dAtA, i, uint64(mapSize))
+			dAtA[i] = 0xa
+			i++
+			i = encodeVarintApi(dAtA, i, uint64(len(k)))
+			i += copy(dAtA[i:], k)
+			dAtA[i] = 0x12
+			i++
+			i = encodeVarintApi(dAtA, i, uint64(len(v)))
+			i += copy(dAtA[i:], v)
+		}
+	}
 	return i, nil
 }
 
@@ -906,6 +932,14 @@ func (m *AllocateResponse) Size() (n int) {
 			n += 1 + l + sovApi(uint64(l))
 		}
 	}
+	if len(m.Annotations) > 0 {
+		for k, v := range m.Annotations {
+			_ = k
+			_ = v
+			mapEntrySize := 1 + len(k) + sovApi(uint64(len(k))) + 1 + len(v) + sovApi(uint64(len(v)))
+			n += mapEntrySize + 1 + sovApi(uint64(mapEntrySize))
+		}
+	}
 	return n
 }
 
@@ -1023,10 +1057,21 @@ func (this *AllocateResponse) String() string {
 		mapStringForEnvs += fmt.Sprintf("%v: %v,", k, this.Envs[k])
 	}
 	mapStringForEnvs += "}"
+	keysForAnnotations := make([]string, 0, len(this.Annotations))
+	for k := range this.Annotations {
+		keysForAnnotations = append(keysForAnnotations, k)
+	}
+	github_com_gogo_protobuf_sortkeys.Strings(keysForAnnotations)
+	mapStringForAnnotations := "map[string]string{"
+	for _, k := range keysForAnnotations {
+		mapStringForAnnotations += fmt.Sprintf("%v: %v,", k, this.Annotations[k])
+	}
+	mapStringForAnnotations += "}"
 	s := strings.Join([]string{`&AllocateResponse{`,
 		`Envs:` + mapStringForEnvs + `,`,
 		`Mounts:` + strings.Replace(fmt.Sprintf("%v", this.Mounts), "Mount", "Mount", 1) + `,`,
 		`Devices:` + strings.Replace(fmt.Sprintf("%v", this.Devices), "DeviceSpec", "DeviceSpec", 1) + `,`,
+		`Annotations:` + mapStringForAnnotations + `,`,
 		`}`,
 	}, "")
 	return s
@@ -1725,6 +1770,122 @@ func (m *AllocateResponse) Unmarshal(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 4:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Annotations", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			var keykey uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				keykey |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			var stringLenmapkey uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLenmapkey |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLenmapkey := int(stringLenmapkey)
+			if intStringLenmapkey < 0 {
+				return ErrInvalidLengthApi
+			}
+			postStringIndexmapkey := iNdEx + intStringLenmapkey
+			if postStringIndexmapkey > l {
+				return io.ErrUnexpectedEOF
+			}
+			mapkey := string(dAtA[iNdEx:postStringIndexmapkey])
+			iNdEx = postStringIndexmapkey
+			if m.Annotations == nil {
+				m.Annotations = make(map[string]string)
+			}
+			if iNdEx < postIndex {
+				var valuekey uint64
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowApi
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					valuekey |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				var stringLenmapvalue uint64
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowApi
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					stringLenmapvalue |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				intStringLenmapvalue := int(stringLenmapvalue)
+				if intStringLenmapvalue < 0 {
+					return ErrInvalidLengthApi
+				}
+				postStringIndexmapvalue := iNdEx + intStringLenmapvalue
+				if postStringIndexmapvalue > l {
+					return io.ErrUnexpectedEOF
+				}
+				mapvalue := string(dAtA[iNdEx:postStringIndexmapvalue])
+				iNdEx = postStringIndexmapvalue
+				m.Annotations[mapkey] = mapvalue
+			} else {
+				var mapvalue string
+				m.Annotations[mapkey] = mapvalue
+			}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipApi(dAtA[iNdEx:])
@@ -2119,41 +2280,43 @@ var (
 func init() { proto.RegisterFile("api.proto", fileDescriptorApi) }
 
 var fileDescriptorApi = []byte{
-	// 562 bytes of a gzipped FileDescriptorProto
+	// 594 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcb, 0x8e, 0xd3, 0x4a,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0x5d, 0x8b, 0xd3, 0x40,
-	0x10, 0x4d, 0x27, 0x77, 0xf2, 0xa8, 0xc9, 0x3c, 0xd4, 0x37, 0x42, 0x96, 0x01, 0x2b, 0x32, 0x42,
+	0x14, 0x6d, 0xd2, 0xdd, 0x6e, 0x7b, 0xdb, 0xdd, 0x2d, 0x63, 0x91, 0x10, 0x35, 0x94, 0x88, 0x50,
-	0x8a, 0x84, 0xf0, 0x0c, 0x61, 0x01, 0x42, 0x2c, 0x18, 0x94, 0x20, 0x8d, 0x86, 0x47, 0x64, 0x16,
+	0x10, 0xd3, 0xb5, 0x3e, 0x28, 0x22, 0x62, 0xa5, 0x15, 0x96, 0xf5, 0xa3, 0xc6, 0x07, 0x1f, 0xcb,
-	0x2c, 0xa3, 0x8e, 0x53, 0xc4, 0x16, 0x76, 0xb7, 0x71, 0xb7, 0x23, 0x65, 0xc7, 0x27, 0xf0, 0x19,
+	0x34, 0x1d, 0x9b, 0xc1, 0x64, 0x26, 0x66, 0x26, 0x85, 0xbe, 0xf9, 0x13, 0xfc, 0x19, 0xfe, 0x94,
-	0x7c, 0xca, 0x2c, 0x59, 0xb2, 0x64, 0xc2, 0x8e, 0xaf, 0x40, 0x6e, 0xdb, 0x79, 0x29, 0x62, 0xc5,
+	0x7d, 0xf4, 0xd1, 0x47, 0xb7, 0xfe, 0x0e, 0x41, 0x3a, 0x49, 0xfa, 0x11, 0x8a, 0x22, 0xf8, 0x96,
-	0xce, 0x75, 0xea, 0x9c, 0xd4, 0xa9, 0xca, 0xb1, 0xa1, 0xc5, 0xe2, 0xc0, 0x89, 0x13, 0xa1, 0x04,
+	0x7b, 0xe6, 0x9e, 0xc9, 0xb9, 0x27, 0xf7, 0x04, 0x6a, 0x38, 0xa2, 0x4e, 0x14, 0x73, 0xc9, 0x51,
-	0x6d, 0x4f, 0x71, 0x1e, 0x78, 0x18, 0x87, 0xe9, 0x2c, 0xe0, 0xe6, 0xc3, 0x59, 0xa0, 0xfc, 0x74,
+	0x63, 0x4a, 0xe6, 0xd4, 0x23, 0x51, 0x90, 0xcc, 0x28, 0x33, 0xef, 0xcd, 0xa8, 0xf4, 0x93, 0x89,
-	0xe2, 0x78, 0x22, 0x3a, 0x9b, 0x89, 0x99, 0x38, 0xd3, 0xa4, 0x49, 0xfa, 0x51, 0x57, 0xba, 0xd0,
+	0xe3, 0xf1, 0xb0, 0x3b, 0xe3, 0x33, 0xde, 0x55, 0x4d, 0x93, 0xe4, 0x83, 0xaa, 0x54, 0xa1, 0x9e,
-	0x4f, 0xb9, 0xd8, 0x0e, 0xe1, 0xc4, 0xc5, 0x59, 0x20, 0x15, 0x26, 0x2e, 0x7e, 0x4e, 0x51, 0x2a,
+	0x52, 0xb2, 0x1d, 0xc0, 0xa9, 0x4b, 0x66, 0x54, 0x48, 0x12, 0xbb, 0xe4, 0x53, 0x42, 0x84, 0x44,
-	0x6a, 0x40, 0x63, 0x8e, 0x89, 0x0c, 0x04, 0x37, 0x48, 0x97, 0xf4, 0x5a, 0x6e, 0x59, 0x52, 0x13,
+	0x06, 0x1c, 0xcd, 0x49, 0x2c, 0x28, 0x67, 0x86, 0xd6, 0xd6, 0x3a, 0x35, 0x37, 0x2f, 0x91, 0x09,
-	0x9a, 0xc8, 0xa7, 0xb1, 0x08, 0xb8, 0x32, 0xaa, 0xba, 0xb5, 0xaa, 0xe9, 0x3d, 0x38, 0x4a, 0x50,
+	0x55, 0xc2, 0xa6, 0x11, 0xa7, 0x4c, 0x1a, 0xba, 0x3a, 0x5a, 0xd7, 0xe8, 0x36, 0x1c, 0xc7, 0x44,
-	0x8a, 0x34, 0xf1, 0x70, 0xcc, 0x59, 0x84, 0x46, 0x4d, 0x13, 0xda, 0x25, 0xf8, 0x96, 0x45, 0x68,
+	0xf0, 0x24, 0xf6, 0xc8, 0x98, 0xe1, 0x90, 0x18, 0x65, 0xd5, 0xd0, 0xc8, 0xc1, 0xd7, 0x38, 0x24,
-	0x37, 0xe0, 0x60, 0x18, 0xc5, 0x6a, 0x61, 0xbf, 0x82, 0xce, 0xeb, 0x40, 0xaa, 0x0b, 0x3e, 0xfd,
+	0xf6, 0x11, 0x1c, 0x0e, 0xc3, 0x48, 0x2e, 0xec, 0x17, 0xd0, 0x7a, 0x49, 0x85, 0xec, 0xb3, 0xe9,
-	0xc0, 0x94, 0xe7, 0xbb, 0x28, 0x63, 0xc1, 0x25, 0x52, 0x07, 0x1a, 0xf9, 0x36, 0xd2, 0x20, 0xdd,
+	0x7b, 0x2c, 0x3d, 0xdf, 0x25, 0x22, 0xe2, 0x4c, 0x10, 0xe4, 0xc0, 0x51, 0x3a, 0x8d, 0x30, 0xb4,
-	0x5a, 0xef, 0xb0, 0xdf, 0x71, 0x36, 0xb7, 0x73, 0x06, 0xba, 0x70, 0x4b, 0x92, 0x7d, 0x0e, 0xf5,
+	0x76, 0xb9, 0x53, 0xef, 0xb5, 0x9c, 0xed, 0xe9, 0x9c, 0x81, 0x2a, 0xdc, 0xbc, 0xc9, 0x3e, 0x83,
-	0x1c, 0xa2, 0xc7, 0x50, 0xbd, 0x1c, 0x14, 0x86, 0xab, 0xc1, 0x80, 0xde, 0x82, 0xba, 0x8f, 0x2c,
+	0x4a, 0x0a, 0xa1, 0x13, 0xd0, 0xcf, 0x07, 0x99, 0x60, 0x9d, 0x0e, 0xd0, 0x75, 0xa8, 0xf8, 0x04,
-	0x54, 0x7e, 0xe1, 0xb4, 0xa8, 0xec, 0x47, 0x70, 0x72, 0x11, 0x86, 0xc2, 0x63, 0x0a, 0xcb, 0x85,
+	0x07, 0xd2, 0xcf, 0x94, 0x66, 0x95, 0x7d, 0x1f, 0x4e, 0xfb, 0x41, 0xc0, 0x3d, 0x2c, 0x49, 0x3e,
-	0x2d, 0x80, 0xe2, 0xf7, 0x2e, 0x07, 0xf9, 0xdc, 0x96, 0xbb, 0x81, 0xd8, 0xbf, 0x09, 0x9c, 0xae,
+	0xb0, 0x05, 0x90, 0xdd, 0x77, 0x3e, 0x48, 0xdf, 0x5b, 0x73, 0xb7, 0x10, 0xfb, 0x97, 0x0e, 0xcd,
-	0x35, 0x85, 0xd3, 0xe7, 0xf0, 0x1f, 0xf2, 0x79, 0x69, 0xb3, 0xb7, 0x6d, 0x73, 0x97, 0xed, 0x0c,
+	0x0d, 0x27, 0x53, 0xfa, 0x04, 0x0e, 0x08, 0x9b, 0xe7, 0x32, 0x3b, 0xbb, 0x32, 0x8b, 0xdd, 0xce,
-	0xf9, 0x5c, 0x0e, 0xb9, 0x4a, 0x16, 0xae, 0x56, 0xd1, 0x07, 0x50, 0x8f, 0x44, 0xca, 0x95, 0x34,
+	0x90, 0xcd, 0xc5, 0x90, 0xc9, 0x78, 0xe1, 0x2a, 0x16, 0xba, 0x0b, 0x95, 0x90, 0x27, 0x4c, 0x0a,
-	0xaa, 0x5a, 0xff, 0xff, 0xb6, 0xfe, 0x4d, 0xd6, 0x73, 0x0b, 0x0a, 0xed, 0xaf, 0x8f, 0x52, 0xd3,
+	0x43, 0x57, 0xfc, 0x6b, 0xbb, 0xfc, 0x57, 0xab, 0x33, 0x37, 0x6b, 0x41, 0xbd, 0x8d, 0x29, 0x65,
-	0x6c, 0x63, 0xdf, 0x51, 0xde, 0xc7, 0xe8, 0xad, 0x0e, 0x63, 0x3e, 0x81, 0xd6, 0x6a, 0x26, 0x3d,
+	0xd5, 0x6d, 0xec, 0x33, 0xe5, 0x5d, 0x44, 0xbc, 0xb5, 0x31, 0xe8, 0x2d, 0xd4, 0x31, 0x63, 0x5c,
-	0x85, 0xda, 0x27, 0x5c, 0x14, 0xc7, 0xc9, 0x1e, 0x69, 0x07, 0x0e, 0xe6, 0x2c, 0x4c, 0xb1, 0x38,
+	0x62, 0x49, 0x39, 0x13, 0xc6, 0x81, 0xe2, 0x75, 0xff, 0xa2, 0xb2, 0xbf, 0x61, 0xa4, 0x62, 0xb7,
-	0x4e, 0x5e, 0x3c, 0xab, 0x3e, 0x25, 0xb6, 0x0f, 0x07, 0x7a, 0x3a, 0xbd, 0x0f, 0xc7, 0x9e, 0xe0,
+	0xef, 0x30, 0x1f, 0x42, 0x6d, 0x3d, 0x06, 0x6a, 0x42, 0xf9, 0x23, 0x59, 0x64, 0x7e, 0xaf, 0x1e,
-	0x8a, 0x05, 0x1c, 0x93, 0x71, 0xcc, 0x94, 0x5f, 0xe8, 0x8f, 0x56, 0xe8, 0x88, 0x29, 0x9f, 0xde,
+	0x51, 0x0b, 0x0e, 0xe7, 0x38, 0x48, 0x48, 0xe6, 0x77, 0x5a, 0x3c, 0xd6, 0x1f, 0x69, 0xe6, 0x53,
-	0x86, 0x96, 0x2f, 0xa4, 0xca, 0x19, 0x45, 0x28, 0x32, 0xa0, 0x6c, 0x26, 0xc8, 0xa6, 0x63, 0xc1,
+	0x68, 0x16, 0x6f, 0xfe, 0x17, 0xbe, 0xed, 0xc3, 0xa1, 0x32, 0x04, 0xdd, 0x81, 0x13, 0x8f, 0x33,
-	0xc3, 0x85, 0x0e, 0x44, 0xd3, 0x6d, 0x66, 0xc0, 0x3b, 0x1e, 0x2e, 0xec, 0x04, 0x60, 0xed, 0xfc,
+	0x89, 0x29, 0x23, 0xf1, 0x38, 0xc2, 0xd2, 0xcf, 0xf8, 0xc7, 0x6b, 0x74, 0x84, 0xa5, 0x8f, 0x6e,
-	0x9f, 0x8c, 0xeb, 0xc2, 0x61, 0x8c, 0x49, 0x14, 0xc8, 0x2c, 0xad, 0xb2, 0x48, 0xe0, 0x26, 0xd4,
+	0x40, 0xcd, 0xe7, 0x42, 0xa6, 0x1d, 0xd9, 0x9e, 0xae, 0x80, 0xfc, 0x30, 0x26, 0x78, 0x3a, 0xe6,
-	0x1f, 0x41, 0x3b, 0x8f, 0x7b, 0xc2, 0x54, 0x96, 0xe8, 0x17, 0xd0, 0x2c, 0xe3, 0x4f, 0xef, 0x6e,
+	0x2c, 0x58, 0xa8, 0x1d, 0xad, 0xba, 0xd5, 0x15, 0xf0, 0x86, 0x05, 0x0b, 0x3b, 0x06, 0xd8, 0x98,
-	0x5f, 0x75, 0xe7, 0xb5, 0x30, 0x77, 0xfe, 0xa2, 0x3c, 0xc7, 0x95, 0xfe, 0x37, 0x02, 0xed, 0x7c,
+	0xf9, 0x5f, 0x5e, 0xd7, 0x86, 0x7a, 0x44, 0xe2, 0x90, 0x0a, 0xa1, 0xbe, 0x43, 0x1a, 0x8a, 0x6d,
-	0x8d, 0x91, 0x6e, 0xd0, 0x2b, 0x68, 0x6f, 0x46, 0x9b, 0xee, 0xd3, 0x99, 0xf6, 0x36, 0xb8, 0xef,
+	0xa8, 0x37, 0x82, 0x46, 0x9a, 0xc0, 0x58, 0xf9, 0x83, 0x9e, 0x41, 0x35, 0x4f, 0x24, 0xba, 0xb5,
-	0x5d, 0xb0, 0x2b, 0xe7, 0x84, 0x5e, 0x41, 0xb3, 0xcc, 0xd2, 0xae, 0xbf, 0x9d, 0x14, 0x9b, 0xd6,
+	0xfb, 0xc1, 0x0a, 0x49, 0x35, 0x0b, 0x5b, 0x93, 0x46, 0xab, 0xd4, 0xfb, 0xaa, 0x41, 0x23, 0x1d,
-	0xdf, 0x23, 0x68, 0x57, 0x5e, 0xde, 0xb9, 0xbe, 0xb1, 0xc8, 0x8f, 0x1b, 0xab, 0xf2, 0x65, 0x69,
+	0x63, 0xa4, 0x0e, 0xd0, 0x05, 0x34, 0xb6, 0xd3, 0x86, 0xf6, 0xf1, 0x4c, 0x7b, 0x17, 0xdc, 0x17,
-	0x91, 0xeb, 0xa5, 0x45, 0xbe, 0x2f, 0x2d, 0xf2, 0x73, 0x69, 0x91, 0xaf, 0xbf, 0xac, 0xca, 0xa4,
+	0x4f, 0xbb, 0x74, 0xa6, 0xa1, 0x0b, 0xa8, 0xe6, 0x8b, 0x53, 0xd4, 0x57, 0x08, 0x96, 0x69, 0xfd,
-	0xae, 0x3f, 0x08, 0x8f, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x17, 0x55, 0xaf, 0xe1, 0x5a, 0x04,
+	0x79, 0xdf, 0xec, 0xd2, 0xf3, 0x9b, 0x97, 0x57, 0x96, 0xf6, 0xfd, 0xca, 0x2a, 0x7d, 0x5e, 0x5a,
+	0xda, 0xe5, 0xd2, 0xd2, 0xbe, 0x2d, 0x2d, 0xed, 0xc7, 0xd2, 0xd2, 0xbe, 0xfc, 0xb4, 0x4a, 0x93,
+	0x8a, 0xfa, 0x47, 0x3d, 0xf8, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x67, 0x68, 0xfd, 0xfd, 0xed, 0x04,
 	0x00, 0x00,
 }

pkg/kubelet/apis/deviceplugin/v1alpha/api.proto

@@ -96,6 +96,8 @@ message AllocateResponse {
 	repeated Mount mounts = 2;
 	// Devices for the container.
 	repeated DeviceSpec devices = 3;
+	// Container annotations to pass to the container runtime
+	map<string, string> annotations = 4;
 }
 
 // Mount specifies a host volume to mount into a container.

pkg/kubelet/cm/container_manager_linux.go

@@ -620,6 +620,7 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe
 	opts.Devices = append(opts.Devices, devOpts.Devices...)
 	opts.Mounts = append(opts.Mounts, devOpts.Mounts...)
 	opts.Envs = append(opts.Envs, devOpts.Envs...)
+	opts.Annotations = append(opts.Annotations, devOpts.Annotations...)
 	return opts, nil
 }
 

pkg/kubelet/cm/devicemanager/pod_devices.go

@@ -191,6 +191,7 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic
 	devsMap := make(map[string]string)
 	mountsMap := make(map[string]string)
 	envsMap := make(map[string]string)
+	annotationsMap := make(map[string]string)
 	// Loops through AllocationResponses of all cached device resources.
 	for _, devices := range resources {
 		resp := devices.allocResp
@@ -198,17 +199,18 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic
 		// Environment variables
 		// Mount points
 		// Device files
+		// Container annotations
 		// These artifacts are per resource per container.
 		// Updates RunContainerOptions.Envs.
 		for k, v := range resp.Envs {
 			if e, ok := envsMap[k]; ok {
-				glog.V(3).Infof("skip existing env %s %s", k, v)
+				glog.V(4).Infof("Skip existing env %s %s", k, v)
 				if e != v {
 					glog.Errorf("Environment variable %s has conflicting setting: %s and %s", k, e, v)
 				}
 				continue
 			}
-			glog.V(4).Infof("add env %s %s", k, v)
+			glog.V(4).Infof("Add env %s %s", k, v)
 			envsMap[k] = v
 			opts.Envs = append(opts.Envs, kubecontainer.EnvVar{Name: k, Value: v})
 		}
@@ -216,14 +218,14 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic
 		// Updates RunContainerOptions.Devices.
 		for _, dev := range resp.Devices {
 			if d, ok := devsMap[dev.ContainerPath]; ok {
-				glog.V(3).Infof("skip existing device %s %s", dev.ContainerPath, dev.HostPath)
+				glog.V(4).Infof("Skip existing device %s %s", dev.ContainerPath, dev.HostPath)
 				if d != dev.HostPath {
 					glog.Errorf("Container device %s has conflicting mapping host devices: %s and %s",
 						dev.ContainerPath, d, dev.HostPath)
 				}
 				continue
 			}
-			glog.V(4).Infof("add device %s %s", dev.ContainerPath, dev.HostPath)
+			glog.V(4).Infof("Add device %s %s", dev.ContainerPath, dev.HostPath)
 			devsMap[dev.ContainerPath] = dev.HostPath
 			opts.Devices = append(opts.Devices, kubecontainer.DeviceInfo{
 				PathOnHost:      dev.HostPath,
@@ -231,17 +233,18 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic
 				Permissions:     dev.Permissions,
 			})
 		}
+
 		// Updates RunContainerOptions.Mounts.
 		for _, mount := range resp.Mounts {
 			if m, ok := mountsMap[mount.ContainerPath]; ok {
-				glog.V(3).Infof("skip existing mount %s %s", mount.ContainerPath, mount.HostPath)
+				glog.V(4).Infof("Skip existing mount %s %s", mount.ContainerPath, mount.HostPath)
 				if m != mount.HostPath {
 					glog.Errorf("Container mount %s has conflicting mapping host mounts: %s and %s",
 						mount.ContainerPath, m, mount.HostPath)
 				}
 				continue
 			}
-			glog.V(4).Infof("add mount %s %s", mount.ContainerPath, mount.HostPath)
+			glog.V(4).Infof("Add mount %s %s", mount.ContainerPath, mount.HostPath)
 			mountsMap[mount.ContainerPath] = mount.HostPath
 			opts.Mounts = append(opts.Mounts, kubecontainer.Mount{
 				Name:          mount.ContainerPath,
@@ -252,6 +255,20 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic
 				SELinuxRelabel: false,
 			})
 		}
+
+		// Updates for Annotations
+		for k, v := range resp.Annotations {
+			if e, ok := annotationsMap[k]; ok {
+				glog.V(4).Infof("Skip existing annotation %s %s", k, v)
+				if e != v {
+					glog.Errorf("Annotation %s has conflicting setting: %s and %s", k, e, v)
+				}
+				continue
+			}
+			glog.V(4).Infof("Add annotation %s %s", k, v)
+			annotationsMap[k] = v
+			opts.Annotations = append(opts.Annotations, kubecontainer.Annotation{Name: k, Value: v})
+		}
 	}
 	return opts
 }

pkg/kubelet/cm/devicemanager/types.go

@@ -66,6 +66,8 @@ type DeviceRunContainerOptions struct {
 	Mounts []kubecontainer.Mount
 	// The host devices mapped into the container.
 	Devices []kubecontainer.DeviceInfo
+	// The Annotations for the container
+	Annotations []kubecontainer.Annotation
 }
 
 // TODO: evaluate whether we need these error definitions.

pkg/kubelet/container/runtime.go

@@ -382,6 +382,11 @@ type EnvVar struct {
 	Value string
 }
 
+type Annotation struct {
+	Name  string
+	Value string
+}
+
 type Mount struct {
 	// Name of the volume mount.
 	// TODO(yifan): Remove this field, as this is not representing the unique name of the mount,
@@ -431,6 +436,10 @@ type RunContainerOptions struct {
 	Devices []DeviceInfo
 	// The port mappings for the containers.
 	PortMappings []PortMapping
+	// The annotations for the container
+	// These annotations are generated by other components (i.e.,
+	// not users). Currently, only device plugins populate the annotations.
+	Annotations []Annotation
 	// If the container has specified the TerminationMessagePath, then
 	// this directory will be used to create and mount the log file to
 	// container.TerminationMessagePath

pkg/kubelet/kuberuntime/kuberuntime_container.go

@@ -202,7 +202,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai
 		Args:        args,
 		WorkingDir:  container.WorkingDir,
 		Labels:      newContainerLabels(container, pod, containerType),
-		Annotations: newContainerAnnotations(container, pod, restartCount),
+		Annotations: newContainerAnnotations(container, pod, restartCount, opts),
 		Devices:     makeDevices(opts),
 		Mounts:      m.makeMounts(opts, container),
 		LogPath:     containerLogsPath,

pkg/kubelet/kuberuntime/kuberuntime_container_test.go

@@ -222,7 +222,7 @@ func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerInde
 		Args:        []string(nil),
 		WorkingDir:  container.WorkingDir,
 		Labels:      newContainerLabels(container, pod, kubecontainer.ContainerTypeRegular),
-		Annotations: newContainerAnnotations(container, pod, restartCount),
+		Annotations: newContainerAnnotations(container, pod, restartCount, opts),
 		Devices:     makeDevices(opts),
 		Mounts:      m.makeMounts(opts, container),
 		LogPath:     containerLogsPath,

pkg/kubelet/kuberuntime/labels.go

@@ -111,8 +111,14 @@ func newContainerLabels(container *v1.Container, pod *v1.Pod, containerType kube
 }
 
 // newContainerAnnotations creates container annotations from v1.Container and v1.Pod.
-func newContainerAnnotations(container *v1.Container, pod *v1.Pod, restartCount int) map[string]string {
+func newContainerAnnotations(container *v1.Container, pod *v1.Pod, restartCount int, opts *kubecontainer.RunContainerOptions) map[string]string {
 	annotations := map[string]string{}
+
+	// Kubelet always overrides device plugin annotations if they are conflicting
+	for _, a := range opts.Annotations {
+		annotations[a.Name] = a.Value
+	}
+
 	annotations[containerHashLabel] = strconv.FormatUint(kubecontainer.HashContainer(container), 16)
 	annotations[containerRestartCountLabel] = strconv.Itoa(restartCount)
 	annotations[containerTerminationMessagePathLabel] = container.TerminationMessagePath

pkg/kubelet/kuberuntime/labels_test.go

@@ -156,6 +156,11 @@ func TestContainerAnnotations(t *testing.T) {
 	restartCount := 5
 	deletionGracePeriod := int64(10)
 	terminationGracePeriod := int64(10)
+	opts := &kubecontainer.RunContainerOptions{
+		Annotations: []kubecontainer.Annotation{
+			{Name: "Foo", Value: "bar"},
+		},
+	}
 	lifecycle := &v1.Lifecycle{
 		// Left PostStart as nil
 		PreStop: &v1.Handler{
@@ -216,11 +221,14 @@ func TestContainerAnnotations(t *testing.T) {
 	}
 
 	// Test whether we can get right information from label
-	annotations := newContainerAnnotations(container, pod, restartCount)
+	annotations := newContainerAnnotations(container, pod, restartCount, opts)
 	containerInfo := getContainerInfoFromAnnotations(annotations)
 	if !reflect.DeepEqual(containerInfo, expected) {
 		t.Errorf("expected %v, got %v", expected, containerInfo)
 	}
+	if v, ok := annotations[opts.Annotations[0].Name]; !ok || v != opts.Annotations[0].Value {
+		t.Errorf("expected annotation %s to exist got %v, %v", opts.Annotations[0].Name, ok, v)
+	}
 
 	// Test when DeletionGracePeriodSeconds, TerminationGracePeriodSeconds and Lifecycle are nil,
 	// the information got from annotations should also be nil
@@ -232,11 +240,14 @@ func TestContainerAnnotations(t *testing.T) {
 	expected.PreStopHandler = nil
 	// Because container is changed, the Hash should be updated
 	expected.Hash = kubecontainer.HashContainer(container)
-	annotations = newContainerAnnotations(container, pod, restartCount)
+	annotations = newContainerAnnotations(container, pod, restartCount, opts)
 	containerInfo = getContainerInfoFromAnnotations(annotations)
 	if !reflect.DeepEqual(containerInfo, expected) {
 		t.Errorf("expected %v, got %v", expected, containerInfo)
 	}
+	if v, ok := annotations[opts.Annotations[0].Name]; !ok || v != opts.Annotations[0].Value {
+		t.Errorf("expected annotation %s to exist got %v, %v", opts.Annotations[0].Name, ok, v)
+	}
 }
 
 func TestPodLabels(t *testing.T) {