diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go
index f907615fc..15634aa68 100644
--- a/storage/remote/queue_manager.go
+++ b/storage/remote/queue_manager.go
@@ -847,6 +847,10 @@ func (t *QueueManager) releaseLabels(ls labels.Labels) {
 // processExternalLabels merges externalLabels into ls. If ls contains
 // a label in externalLabels, the value in ls wins.
 func processExternalLabels(ls labels.Labels, externalLabels []labels.Label) labels.Labels {
+	if len(externalLabels) == 0 {
+		return ls
+	}
+
 	b := labels.NewScratchBuilder(ls.Len() + len(externalLabels))
 	j := 0
 	ls.Range(func(l labels.Label) {