diff --git a/api/git/git_integration_test.go b/api/git/git_integration_test.go
index 77f2a6429..45504101e 100644
--- a/api/git/git_integration_test.go
+++ b/api/git/git_integration_test.go
@@ -297,9 +297,9 @@ func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) {
 	assert.GreaterOrEqual(t, len(refs), 1)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 
-	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
+	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false)
 	assert.Error(t, err)
-	assert.Equal(t, 0, service.repoRefCache.Len())
+	assert.Equal(t, 1, service.repoRefCache.Len())
 }
 
 func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
@@ -325,10 +325,13 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
 	assert.GreaterOrEqual(t, len(files), 1)
 	assert.Equal(t, 2, service.repoFileCache.Len())
 
+	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false)
+	assert.Error(t, err)
+	assert.Equal(t, 1, service.repoRefCache.Len())
+
 	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
 	assert.Error(t, err)
-	assert.Equal(t, 0, service.repoRefCache.Len())
-
+	assert.Equal(t, 1, service.repoRefCache.Len())
 	// The relevant file caches should be removed too
 	assert.Equal(t, 0, service.repoFileCache.Len())
 }
diff --git a/api/git/service.go b/api/git/service.go
index 0ce24596a..a98c8984c 100644
--- a/api/git/service.go
+++ b/api/git/service.go
@@ -163,9 +163,10 @@ func (service *Service) LatestCommitID(repositoryURL, referenceName, username, p
 
 // ListRefs will list target repository's references without cloning the repository
 func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
+	refCacheKey := generateCacheKey(repositoryURL, password)
 	if service.cacheEnabled && hardRefresh {
 		// Should remove the cache explicitly, so that the following normal list can show the correct result
-		service.repoRefCache.Remove(repositoryURL)
+		service.repoRefCache.Remove(refCacheKey)
 		// Remove file caches pointed to the same repository
 		for _, fileCacheKey := range service.repoFileCache.Keys() {
 			key, ok := fileCacheKey.(string)
@@ -179,7 +180,7 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
 
 	if service.repoRefCache != nil {
 		// Lookup the refs cache first
-		cache, ok := service.repoRefCache.Get(repositoryURL)
+		cache, ok := service.repoRefCache.Get(refCacheKey)
 		if ok {
 			refs, success := cache.([]string)
 			if success {
@@ -211,7 +212,7 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
 	}
 
 	if service.cacheEnabled && service.repoRefCache != nil {
-		service.repoRefCache.Add(options.repositoryUrl, refs)
+		service.repoRefCache.Add(refCacheKey, refs)
 	}
 	return refs, nil
 }