Try to use available URI if all pooled requests are sleeping

src/FileEntry.cc

@@ -136,16 +136,14 @@ OutputIterator enumerateInFlightHosts(InputIterator first, InputIterator last,
 }
 } // namespace
 
-std::shared_ptr<Request> FileEntry::getRequest(
+std::shared_ptr<Request> FileEntry::getRequestWithInFlightHosts(
     URISelector* selector, bool uriReuse,
     const std::vector<std::pair<size_t, std::string>>& usedHosts,
-    const std::string& referer, const std::string& method)
+    const std::string& referer, const std::string& method,
+    const std::vector<std::string>& inFlightHosts)
 {
   std::shared_ptr<Request> req;
-  if (requestPool_.empty()) {
+
-    std::vector<std::string> inFlightHosts;
-    enumerateInFlightHosts(inFlightRequests_.begin(), inFlightRequests_.end(),
-                           std::back_inserter(inFlightHosts));
   for (int g = 0; g < 2; ++g) {
     std::vector<std::string> pending;
     std::vector<std::string> ignoreHost;
@@ -156,7 +154,7 @@ std::shared_ptr<Request> FileEntry::getRequest(
       }
       req = std::make_shared<Request>();
       if (req->setUri(uri)) {
-          if (std::count(inFlightHosts.begin(), inFlightHosts.end(),
+        if (std::count(std::begin(inFlightHosts), std::end(inFlightHosts),
                        req->getHost()) >= maxConnectionPerServer_) {
           pending.push_back(uri);
           ignoreHost.push_back(req->getHost());
@@ -179,36 +177,66 @@ std::shared_ptr<Request> FileEntry::getRequest(
         req.reset();
       }
     }
-      uris_.insert(uris_.begin(), pending.begin(), pending.end());
+    uris_.insert(std::begin(uris_), std::begin(pending), std::end(pending));
     if (g == 0 && uriReuse && !req && uris_.size() == pending.size()) {
       // Reuse URIs other than ones in pending
       reuseUri(ignoreHost);
+      continue;
     }
-      else {
+
-        break;
+    return req;
   }
+}
+
+std::shared_ptr<Request> FileEntry::getRequest(
+    URISelector* selector, bool uriReuse,
+    const std::vector<std::pair<size_t, std::string>>& usedHosts,
+    const std::string& referer, const std::string& method)
+{
+  std::shared_ptr<Request> req;
+  if (requestPool_.empty()) {
+    std::vector<std::string> inFlightHosts;
+    enumerateInFlightHosts(std::begin(inFlightRequests_),
+                           std::end(inFlightRequests_),
+                           std::back_inserter(inFlightHosts));
+    return getRequestWithInFlightHosts(selector, uriReuse, usedHosts, referer,
+                                       method, inFlightHosts);
   }
-  }
+
-  else {
   // Skip Request object if it is still
   // sleeping(Request::getWakeTime() < global::wallclock()).  If all
-    // pooled objects are sleeping, return first one.  Caller should
+  // pooled objects are sleeping, we may return first one.  Caller
-    // inspect returned object's getWakeTime().
+  // should inspect returned object's getWakeTime().
-    auto i = requestPool_.begin();
+  auto i = std::begin(requestPool_);
-    auto eoi = requestPool_.end();
+  for (; i != std::end(requestPool_); ++i) {
-    for (; i != eoi; ++i) {
     if ((*i)->getWakeTime() <= global::wallclock()) {
       break;
     }
   }
-    if (i == eoi) {
+  if (i == std::end(requestPool_)) {
-      i = requestPool_.begin();
+    // all requests are sleeping; try to another URI
+    std::vector<std::string> inFlightHosts;
+    enumerateInFlightHosts(std::begin(inFlightRequests_),
+                           std::end(inFlightRequests_),
+                           std::back_inserter(inFlightHosts));
+    enumerateInFlightHosts(std::begin(requestPool_), std::end(requestPool_),
+                           std::back_inserter(inFlightHosts));
+
+    req = getRequestWithInFlightHosts(selector, uriReuse, usedHosts, referer,
+                                      method, inFlightHosts);
+    if (!req || req->getUri() == (*std::begin(requestPool_))->getUri()) {
+      i = std::begin(requestPool_);
     }
+  }
+
+  if (i != std::end(requestPool_)) {
     req = *i;
     requestPool_.erase(i);
-    inFlightRequests_.insert(req);
     A2_LOG_DEBUG(fmt("Picked up from pool: %s", req->getUri().c_str()));
   }
+
+  inFlightRequests_.insert(req);
+
   return req;
 }
 

src/FileEntry.h

@@ -97,6 +97,12 @@ private:
 
   void storePool(const std::shared_ptr<Request>& request);
 
+  std::shared_ptr<Request> getRequestWithInFlightHosts(
+      URISelector* selector, bool uriReuse,
+      const std::vector<std::pair<size_t, std::string>>& usedHosts,
+      const std::string& referer, const std::string& method,
+      const std::vector<std::string>& inFlightHosts);
+
 public:
   FileEntry();