Merge pull request #10502 from ydb-platform/mergelibs-241016-1210

Library import 241016-1210

1 files changed, 1006 insertions, 1186 deletions

diff --git a/contrib/libs/curl/lib/multi.c b/contrib/libs/curl/lib/multi.c
index 78e5c0a1e5..5456113be7 100644
--- a/contrib/libs/curl/lib/multi.c
+++ b/contrib/libs/curl/lib/multi.c
@@ -57,7 +57,7 @@
 
 /*
   CURL_SOCKET_HASH_TABLE_SIZE should be a prime number. Increasing it from 97
-  to 911 takes on a 32-bit machine 4 x 804 = 3211 more bytes. Still, every
+  to 911 takes on a 32-bit machine 4 x 804 = 3211 more bytes.  Still, every
   CURL handle takes 45-50 K memory, therefore this 3K are not significant.
 */
 #ifndef CURL_SOCKET_HASH_TABLE_SIZE
@@ -86,26 +86,19 @@
   ((x) && (x)->magic == CURL_MULTI_HANDLE)
 #endif
 
-static void move_pending_to_connect(struct Curl_multi *multi,
-                                    struct Curl_easy *data);
 static CURLMcode singlesocket(struct Curl_multi *multi,
                               struct Curl_easy *data);
 static CURLMcode add_next_timeout(struct curltime now,
                                   struct Curl_multi *multi,
                                   struct Curl_easy *d);
 static CURLMcode multi_timeout(struct Curl_multi *multi,
-                               struct curltime *expire_time,
                                long *timeout_ms);
 static void process_pending_handles(struct Curl_multi *multi);
-static void multi_xfer_bufs_free(struct Curl_multi *multi);
-static void Curl_expire_ex(struct Curl_easy *data, const struct curltime *nowp,
-                           timediff_t milli, expire_id id);
 
 #ifdef DEBUGBUILD
 static const char * const multi_statename[]={
   "INIT",
   "PENDING",
-  "SETUP",
   "CONNECT",
   "RESOLVING",
   "CONNECTING",
@@ -138,7 +131,7 @@ static void init_completed(struct Curl_easy *data)
 {
   /* this is a completed transfer */
 
-  /* Important: reset the conn pointer so that we do not point to memory
+  /* Important: reset the conn pointer so that we don't point to memory
      that could be freed anytime */
   Curl_detach_connection(data);
   Curl_expire_clear(data); /* stop all timers */
@@ -155,7 +148,6 @@ static void mstate(struct Curl_easy *data, CURLMstate state
   static const init_multistate_func finit[MSTATE_LAST] = {
     NULL,              /* INIT */
     NULL,              /* PENDING */
-    NULL,              /* SETUP */
     Curl_init_CONNECT, /* CONNECT */
     NULL,              /* RESOLVING */
     NULL,              /* CONNECTING */
@@ -178,7 +170,7 @@ static void mstate(struct Curl_easy *data, CURLMstate state
 #endif
 
   if(oldstate == state)
-    /* do not bother when the new state is the same as the old state */
+    /* don't bother when the new state is the same as the old state */
     return;
 
   data->mstate = state;
@@ -194,13 +186,9 @@ static void mstate(struct Curl_easy *data, CURLMstate state
 #endif
 
   if(state == MSTATE_COMPLETED) {
-    /* changing to COMPLETED means there is one less easy handle 'alive' */
+    /* changing to COMPLETED means there's one less easy handle 'alive' */
     DEBUGASSERT(data->multi->num_alive > 0);
     data->multi->num_alive--;
-    if(!data->multi->num_alive) {
-      /* free the transfer buffer when we have no more active transfers */
-      multi_xfer_bufs_free(data->multi);
-    }
   }
 
   /* if this state has an init-function, run it */
@@ -250,8 +238,10 @@ static size_t trhash(void *key, size_t key_length, size_t slots_num)
 
 static size_t trhash_compare(void *k1, size_t k1_len, void *k2, size_t k2_len)
 {
+  (void)k1_len;
   (void)k2_len;
-  return !memcmp(k1, k2, k1_len);
+
+  return *(struct Curl_easy **)k1 == *(struct Curl_easy **)k2;
 }
 
 static void trhash_dtor(void *nada)
@@ -344,7 +334,7 @@ static size_t hash_fd(void *key, size_t key_length, size_t slots_num)
   curl_socket_t fd = *((curl_socket_t *) key);
   (void) key_length;
 
-  return (fd % (curl_socket_t)slots_num);
+  return (fd % slots_num);
 }
 
 /*
@@ -355,33 +345,22 @@ static size_t hash_fd(void *key, size_t key_length, size_t slots_num)
  * "Some tests at 7000 and 9000 connections showed that the socket hash lookup
  * is somewhat of a bottle neck. Its current implementation may be a bit too
  * limiting. It simply has a fixed-size array, and on each entry in the array
- * it has a linked list with entries. The hash only checks which list to scan
- * through. The code I had used so for used a list with merely 7 slots (as
- * that is what the DNS hash uses) but with 7000 connections that would make
- * an average of 1000 nodes in each list to run through. I upped that to 97
- * slots (I believe a prime is suitable) and noticed a significant speed
- * increase. I need to reconsider the hash implementation or use a rather
+ * it has a linked list with entries. So the hash only checks which list to
+ * scan through. The code I had used so for used a list with merely 7 slots
+ * (as that is what the DNS hash uses) but with 7000 connections that would
+ * make an average of 1000 nodes in each list to run through. I upped that to
+ * 97 slots (I believe a prime is suitable) and noticed a significant speed
+ * increase.  I need to reconsider the hash implementation or use a rather
  * large default value like this. At 9000 connections I was still below 10us
  * per call."
  *
  */
-static void sh_init(struct Curl_hash *hash, size_t hashsize)
+static void sh_init(struct Curl_hash *hash, int hashsize)
 {
   Curl_hash_init(hash, hashsize, hash_fd, fd_key_compare,
                  sh_freeentry);
 }
 
-/* multi->proto_hash destructor. Should never be called as elements
- * MUST be added with their own destructor */
-static void ph_freeentry(void *p)
-{
-  (void)p;
-  /* Will always be FALSE. Cannot use a 0 assert here since compilers
-   * are not in agreement if they then want a NORETURN attribute or
-   * not. *sigh* */
-  DEBUGASSERT(p == NULL);
-}
-
 /*
  * multi_addmsg()
  *
@@ -390,12 +369,13 @@ static void ph_freeentry(void *p)
  */
 static void multi_addmsg(struct Curl_multi *multi, struct Curl_message *msg)
 {
-  Curl_llist_append(&multi->msglist, msg, &msg->list);
+  Curl_llist_insert_next(&multi->msglist, multi->msglist.tail, msg,
+                         &msg->list);
 }
 
-struct Curl_multi *Curl_multi_handle(size_t hashsize, /* socket hash */
-                                     size_t chashsize, /* connection hash */
-                                     size_t dnssize) /* dns hash */
+struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */
+                                     int chashsize, /* connection hash */
+                                     int dnssize) /* dns hash */
 {
   struct Curl_multi *multi = calloc(1, sizeof(struct Curl_multi));
 
@@ -408,21 +388,15 @@ struct Curl_multi *Curl_multi_handle(size_t hashsize, /* socket hash */
 
   sh_init(&multi->sockhash, hashsize);
 
-  Curl_hash_init(&multi->proto_hash, 23,
-                 Curl_hash_str, Curl_str_key_compare, ph_freeentry);
-
-  if(Curl_cpool_init(&multi->cpool, Curl_on_disconnect,
-                         multi, NULL, chashsize))
+  if(Curl_conncache_init(&multi->conn_cache, chashsize))
     goto error;
 
   Curl_llist_init(&multi->msglist, NULL);
-  Curl_llist_init(&multi->process, NULL);
   Curl_llist_init(&multi->pending, NULL);
   Curl_llist_init(&multi->msgsent, NULL);
 
   multi->multiplexing = TRUE;
   multi->max_concurrent_streams = 100;
-  multi->last_timeout_ms = -1;
 
 #ifdef USE_WINSOCK
   multi->wsa_event = WSACreateEvent();
@@ -430,7 +404,14 @@ struct Curl_multi *Curl_multi_handle(size_t hashsize, /* socket hash */
     goto error;
 #else
 #ifdef ENABLE_WAKEUP
-  if(wakeup_create(multi->wakeup_pair, TRUE) < 0) {
+  if(wakeup_create(multi->wakeup_pair) < 0) {
+    multi->wakeup_pair[0] = CURL_SOCKET_BAD;
+    multi->wakeup_pair[1] = CURL_SOCKET_BAD;
+  }
+  else if(curlx_nonblock(multi->wakeup_pair[0], TRUE) < 0 ||
+          curlx_nonblock(multi->wakeup_pair[1], TRUE) < 0) {
+    wakeup_close(multi->wakeup_pair[0]);
+    wakeup_close(multi->wakeup_pair[1]);
     multi->wakeup_pair[0] = CURL_SOCKET_BAD;
     multi->wakeup_pair[1] = CURL_SOCKET_BAD;
   }
@@ -442,9 +423,8 @@ struct Curl_multi *Curl_multi_handle(size_t hashsize, /* socket hash */
 error:
 
   sockhash_destroy(&multi->sockhash);
-  Curl_hash_destroy(&multi->proto_hash);
   Curl_hash_destroy(&multi->hostcache);
-  Curl_cpool_destroy(&multi->cpool);
+  Curl_conncache_destroy(&multi->conn_cache);
   free(multi);
   return NULL;
 }
@@ -470,6 +450,52 @@ static void multi_warn_debug(struct Curl_multi *multi, struct Curl_easy *data)
 #define multi_warn_debug(x,y) Curl_nop_stmt
 #endif
 
+/* returns TRUE if the easy handle is supposed to be present in the main link
+   list */
+static bool in_main_list(struct Curl_easy *data)
+{
+  return ((data->mstate != MSTATE_PENDING) &&
+          (data->mstate != MSTATE_MSGSENT));
+}
+
+static void link_easy(struct Curl_multi *multi,
+                      struct Curl_easy *data)
+{
+  /* We add the new easy entry last in the list. */
+  data->next = NULL; /* end of the line */
+  if(multi->easyp) {
+    struct Curl_easy *last = multi->easylp;
+    last->next = data;
+    data->prev = last;
+    multi->easylp = data; /* the new last node */
+  }
+  else {
+    /* first node, make prev NULL! */
+    data->prev = NULL;
+    multi->easylp = multi->easyp = data; /* both first and last */
+  }
+}
+
+/* unlink the given easy handle from the linked list of easy handles */
+static void unlink_easy(struct Curl_multi *multi,
+                        struct Curl_easy *data)
+{
+  /* make the previous node point to our next */
+  if(data->prev)
+    data->prev->next = data->next;
+  else
+    multi->easyp = data->next; /* point to first node */
+
+  /* make our next point to our previous node */
+  if(data->next)
+    data->next->prev = data->prev;
+  else
+    multi->easylp = data->prev; /* point to last node */
+
+  data->prev = data->next = NULL;
+}
+
+
 CURLMcode curl_multi_add_handle(struct Curl_multi *multi,
                                 struct Curl_easy *data)
 {
@@ -499,27 +525,18 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi,
     multi->dead = FALSE;
   }
 
-  if(data->multi_easy) {
-    /* if this easy handle was previously used for curl_easy_perform(), there
-       is a private multi handle here that we can kill */
-    curl_multi_cleanup(data->multi_easy);
-    data->multi_easy = NULL;
-  }
-
   /* Initialize timeout list for this handle */
   Curl_llist_init(&data->state.timeoutlist, NULL);
 
   /*
-   * No failure allowed in this function beyond this point. No modification of
-   * easy nor multi handle allowed before this except for potential multi's
-   * connection pool growing which will not be undone in this function no
-   * matter what.
+   * No failure allowed in this function beyond this point. And no
+   * modification of easy nor multi handle allowed before this except for
+   * potential multi's connection cache growing which won't be undone in this
+   * function no matter what.
    */
   if(data->set.errorbuffer)
     data->set.errorbuffer[0] = 0;
 
-  data->state.os_errno = 0;
-
   /* make the Curl_easy refer back to this multi handle - before Curl_expire()
      is called. */
   data->multi = multi;
@@ -532,11 +549,21 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi,
      happen. */
   Curl_expire(data, 0, EXPIRE_RUN_NOW);
 
+  /* A somewhat crude work-around for a little glitch in Curl_update_timer()
+     that happens if the lastcall time is set to the same time when the handle
+     is removed as when the next handle is added, as then the check in
+     Curl_update_timer() that prevents calling the application multiple times
+     with the same timer info will not trigger and then the new handle's
+     timeout will not be notified to the app.
+
+     The work-around is thus simply to clear the 'lastcall' variable to force
+     Curl_update_timer() to always trigger a callback to the app when a new
+     easy handle is added */
+  memset(&multi->timer_lastcall, 0, sizeof(multi->timer_lastcall));
+
   rc = Curl_update_timer(multi);
-  if(rc) {
-    data->multi = NULL; /* not anymore */
+  if(rc)
     return rc;
-  }
 
   /* set the easy handle */
   multistate(data, MSTATE_INIT);
@@ -549,6 +576,13 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi,
     data->dns.hostcachetype = HCACHE_MULTI;
   }
 
+  /* Point to the shared or multi handle connection cache */
+  if(data->share && (data->share->specifier & (1<< CURL_LOCK_DATA_CONNECT)))
+    data->state.conn_cache = &data->share->conn_cache;
+  else
+    data->state.conn_cache = &multi->conn_cache;
+  data->state.lastconnect_id = -1;
+
 #ifdef USE_LIBPSL
   /* Do the same for PSL. */
   if(data->share && (data->share->specifier & (1 << CURL_LOCK_DATA_PSL)))
@@ -557,8 +591,7 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi,
     data->psl = &multi->psl;
 #endif
 
-  /* add the easy handle to the process list */
-  Curl_llist_append(&multi->process, data, &data->multi_queue);
+  link_easy(multi, data);
 
   /* increase the node-counter */
   multi->num_easy++;
@@ -566,12 +599,21 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi,
   /* increase the alive-counter */
   multi->num_alive++;
 
-  /* the identifier inside the multi instance */
-  data->mid = multi->next_easy_mid++;
-  if(multi->next_easy_mid <= 0)
-    multi->next_easy_mid = 0;
+  CONNCACHE_LOCK(data);
+  /* The closure handle only ever has default timeouts set. To improve the
+     state somewhat we clone the timeouts from each added handle so that the
+     closure handle always has the same timeouts as the most recently added
+     easy handle. */
+  data->state.conn_cache->closure_handle->set.timeout = data->set.timeout;
+  data->state.conn_cache->closure_handle->set.server_response_timeout =
+    data->set.server_response_timeout;
+  data->state.conn_cache->closure_handle->set.no_signal =
+    data->set.no_signal;
+  data->id = data->state.conn_cache->next_easy_id++;
+  if(data->state.conn_cache->next_easy_id <= 0)
+    data->state.conn_cache->next_easy_id = 0;
+  CONNCACHE_UNLOCK(data);
 
-  Curl_cpool_xfer_init(data);
   multi_warn_debug(multi, data);
 
   return CURLM_OK;
@@ -593,101 +635,13 @@ static void debug_print_sock_hash(void *p)
 }
 #endif
 
-struct multi_done_ctx {
-  BIT(premature);
-};
-
-static void multi_done_locked(struct connectdata *conn,
-                              struct Curl_easy *data,
-                              void *userdata)
-{
-  struct multi_done_ctx *mdctx = userdata;
-
-  Curl_detach_connection(data);
-
-  if(CONN_INUSE(conn)) {
-    /* Stop if still used. */
-    DEBUGF(infof(data, "Connection still in use %zu, "
-                 "no more multi_done now!",
-                 Curl_llist_count(&conn->easyq)));
-    return;
-  }
-
-  data->state.done = TRUE; /* called just now! */
-  data->state.recent_conn_id = conn->connection_id;
-
-  if(conn->dns_entry)
-    Curl_resolv_unlink(data, &conn->dns_entry); /* done with this */
-  Curl_hostcache_prune(data);
-
-  /* if data->set.reuse_forbid is TRUE, it means the libcurl client has
-     forced us to close this connection. This is ignored for requests taking
-     place in a NTLM/NEGOTIATE authentication handshake
-
-     if conn->bits.close is TRUE, it means that the connection should be
-     closed in spite of all our efforts to be nice, due to protocol
-     restrictions in our or the server's end
-
-     if premature is TRUE, it means this connection was said to be DONE before
-     the entire request operation is complete and thus we cannot know in what
-     state it is for reusing, so we are forced to close it. In a perfect world
-     we can add code that keep track of if we really must close it here or not,
-     but currently we have no such detail knowledge.
-  */
-
-  if((data->set.reuse_forbid
-#if defined(USE_NTLM)
-      && !(conn->http_ntlm_state == NTLMSTATE_TYPE2 ||
-           conn->proxy_ntlm_state == NTLMSTATE_TYPE2)
-#endif
-#if defined(USE_SPNEGO)
-      && !(conn->http_negotiate_state == GSS_AUTHRECV ||
-           conn->proxy_negotiate_state == GSS_AUTHRECV)
-#endif
-     ) || conn->bits.close
-       || (mdctx->premature && !Curl_conn_is_multiplex(conn, FIRSTSOCKET))) {
-    DEBUGF(infof(data, "multi_done, not reusing connection=%"
-                 FMT_OFF_T ", forbid=%d"
-                 ", close=%d, premature=%d, conn_multiplex=%d",
-                 conn->connection_id, data->set.reuse_forbid,
-                 conn->bits.close, mdctx->premature,
-                 Curl_conn_is_multiplex(conn, FIRSTSOCKET)));
-    connclose(conn, "disconnecting");
-    Curl_cpool_disconnect(data, conn, mdctx->premature);
-  }
-  else {
-    /* the connection is no longer in use by any transfer */
-    if(Curl_cpool_conn_now_idle(data, conn)) {
-      /* connection kept in the cpool */
-      const char *host =
-#ifndef CURL_DISABLE_PROXY
-        conn->bits.socksproxy ?
-        conn->socks_proxy.host.dispname :
-        conn->bits.httpproxy ? conn->http_proxy.host.dispname :
-#endif
-        conn->bits.conn_to_host ? conn->conn_to_host.dispname :
-        conn->host.dispname;
-      data->state.lastconnect_id = conn->connection_id;
-      infof(data, "Connection #%" FMT_OFF_T " to host %s left intact",
-            conn->connection_id, host);
-    }
-    else {
-      /* connection was removed from the cpool and destroyed. */
-      data->state.lastconnect_id = -1;
-    }
-  }
-}
-
 static CURLcode multi_done(struct Curl_easy *data,
                            CURLcode status,  /* an error if this is called
                                                 after an error was detected */
                            bool premature)
 {
-  CURLcode result, r2;
+  CURLcode result;
   struct connectdata *conn = data->conn;
-  struct multi_done_ctx mdctx;
-
-  memset(&mdctx, 0, sizeof(mdctx));
 
 #if defined(DEBUGBUILD) && !defined(CURL_DISABLE_VERBOSE_STRINGS)
   DEBUGF(infof(data, "multi_done[%s]: status: %d prem: %d done: %d",
@@ -713,12 +667,11 @@ static CURLcode multi_done(struct Curl_easy *data,
   case CURLE_ABORTED_BY_CALLBACK:
   case CURLE_READ_ERROR:
   case CURLE_WRITE_ERROR:
-    /* When we are aborted due to a callback return code it basically have to
-       be counted as premature as there is trouble ahead if we do not. We have
+    /* When we're aborted due to a callback return code it basically have to
+       be counted as premature as there is trouble ahead if we don't. We have
        many callbacks and protocols work differently, we could potentially do
        this more fine-grained in the future. */
     premature = TRUE;
-    FALLTHROUGH();
   default:
     break;
   }
@@ -737,35 +690,116 @@ static CURLcode multi_done(struct Curl_easy *data,
       result = CURLE_ABORTED_BY_CALLBACK;
   }
 
-  /* Make sure that transfer client writes are really done now. */
-  r2 = Curl_xfer_write_done(data, premature);
-  if(r2 && !result)
-    result = r2;
-
   /* Inform connection filters that this transfer is done */
   Curl_conn_ev_data_done(data, premature);
 
   process_pending_handles(data->multi); /* connection / multiplex */
 
-  if(!result)
-    result = Curl_req_done(&data->req, data, premature);
+  Curl_safefree(data->state.ulbuf);
+
+  Curl_client_cleanup(data);
+
+  CONNCACHE_LOCK(data);
+  Curl_detach_connection(data);
+  if(CONN_INUSE(conn)) {
+    /* Stop if still used. */
+    CONNCACHE_UNLOCK(data);
+    DEBUGF(infof(data, "Connection still in use %zu, "
+                 "no more multi_done now!",
+                 conn->easyq.size));
+    return CURLE_OK;
+  }
+
+  data->state.done = TRUE; /* called just now! */
+
+  if(conn->dns_entry) {
+    Curl_resolv_unlock(data, conn->dns_entry); /* done with this */
+    conn->dns_entry = NULL;
+  }
+  Curl_hostcache_prune(data);
+
+  /* if data->set.reuse_forbid is TRUE, it means the libcurl client has
+     forced us to close this connection. This is ignored for requests taking
+     place in a NTLM/NEGOTIATE authentication handshake
+
+     if conn->bits.close is TRUE, it means that the connection should be
+     closed in spite of all our efforts to be nice, due to protocol
+     restrictions in our or the server's end
 
-  /* Under the potential connection pool's share lock, decide what to
-   * do with the transfer's connection. */
-  mdctx.premature = premature;
-  Curl_cpool_do_locked(data, data->conn, multi_done_locked, &mdctx);
+     if premature is TRUE, it means this connection was said to be DONE before
+     the entire request operation is complete and thus we can't know in what
+     state it is for reusing, so we're forced to close it. In a perfect world
+     we can add code that keep track of if we really must close it here or not,
+     but currently we have no such detail knowledge.
+  */
 
+  data->state.recent_conn_id = conn->connection_id;
+  if((data->set.reuse_forbid
+#if defined(USE_NTLM)
+      && !(conn->http_ntlm_state == NTLMSTATE_TYPE2 ||
+           conn->proxy_ntlm_state == NTLMSTATE_TYPE2)
+#endif
+#if defined(USE_SPNEGO)
+      && !(conn->http_negotiate_state == GSS_AUTHRECV ||
+           conn->proxy_negotiate_state == GSS_AUTHRECV)
+#endif
+     ) || conn->bits.close
+       || (premature && !Curl_conn_is_multiplex(conn, FIRSTSOCKET))) {
+    DEBUGF(infof(data, "multi_done, not reusing connection=%"
+                       CURL_FORMAT_CURL_OFF_T ", forbid=%d"
+                       ", close=%d, premature=%d, conn_multiplex=%d",
+                 conn->connection_id,
+                 data->set.reuse_forbid, conn->bits.close, premature,
+                 Curl_conn_is_multiplex(conn, FIRSTSOCKET)));
+    connclose(conn, "disconnecting");
+    Curl_conncache_remove_conn(data, conn, FALSE);
+    CONNCACHE_UNLOCK(data);
+    Curl_disconnect(data, conn, premature);
+  }
+  else {
+    char buffer[256];
+    const char *host =
+#ifndef CURL_DISABLE_PROXY
+      conn->bits.socksproxy ?
+      conn->socks_proxy.host.dispname :
+      conn->bits.httpproxy ? conn->http_proxy.host.dispname :
+#endif
+      conn->bits.conn_to_host ? conn->conn_to_host.dispname :
+      conn->host.dispname;
+    /* create string before returning the connection */
+    curl_off_t connection_id = conn->connection_id;
+    msnprintf(buffer, sizeof(buffer),
+              "Connection #%" CURL_FORMAT_CURL_OFF_T " to host %s left intact",
+              connection_id, host);
+    /* the connection is no longer in use by this transfer */
+    CONNCACHE_UNLOCK(data);
+    if(Curl_conncache_return_conn(data, conn)) {
+      /* remember the most recently used connection */
+      data->state.lastconnect_id = connection_id;
+      data->state.recent_conn_id = connection_id;
+      infof(data, "%s", buffer);
+    }
+    else
+      data->state.lastconnect_id = -1;
+  }
+
+  Curl_safefree(data->state.buffer);
   return result;
 }
 
-static void close_connect_only(struct connectdata *conn,
-                               struct Curl_easy *data,
-                               void *userdata)
+static int close_connect_only(struct Curl_easy *data,
+                              struct connectdata *conn, void *param)
 {
-  (void)userdata;
-  (void)data;
-  if(conn->connect_only)
-    connclose(conn, "Removing connect-only easy handle");
+  (void)param;
+  if(data->state.lastconnect_id != conn->connection_id)
+    return 0;
+
+  if(!conn->connect_only)
+    return 1;
+
+  connclose(conn, "Removing connect-only easy handle");
+
+  return 1;
 }
 
 CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
@@ -773,16 +807,15 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
 {
   struct Curl_easy *easy = data;
   bool premature;
-  struct Curl_llist_node *e;
+  struct Curl_llist_element *e;
   CURLMcode rc;
-  bool removed_timer = FALSE;
 
   /* First, make some basic checks that the CURLM handle is a good handle */
   if(!GOOD_MULTI_HANDLE(multi))
     return CURLM_BAD_HANDLE;
 
   /* Verify that we got a somewhat good easy handle too */
-  if(!GOOD_EASY_HANDLE(data) || !multi->num_easy)
+  if(!GOOD_EASY_HANDLE(data))
     return CURLM_BAD_EASY_HANDLE;
 
   /* Prevent users from trying to remove same easy handle more than once */
@@ -809,7 +842,7 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
   if(data->conn &&
      data->mstate > MSTATE_DO &&
      data->mstate < MSTATE_COMPLETED) {
-    /* Set connection owner so that the DONE function closes it. We can
+    /* Set connection owner so that the DONE function closes it.  We can
        safely do this here since connection is killed. */
     streamclose(data->conn, "Removed with partial response");
   }
@@ -818,7 +851,7 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
     /* multi_done() clears the association between the easy handle and the
        connection.
 
-       Note that this ignores the return code simply because there is
+       Note that this ignores the return code simply because there's
        nothing really useful to do with it anyway! */
     (void)multi_done(data, data->result, premature);
   }
@@ -826,10 +859,18 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
   /* The timer must be shut down before data->multi is set to NULL, else the
      timenode will remain in the splay tree after curl_easy_cleanup is
      called. Do it after multi_done() in case that sets another time! */
-  removed_timer = Curl_expire_clear(data);
+  Curl_expire_clear(data);
 
-  /* the handle is in a list, remove it from whichever it is */
-  Curl_node_remove(&data->multi_queue);
+  if(data->connect_queue.ptr) {
+    /* the handle is in the pending or msgsent lists, so go ahead and remove
+       it */
+    if(data->mstate == MSTATE_PENDING)
+      Curl_llist_remove(&multi->pending, &data->connect_queue, NULL);
+    else
+      Curl_llist_remove(&multi->msgsent, &data->connect_queue, NULL);
+  }
+  if(in_main_list(data))
+    unlink_easy(multi, data);
 
   if(data->dns.hostcachetype == HCACHE_MULTI) {
     /* stop using the multi handle's DNS cache, *after* the possible
@@ -844,7 +885,7 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
      what we want */
   data->mstate = MSTATE_COMPLETED;
 
-  /* This ignores the return code even in case of problems because there is
+  /* This ignores the return code even in case of problems because there's
      nothing more to do about that, here */
   (void)singlesocket(multi, easy); /* to let the application know what sockets
                                       that vanish with this handle */
@@ -856,7 +897,7 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
     /* This removes a handle that was part the multi interface that used
        CONNECT_ONLY, that connection is now left alive but since this handle
        has bits.close set nothing can use that transfer anymore and it is
-       forbidden from reuse. This easy handle cannot find the connection
+       forbidden from reuse. And this easy handle cannot find the connection
        anymore once removed from the multi handle
 
        Better close the connection here, at once.
@@ -865,14 +906,15 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
     curl_socket_t s;
     s = Curl_getconnectinfo(data, &c);
     if((s != CURL_SOCKET_BAD) && c) {
-      Curl_cpool_disconnect(data, c, TRUE);
+      Curl_conncache_remove_conn(data, c, TRUE);
+      Curl_disconnect(data, c, TRUE);
     }
   }
 
   if(data->state.lastconnect_id != -1) {
     /* Mark any connect-only connection for closure */
-    Curl_cpool_do_by_id(data, data->state.lastconnect_id,
-                            close_connect_only, NULL);
+    Curl_conncache_foreach(data, data->state.conn_cache,
+                           NULL, close_connect_only);
   }
 
 #ifdef USE_LIBPSL
@@ -881,31 +923,33 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
     data->psl = NULL;
 #endif
 
-  /* make sure there is no pending message in the queue sent from this easy
+  /* as this was using a shared connection cache we clear the pointer to that
+     since we're not part of that multi handle anymore */
+  data->state.conn_cache = NULL;
+
+  data->multi = NULL; /* clear the association to this multi handle */
+
+  /* make sure there's no pending message in the queue sent from this easy
      handle */
-  for(e = Curl_llist_head(&multi->msglist); e; e = Curl_node_next(e)) {
-    struct Curl_message *msg = Curl_node_elem(e);
+  for(e = multi->msglist.head; e; e = e->next) {
+    struct Curl_message *msg = e->ptr;
 
     if(msg->extmsg.easy_handle == easy) {
-      Curl_node_remove(e);
+      Curl_llist_remove(&multi->msglist, e, NULL);
       /* there can only be one from this specific handle */
       break;
     }
   }
 
-  data->multi = NULL; /* clear the association to this multi handle */
-  data->mid = -1;
-
   /* NOTE NOTE NOTE
      We do not touch the easy handle here! */
   multi->num_easy--; /* one less to care about now */
+
   process_pending_handles(multi);
 
-  if(removed_timer) {
-    rc = Curl_update_timer(multi);
-    if(rc)
-      return rc;
-  }
+  rc = Curl_update_timer(multi);
+  if(rc)
+    return rc;
   return CURLM_OK;
 }
 
@@ -926,7 +970,7 @@ void Curl_detach_connection(struct Curl_easy *data)
   struct connectdata *conn = data->conn;
   if(conn) {
     Curl_conn_ev_data_detach(conn, data);
-    Curl_node_remove(&data->conn_queue);
+    Curl_llist_remove(&conn->easyq, &data->conn_queue, NULL);
   }
   data->conn = NULL;
 }
@@ -937,114 +981,43 @@ void Curl_detach_connection(struct Curl_easy *data)
  * This is the only function that should assign data->conn
  */
 void Curl_attach_connection(struct Curl_easy *data,
-                            struct connectdata *conn)
+                             struct connectdata *conn)
 {
-  DEBUGASSERT(data);
   DEBUGASSERT(!data->conn);
   DEBUGASSERT(conn);
   data->conn = conn;
-  Curl_llist_append(&conn->easyq, data, &data->conn_queue);
+  Curl_llist_insert_next(&conn->easyq, conn->easyq.tail, data,
+                         &data->conn_queue);
   if(conn->handler && conn->handler->attach)
     conn->handler->attach(data, conn);
   Curl_conn_ev_data_attach(conn, data);
 }
 
-static int connecting_getsock(struct Curl_easy *data, curl_socket_t *socks)
+static int domore_getsock(struct Curl_easy *data,
+                          struct connectdata *conn,
+                          curl_socket_t *socks)
 {
-  struct connectdata *conn = data->conn;
-  curl_socket_t sockfd;
-
-  if(!conn)
-    return GETSOCK_BLANK;
-  sockfd = Curl_conn_get_socket(data, FIRSTSOCKET);
-  if(sockfd != CURL_SOCKET_BAD) {
-    /* Default is to wait to something from the server */
-    socks[0] = sockfd;
-    return GETSOCK_READSOCK(0);
-  }
-  return GETSOCK_BLANK;
-}
-
-static int protocol_getsock(struct Curl_easy *data, curl_socket_t *socks)
-{
-  struct connectdata *conn = data->conn;
-  curl_socket_t sockfd;
-
-  if(!conn)
-    return GETSOCK_BLANK;
-  if(conn->handler->proto_getsock)
-    return conn->handler->proto_getsock(data, conn, socks);
-  sockfd = Curl_conn_get_socket(data, FIRSTSOCKET);
-  if(sockfd != CURL_SOCKET_BAD) {
-    /* Default is to wait to something from the server */
-    socks[0] = sockfd;
-    return GETSOCK_READSOCK(0);
-  }
-  return GETSOCK_BLANK;
-}
-
-static int domore_getsock(struct Curl_easy *data, curl_socket_t *socks)
-{
-  struct connectdata *conn = data->conn;
-  if(!conn)
-    return GETSOCK_BLANK;
-  if(conn->handler->domore_getsock)
+  if(conn && conn->handler->domore_getsock)
     return conn->handler->domore_getsock(data, conn, socks);
-  else if(conn->sockfd != CURL_SOCKET_BAD) {
-    /* Default is that we want to send something to the server */
-    socks[0] = conn->sockfd;
-    return GETSOCK_WRITESOCK(0);
-  }
   return GETSOCK_BLANK;
 }
 
-static int doing_getsock(struct Curl_easy *data, curl_socket_t *socks)
+static int doing_getsock(struct Curl_easy *data,
+                         struct connectdata *conn,
+                         curl_socket_t *socks)
 {
-  struct connectdata *conn = data->conn;
-  if(!conn)
-    return GETSOCK_BLANK;
-  if(conn->handler->doing_getsock)
+  if(conn && conn->handler->doing_getsock)
     return conn->handler->doing_getsock(data, conn, socks);
-  else if(conn->sockfd != CURL_SOCKET_BAD) {
-    /* Default is that we want to send something to the server */
-    socks[0] = conn->sockfd;
-    return GETSOCK_WRITESOCK(0);
-  }
   return GETSOCK_BLANK;
 }
 
-static int perform_getsock(struct Curl_easy *data, curl_socket_t *sock)
+static int protocol_getsock(struct Curl_easy *data,
+                            struct connectdata *conn,
+                            curl_socket_t *socks)
 {
-  struct connectdata *conn = data->conn;
-  if(!conn)
-    return GETSOCK_BLANK;
-  else if(conn->handler->perform_getsock)
-    return conn->handler->perform_getsock(data, conn, sock);
-  else {
-    /* Default is to obey the data->req.keepon flags for send/recv */
-    int bitmap = GETSOCK_BLANK;
-    unsigned sockindex = 0;
-    if(CURL_WANT_RECV(data)) {
-      DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD);
-      bitmap |= GETSOCK_READSOCK(sockindex);
-      sock[sockindex] = conn->sockfd;
-    }
-
-    if(Curl_req_want_send(data)) {
-      if((conn->sockfd != conn->writesockfd) ||
-         bitmap == GETSOCK_BLANK) {
-        /* only if they are not the same socket and we have a readable
-           one, we increase index */
-        if(bitmap != GETSOCK_BLANK)
-          sockindex++; /* increase index if we need two entries */
-
-        DEBUGASSERT(conn->writesockfd != CURL_SOCKET_BAD);
-        sock[sockindex] = conn->writesockfd;
-      }
-      bitmap |= GETSOCK_WRITESOCK(sockindex);
-    }
-    return bitmap;
-  }
+  if(conn->handler->proto_getsock)
+    return conn->handler->proto_getsock(data, conn, socks);
+  return GETSOCK_BLANK;
 }
 
 /* Initializes `poll_set` with the current socket poll actions needed
@@ -1052,7 +1025,6 @@ static int perform_getsock(struct Curl_easy *data, curl_socket_t *sock)
 static void multi_getsock(struct Curl_easy *data,
                           struct easy_pollset *ps)
 {
-  bool expect_sockets = TRUE;
   /* The no connection case can happen when this is called from
      curl_multi_remove_handle() => singlesocket() => multi_getsock().
   */
@@ -1061,75 +1033,45 @@ static void multi_getsock(struct Curl_easy *data,
     return;
 
   switch(data->mstate) {
-  case MSTATE_INIT:
-  case MSTATE_PENDING:
-  case MSTATE_SETUP:
-  case MSTATE_CONNECT:
-    /* nothing to poll for yet */
-    expect_sockets = FALSE;
+  default:
     break;
 
   case MSTATE_RESOLVING:
-    Curl_pollset_add_socks(data, ps, Curl_resolv_getsock);
-    /* connection filters are not involved in this phase. It's ok if we get no
-     * sockets to wait for. Resolving can wake up from other sources. */
-    expect_sockets = FALSE;
-    break;
-
-  case MSTATE_CONNECTING:
-  case MSTATE_TUNNELING:
-    Curl_pollset_add_socks(data, ps, connecting_getsock);
-    Curl_conn_adjust_pollset(data, ps);
-    break;
+    Curl_pollset_add_socks2(data, ps, Curl_resolv_getsock);
+    /* connection filters are not involved in this phase */
+    return;
 
-  case MSTATE_PROTOCONNECT:
   case MSTATE_PROTOCONNECTING:
+  case MSTATE_PROTOCONNECT:
     Curl_pollset_add_socks(data, ps, protocol_getsock);
-    Curl_conn_adjust_pollset(data, ps);
     break;
 
   case MSTATE_DO:
   case MSTATE_DOING:
     Curl_pollset_add_socks(data, ps, doing_getsock);
-    Curl_conn_adjust_pollset(data, ps);
+    break;
+
+  case MSTATE_TUNNELING:
+  case MSTATE_CONNECTING:
     break;
 
   case MSTATE_DOING_MORE:
     Curl_pollset_add_socks(data, ps, domore_getsock);
-    Curl_conn_adjust_pollset(data, ps);
     break;
 
-  case MSTATE_DID: /* same as PERFORMING in regard to polling */
+  case MSTATE_DID: /* since is set after DO is completed, we switch to
+                        waiting for the same as the PERFORMING state */
   case MSTATE_PERFORMING:
-    Curl_pollset_add_socks(data, ps, perform_getsock);
-    Curl_conn_adjust_pollset(data, ps);
+    Curl_pollset_add_socks(data, ps, Curl_single_getsock);
     break;
 
   case MSTATE_RATELIMITING:
-    /* we need to let time pass, ignore socket(s) */
-    expect_sockets = FALSE;
-    break;
-
-  case MSTATE_DONE:
-  case MSTATE_COMPLETED:
-  case MSTATE_MSGSENT:
-    /* nothing more to poll for */
-    expect_sockets = FALSE;
-    break;
-
-  default:
-    failf(data, "multi_getsock: unexpected multi state %d", data->mstate);
-    DEBUGASSERT(0);
-    expect_sockets = FALSE;
-    break;
+    /* nothing to wait for */
+    return;
   }
 
-  if(expect_sockets && !ps->num &&
-     !(data->req.keepon & (KEEP_RECV_PAUSE|KEEP_SEND_PAUSE)) &&
-     Curl_conn_is_ip_connected(data, FIRSTSOCKET)) {
-    infof(data, "WARNING: no socket in pollset, transfer may stall!");
-    DEBUGASSERT(0);
-  }
+  /* Let connection filters add/remove as needed */
+  Curl_conn_adjust_pollset(data, ps);
 }
 
 CURLMcode curl_multi_fdset(struct Curl_multi *multi,
@@ -1139,8 +1081,10 @@ CURLMcode curl_multi_fdset(struct Curl_multi *multi,
   /* Scan through all the easy handles to get the file descriptors set.
      Some easy handles may not have connected to the remote host yet,
      and then we must make sure that is done. */
+  struct Curl_easy *data;
   int this_max_fd = -1;
-  struct Curl_llist_node *e;
+  struct easy_pollset ps;
+  unsigned int i;
   (void)exc_fd_set; /* not used */
 
   if(!GOOD_MULTI_HANDLE(multi))
@@ -1149,22 +1093,20 @@ CURLMcode curl_multi_fdset(struct Curl_multi *multi,
   if(multi->in_callback)
     return CURLM_RECURSIVE_API_CALL;
 
-  for(e = Curl_llist_head(&multi->process); e; e = Curl_node_next(e)) {
-    struct Curl_easy *data = Curl_node_elem(e);
-    unsigned int i;
+  memset(&ps, 0, sizeof(ps));
+  for(data = multi->easyp; data; data = data->next) {
+    multi_getsock(data, &ps);
 
-    multi_getsock(data, &data->last_poll);
-
-    for(i = 0; i < data->last_poll.num; i++) {
-      if(!FDSET_SOCK(data->last_poll.sockets[i]))
-        /* pretend it does not exist */
+    for(i = 0; i < ps.num; i++) {
+      if(!FDSET_SOCK(ps.sockets[i]))
+        /* pretend it doesn't exist */
         continue;
-      if(data->last_poll.actions[i] & CURL_POLL_IN)
-        FD_SET(data->last_poll.sockets[i], read_fd_set);
-      if(data->last_poll.actions[i] & CURL_POLL_OUT)
-        FD_SET(data->last_poll.sockets[i], write_fd_set);
-      if((int)data->last_poll.sockets[i] > this_max_fd)
-        this_max_fd = (int)data->last_poll.sockets[i];
+      if(ps.actions[i] & CURL_POLL_IN)
+        FD_SET(ps.sockets[i], read_fd_set);
+      if(ps.actions[i] & CURL_POLL_OUT)
+        FD_SET(ps.sockets[i], write_fd_set);
+      if((int)ps.sockets[i] > this_max_fd)
+        this_max_fd = (int)ps.sockets[i];
     }
   }
 
@@ -1173,47 +1115,8 @@ CURLMcode curl_multi_fdset(struct Curl_multi *multi,
   return CURLM_OK;
 }
 
-CURLMcode curl_multi_waitfds(struct Curl_multi *multi,
-                             struct curl_waitfd *ufds,
-                             unsigned int size,
-                             unsigned int *fd_count)
-{
-  struct curl_waitfds cwfds;
-  CURLMcode result = CURLM_OK;
-  struct Curl_llist_node *e;
-
-  if(!ufds)
-    return CURLM_BAD_FUNCTION_ARGUMENT;
-
-  if(!GOOD_MULTI_HANDLE(multi))
-    return CURLM_BAD_HANDLE;
-
-  if(multi->in_callback)
-    return CURLM_RECURSIVE_API_CALL;
-
-  Curl_waitfds_init(&cwfds, ufds, size);
-  for(e = Curl_llist_head(&multi->process); e; e = Curl_node_next(e)) {
-    struct Curl_easy *data = Curl_node_elem(e);
-    multi_getsock(data, &data->last_poll);
-    if(Curl_waitfds_add_ps(&cwfds, &data->last_poll)) {
-      result = CURLM_OUT_OF_MEMORY;
-      goto out;
-    }
-  }
-
-  if(Curl_cpool_add_waitfds(&multi->cpool, &cwfds)) {
-    result = CURLM_OUT_OF_MEMORY;
-    goto out;
-  }
-
-out:
-  if(fd_count)
-    *fd_count = cwfds.n;
-  return result;
-}
-
 #ifdef USE_WINSOCK
-/* Reset FD_WRITE for TCP sockets. Nothing is actually sent. UDP sockets cannot
+/* Reset FD_WRITE for TCP sockets. Nothing is actually sent. UDP sockets can't
  * be reset this way because an empty datagram would be sent. #9203
  *
  * "On Windows the internal state of FD_WRITE as returned from
@@ -1238,16 +1141,16 @@ static CURLMcode multi_wait(struct Curl_multi *multi,
                             bool extrawait, /* when no socket, wait */
                             bool use_wakeup)
 {
+  struct Curl_easy *data;
+  struct easy_pollset ps;
   size_t i;
-  struct curltime expire_time;
+  unsigned int nfds = 0;
+  unsigned int curlfds;
   long timeout_internal;
   int retcode = 0;
   struct pollfd a_few_on_stack[NUM_POLLS_ON_STACK];
-  struct curl_pollfds cpfds;
-  unsigned int curl_nfds = 0; /* how many pfds are for curl transfers */
-  CURLMcode result = CURLM_OK;
-  struct Curl_llist_node *e;
-
+  struct pollfd *ufds = &a_few_on_stack[0];
+  bool ufds_malloc = FALSE;
 #ifdef USE_WINSOCK
   WSANETWORKEVENTS wsa_events;
   DEBUGASSERT(multi->wsa_event != WSA_INVALID_EVENT);
@@ -1265,108 +1168,148 @@ static CURLMcode multi_wait(struct Curl_multi *multi,
   if(timeout_ms < 0)
     return CURLM_BAD_FUNCTION_ARGUMENT;
 
-  Curl_pollfds_init(&cpfds, a_few_on_stack, NUM_POLLS_ON_STACK);
+  /* Count up how many fds we have from the multi handle */
+  memset(&ps, 0, sizeof(ps));
+  for(data = multi->easyp; data; data = data->next) {
+    multi_getsock(data, &ps);
+    nfds += ps.num;
+  }
 
-  /* Add the curl handles to our pollfds first */
-  for(e = Curl_llist_head(&multi->process); e; e = Curl_node_next(e)) {
-    struct Curl_easy *data = Curl_node_elem(e);
+  /* If the internally desired timeout is actually shorter than requested from
+     the outside, then use the shorter time! But only if the internal timer
+     is actually larger than -1! */
+  (void)multi_timeout(multi, &timeout_internal);
+  if((timeout_internal >= 0) && (timeout_internal < (long)timeout_ms))
+    timeout_ms = (int)timeout_internal;
 
-    multi_getsock(data, &data->last_poll);
-    if(Curl_pollfds_add_ps(&cpfds, &data->last_poll)) {
-      result = CURLM_OUT_OF_MEMORY;
-      goto out;
-    }
+  curlfds = nfds; /* number of internal file descriptors */
+  nfds += extra_nfds; /* add the externally provided ones */
+
+#ifdef ENABLE_WAKEUP
+#ifdef USE_WINSOCK
+  if(use_wakeup) {
+#else
+  if(use_wakeup && multi->wakeup_pair[0] != CURL_SOCKET_BAD) {
+#endif
+    ++nfds;
   }
+#endif
 
-  if(Curl_cpool_add_pollfds(&multi->cpool, &cpfds)) {
-    result = CURLM_OUT_OF_MEMORY;
-    goto out;
+  if(nfds > NUM_POLLS_ON_STACK) {
+    /* 'nfds' is a 32 bit value and 'struct pollfd' is typically 8 bytes
+       big, so at 2^29 sockets this value might wrap. When a process gets
+       the capability to actually handle over 500 million sockets this
+       calculation needs a integer overflow check. */
+    ufds = malloc(nfds * sizeof(struct pollfd));
+    if(!ufds)
+      return CURLM_OUT_OF_MEMORY;
+    ufds_malloc = TRUE;
   }
+  nfds = 0;
 
-  curl_nfds = cpfds.n; /* what curl internally uses in cpfds */
-  /* Add external file descriptions from poll-like struct curl_waitfd */
-  for(i = 0; i < extra_nfds; i++) {
-    unsigned short events = 0;
-    if(extra_fds[i].events & CURL_WAIT_POLLIN)
-      events |= POLLIN;
-    if(extra_fds[i].events & CURL_WAIT_POLLPRI)
-      events |= POLLPRI;
-    if(extra_fds[i].events & CURL_WAIT_POLLOUT)
-      events |= POLLOUT;
-    if(Curl_pollfds_add_sock(&cpfds, extra_fds[i].fd, events)) {
-      result = CURLM_OUT_OF_MEMORY;
-      goto out;
+  /* only do the second loop if we found descriptors in the first stage run
+     above */
+
+  if(curlfds) {
+    /* Add the curl handles to our pollfds first */
+    for(data = multi->easyp; data; data = data->next) {
+      multi_getsock(data, &ps);
+
+      for(i = 0; i < ps.num; i++) {
+        struct pollfd *ufd = &ufds[nfds++];
+#ifdef USE_WINSOCK
+        long mask = 0;
+#endif
+        ufd->fd = ps.sockets[i];
+        ufd->events = 0;
+        if(ps.actions[i] & CURL_POLL_IN) {
+#ifdef USE_WINSOCK
+          mask |= FD_READ|FD_ACCEPT|FD_CLOSE;
+#endif
+          ufd->events |= POLLIN;
+        }
+        if(ps.actions[i] & CURL_POLL_OUT) {
+#ifdef USE_WINSOCK
+          mask |= FD_WRITE|FD_CONNECT|FD_CLOSE;
+          reset_socket_fdwrite(ps.sockets[i]);
+#endif
+          ufd->events |= POLLOUT;
+        }
+#ifdef USE_WINSOCK
+        if(WSAEventSelect(ps.sockets[i], multi->wsa_event, mask) != 0) {
+          if(ufds_malloc)
+            free(ufds);
+          return CURLM_INTERNAL_ERROR;
+        }
+#endif
+      }
     }
   }
 
+  /* Add external file descriptions from poll-like struct curl_waitfd */
+  for(i = 0; i < extra_nfds; i++) {
 #ifdef USE_WINSOCK
-  /* Set the WSA events based on the collected pollds */
-  for(i = 0; i < cpfds.n; i++) {
     long mask = 0;
-    if(cpfds.pfds[i].events & POLLIN)
+    if(extra_fds[i].events & CURL_WAIT_POLLIN)
       mask |= FD_READ|FD_ACCEPT|FD_CLOSE;
-    if(cpfds.pfds[i].events & POLLPRI)
+    if(extra_fds[i].events & CURL_WAIT_POLLPRI)
       mask |= FD_OOB;
-    if(cpfds.pfds[i].events & POLLOUT) {
+    if(extra_fds[i].events & CURL_WAIT_POLLOUT) {
       mask |= FD_WRITE|FD_CONNECT|FD_CLOSE;
-      reset_socket_fdwrite(cpfds.pfds[i].fd);
+      reset_socket_fdwrite(extra_fds[i].fd);
     }
-    if(mask) {
-      if(WSAEventSelect(cpfds.pfds[i].fd, multi->wsa_event, mask) != 0) {
-        result = CURLM_OUT_OF_MEMORY;
-        goto out;
-      }
+    if(WSAEventSelect(extra_fds[i].fd, multi->wsa_event, mask) != 0) {
+      if(ufds_malloc)
+        free(ufds);
+      return CURLM_INTERNAL_ERROR;
     }
-  }
 #endif
+    ufds[nfds].fd = extra_fds[i].fd;
+    ufds[nfds].events = 0;
+    if(extra_fds[i].events & CURL_WAIT_POLLIN)
+      ufds[nfds].events |= POLLIN;
+    if(extra_fds[i].events & CURL_WAIT_POLLPRI)
+      ufds[nfds].events |= POLLPRI;
+    if(extra_fds[i].events & CURL_WAIT_POLLOUT)
+      ufds[nfds].events |= POLLOUT;
+    ++nfds;
+  }
 
 #ifdef ENABLE_WAKEUP
 #ifndef USE_WINSOCK
   if(use_wakeup && multi->wakeup_pair[0] != CURL_SOCKET_BAD) {
-    if(Curl_pollfds_add_sock(&cpfds, multi->wakeup_pair[0], POLLIN)) {
-      result = CURLM_OUT_OF_MEMORY;
-      goto out;
-    }
+    ufds[nfds].fd = multi->wakeup_pair[0];
+    ufds[nfds].events = POLLIN;
+    ++nfds;
   }
 #endif
 #endif
 
-  /* We check the internal timeout *AFTER* we collected all sockets to
-   * poll. Collecting the sockets may install new timers by protocols
-   * and connection filters.
-   * Use the shorter one of the internal and the caller requested timeout. */
-  (void)multi_timeout(multi, &expire_time, &timeout_internal);
-  if((timeout_internal >= 0) && (timeout_internal < (long)timeout_ms))
-    timeout_ms = (int)timeout_internal;
-
 #if defined(ENABLE_WAKEUP) && defined(USE_WINSOCK)
-  if(cpfds.n || use_wakeup) {
+  if(nfds || use_wakeup) {
 #else
-  if(cpfds.n) {
+  if(nfds) {
 #endif
     int pollrc;
 #ifdef USE_WINSOCK
-    if(cpfds.n)         /* just pre-check with Winsock */
-      pollrc = Curl_poll(cpfds.pfds, cpfds.n, 0);
+    if(nfds)
+      pollrc = Curl_poll(ufds, nfds, 0); /* just pre-check with WinSock */
     else
       pollrc = 0;
 #else
-    pollrc = Curl_poll(cpfds.pfds, cpfds.n, timeout_ms); /* wait... */
+    pollrc = Curl_poll(ufds, nfds, timeout_ms); /* wait... */
 #endif
-    if(pollrc < 0) {
-      result = CURLM_UNRECOVERABLE_POLL;
-      goto out;
-    }
+    if(pollrc < 0)
+      return CURLM_UNRECOVERABLE_POLL;
 
     if(pollrc > 0) {
       retcode = pollrc;
 #ifdef USE_WINSOCK
     }
     else { /* now wait... if not ready during the pre-check (pollrc == 0) */
-      WSAWaitForMultipleEvents(1, &multi->wsa_event, FALSE, (DWORD)timeout_ms,
-                               FALSE);
+      WSAWaitForMultipleEvents(1, &multi->wsa_event, FALSE, timeout_ms, FALSE);
     }
-    /* With Winsock, we have to run the following section unconditionally
+    /* With WinSock, we have to run the following section unconditionally
        to call WSAEventSelect(fd, event, 0) on all the sockets */
     {
 #endif
@@ -1374,7 +1317,7 @@ static CURLMcode multi_wait(struct Curl_multi *multi,
          struct, the bit values of the actual underlying poll() implementation
          may not be the same as the ones in the public libcurl API! */
       for(i = 0; i < extra_nfds; i++) {
-        unsigned r = (unsigned)cpfds.pfds[curl_nfds + i].revents;
+        unsigned r = ufds[curlfds + i].revents;
         unsigned short mask = 0;
 #ifdef USE_WINSOCK
         curl_socket_t s = extra_fds[i].fd;
@@ -1391,7 +1334,7 @@ static CURLMcode multi_wait(struct Curl_multi *multi,
         }
         WSAEventSelect(s, multi->wsa_event, 0);
         if(!pollrc) {
-          extra_fds[i].revents = (short)mask;
+          extra_fds[i].revents = mask;
           continue;
         }
 #endif
@@ -1401,25 +1344,25 @@ static CURLMcode multi_wait(struct Curl_multi *multi,
           mask |= CURL_WAIT_POLLOUT;
         if(r & POLLPRI)
           mask |= CURL_WAIT_POLLPRI;
-        extra_fds[i].revents = (short)mask;
+        extra_fds[i].revents = mask;
       }
 
 #ifdef USE_WINSOCK
       /* Count up all our own sockets that had activity,
          and remove them from the event. */
-      if(curl_nfds) {
-        for(e = Curl_llist_head(&multi->process); e && !result;
-            e = Curl_node_next(e)) {
-          struct Curl_easy *data = Curl_node_elem(e);
+      if(curlfds) {
+
+        for(data = multi->easyp; data; data = data->next) {
+          multi_getsock(data, &ps);
 
-          for(i = 0; i < data->last_poll.num; i++) {
+          for(i = 0; i < ps.num; i++) {
             wsa_events.lNetworkEvents = 0;
-            if(WSAEnumNetworkEvents(data->last_poll.sockets[i], NULL,
+            if(WSAEnumNetworkEvents(ps.sockets[i], NULL,
                                     &wsa_events) == 0) {
               if(ret && !pollrc && wsa_events.lNetworkEvents)
                 retcode++;
             }
-            WSAEventSelect(data->last_poll.sockets[i], multi->wsa_event, 0);
+            WSAEventSelect(ps.sockets[i], multi->wsa_event, 0);
           }
         }
       }
@@ -1428,7 +1371,7 @@ static CURLMcode multi_wait(struct Curl_multi *multi,
 #else
 #ifdef ENABLE_WAKEUP
       if(use_wakeup && multi->wakeup_pair[0] != CURL_SOCKET_BAD) {
-        if(cpfds.pfds[curl_nfds + extra_nfds].revents & POLLIN) {
+        if(ufds[curlfds + extra_nfds].revents & POLLIN) {
           char buf[64];
           ssize_t nread;
           while(1) {
@@ -1452,16 +1395,18 @@ static CURLMcode multi_wait(struct Curl_multi *multi,
     }
   }
 
+  if(ufds_malloc)
+    free(ufds);
   if(ret)
     *ret = retcode;
 #if defined(ENABLE_WAKEUP) && defined(USE_WINSOCK)
-  if(extrawait && !cpfds.n && !use_wakeup) {
+  if(extrawait && !nfds && !use_wakeup) {
 #else
-  if(extrawait && !cpfds.n) {
+  if(extrawait && !nfds) {
 #endif
     long sleep_ms = 0;
 
-    /* Avoid busy-looping when there is nothing particular to wait for */
+    /* Avoid busy-looping when there's nothing particular to wait for */
     if(!curl_multi_timeout(multi, &sleep_ms) && sleep_ms) {
       if(sleep_ms > timeout_ms)
         sleep_ms = timeout_ms;
@@ -1473,9 +1418,7 @@ static CURLMcode multi_wait(struct Curl_multi *multi,
     }
   }
 
-out:
-  Curl_pollfds_cleanup(&cpfds);
-  return result;
+  return CURLM_OK;
 }
 
 CURLMcode curl_multi_wait(struct Curl_multi *multi,
@@ -1504,15 +1447,6 @@ CURLMcode curl_multi_wakeup(struct Curl_multi *multi)
      it has to be careful only to access parts of the
      Curl_multi struct that are constant */
 
-#if defined(ENABLE_WAKEUP) && !defined(USE_WINSOCK)
-#ifdef USE_EVENTFD
-  const void *buf;
-  const uint64_t val = 1;
-#else
-  char buf[1];
-#endif
-#endif
-
   /* GOOD_MULTI_HANDLE can be safely called */
   if(!GOOD_MULTI_HANDLE(multi))
     return CURLM_BAD_HANDLE;
@@ -1526,11 +1460,8 @@ CURLMcode curl_multi_wakeup(struct Curl_multi *multi)
      making it safe to access from another thread after the init part
      and before cleanup */
   if(multi->wakeup_pair[1] != CURL_SOCKET_BAD) {
-#ifdef USE_EVENTFD
-    buf = &val;
-#else
+    char buf[1];
     buf[0] = 1;
-#endif
     while(1) {
       /* swrite() is not thread-safe in general, because concurrent calls
          can have their messages interleaved, but in this case the content
@@ -1539,7 +1470,7 @@ CURLMcode curl_multi_wakeup(struct Curl_multi *multi)
          The write socket is set to non-blocking, this way this function
          cannot block, making it safe to call even from the same thread
          that will call curl_multi_wait(). If swrite() returns that it
-         would block, it is considered successful because it means that
+         would block, it's considered successful because it means that
          previous calls to this function will wake up the poll(). */
       if(wakeup_write(multi->wakeup_pair[1], buf, sizeof(buf)) < 0) {
         int err = SOCKERRNO;
@@ -1603,7 +1534,7 @@ CURLMcode Curl_multi_add_perform(struct Curl_multi *multi,
   if(!rc) {
     struct SingleRequest *k = &data->req;
 
-    /* pass in NULL for 'conn' here since we do not want to init the
+    /* pass in NULL for 'conn' here since we don't want to init the
        connection, only this transfer */
     Curl_init_do(data, NULL);
 
@@ -1635,7 +1566,7 @@ static CURLcode multi_do(struct Curl_easy *data, bool *done)
  * second connection.
  *
  * 'complete' can return 0 for incomplete, 1 for done and -1 for go back to
- * DOING state there is more work to do!
+ * DOING state there's more work to do!
  */
 
 static CURLcode multi_do_more(struct Curl_easy *data, int *complete)
@@ -1660,47 +1591,47 @@ static bool multi_handle_timeout(struct Curl_easy *data,
                                  CURLcode *result,
                                  bool connect_timeout)
 {
-  timediff_t timeout_ms = Curl_timeleft(data, now, connect_timeout);
+  timediff_t timeout_ms;
+  timeout_ms = Curl_timeleft(data, now, connect_timeout);
+
   if(timeout_ms < 0) {
     /* Handle timed out */
-    struct curltime since;
-    if(connect_timeout)
-      since = data->progress.t_startsingle;
-    else
-      since = data->progress.t_startop;
     if(data->mstate == MSTATE_RESOLVING)
-      failf(data, "Resolving timed out after %" FMT_TIMEDIFF_T
-            " milliseconds", Curl_timediff(*now, since));
+      failf(data, "Resolving timed out after %" CURL_FORMAT_TIMEDIFF_T
+            " milliseconds",
+            Curl_timediff(*now, data->progress.t_startsingle));
     else if(data->mstate == MSTATE_CONNECTING)
-      failf(data, "Connection timed out after %" FMT_TIMEDIFF_T
-            " milliseconds", Curl_timediff(*now, since));
+      failf(data, "Connection timed out after %" CURL_FORMAT_TIMEDIFF_T
+            " milliseconds",
+            Curl_timediff(*now, data->progress.t_startsingle));
     else {
       struct SingleRequest *k = &data->req;
       if(k->size != -1) {
-        failf(data, "Operation timed out after %" FMT_TIMEDIFF_T
-              " milliseconds with %" FMT_OFF_T " out of %"
-              FMT_OFF_T " bytes received",
-              Curl_timediff(*now, since), k->bytecount, k->size);
+        failf(data, "Operation timed out after %" CURL_FORMAT_TIMEDIFF_T
+              " milliseconds with %" CURL_FORMAT_CURL_OFF_T " out of %"
+              CURL_FORMAT_CURL_OFF_T " bytes received",
+              Curl_timediff(*now, data->progress.t_startsingle),
+              k->bytecount, k->size);
       }
       else {
-        failf(data, "Operation timed out after %" FMT_TIMEDIFF_T
-              " milliseconds with %" FMT_OFF_T " bytes received",
-              Curl_timediff(*now, since), k->bytecount);
+        failf(data, "Operation timed out after %" CURL_FORMAT_TIMEDIFF_T
+              " milliseconds with %" CURL_FORMAT_CURL_OFF_T
+              " bytes received",
+              Curl_timediff(*now, data->progress.t_startsingle),
+              k->bytecount);
       }
     }
-    *result = CURLE_OPERATION_TIMEDOUT;
-    if(data->conn) {
-      /* Force connection closed if the connection has indeed been used */
-      if(data->mstate > MSTATE_DO) {
-        streamclose(data->conn, "Disconnect due to timeout");
-        *stream_error = TRUE;
-      }
-      (void)multi_done(data, *result, TRUE);
+
+    /* Force connection closed if the connection has indeed been used */
+    if(data->mstate > MSTATE_DO) {
+      streamclose(data->conn, "Disconnected with pending data");
+      *stream_error = TRUE;
     }
-    return TRUE;
+    *result = CURLE_OPERATION_TIMEDOUT;
+    (void)multi_done(data, *result, TRUE);
   }
 
-  return FALSE;
+  return (timeout_ms < 0);
 }
 
 /*
@@ -1763,10 +1694,10 @@ static CURLcode protocol_connect(struct Curl_easy *data,
      && conn->bits.protoconnstart) {
     /* We already are connected, get back. This may happen when the connect
        worked fine in the first call, like when we connect to a local server
-       or proxy. Note that we do not know if the protocol is actually done.
+       or proxy. Note that we don't know if the protocol is actually done.
 
-       Unless this protocol does not have any protocol-connect callback, as
-       then we know we are done. */
+       Unless this protocol doesn't have any protocol-connect callback, as
+       then we know we're done. */
     if(!conn->handler->connecting)
       *protocol_done = TRUE;
 
@@ -1783,7 +1714,7 @@ static CURLcode protocol_connect(struct Curl_easy *data,
     else
       *protocol_done = TRUE;
 
-    /* it has started, possibly even completed but that knowledge is not stored
+    /* it has started, possibly even completed but that knowledge isn't stored
        in this bit! */
     if(!result)
       conn->bits.protoconnstart = TRUE;
@@ -1792,23 +1723,108 @@ static CURLcode protocol_connect(struct Curl_easy *data,
   return result; /* pass back status */
 }
 
-static void set_in_callback(struct Curl_multi *multi, bool value)
+/*
+ * readrewind() rewinds the read stream. This is typically used for HTTP
+ * POST/PUT with multi-pass authentication when a sending was denied and a
+ * resend is necessary.
+ */
+static CURLcode readrewind(struct Curl_easy *data)
 {
-  multi->in_callback = value;
+  curl_mimepart *mimepart = &data->set.mimepost;
+  DEBUGASSERT(data->conn);
+
+  data->state.rewindbeforesend = FALSE; /* we rewind now */
+
+  /* explicitly switch off sending data on this connection now since we are
+     about to restart a new transfer and thus we want to avoid inadvertently
+     sending more data on the existing connection until the next transfer
+     starts */
+  data->req.keepon &= ~KEEP_SEND;
+
+  /* We have sent away data. If not using CURLOPT_POSTFIELDS or
+     CURLOPT_HTTPPOST, call app to rewind
+  */
+#ifndef CURL_DISABLE_HTTP
+  if(data->conn->handler->protocol & PROTO_FAMILY_HTTP) {
+    if(data->state.mimepost)
+      mimepart = data->state.mimepost;
+  }
+#endif
+  if(data->set.postfields ||
+     (data->state.httpreq == HTTPREQ_GET) ||
+     (data->state.httpreq == HTTPREQ_HEAD))
+    ; /* no need to rewind */
+  else if(data->state.httpreq == HTTPREQ_POST_MIME ||
+          data->state.httpreq == HTTPREQ_POST_FORM) {
+    CURLcode result = Curl_mime_rewind(mimepart);
+    if(result) {
+      failf(data, "Cannot rewind mime/post data");
+      return result;
+    }
+  }
+  else {
+    if(data->set.seek_func) {
+      int err;
+
+      Curl_set_in_callback(data, true);
+      err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET);
+      Curl_set_in_callback(data, false);
+      if(err) {
+        failf(data, "seek callback returned error %d", (int)err);
+        return CURLE_SEND_FAIL_REWIND;
+      }
+    }
+    else if(data->set.ioctl_func) {
+      curlioerr err;
+
+      Curl_set_in_callback(data, true);
+      err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD,
+                                   data->set.ioctl_client);
+      Curl_set_in_callback(data, false);
+      infof(data, "the ioctl callback returned %d", (int)err);
+
+      if(err) {
+        failf(data, "ioctl callback returned error %d", (int)err);
+        return CURLE_SEND_FAIL_REWIND;
+      }
+    }
+    else {
+      /* If no CURLOPT_READFUNCTION is used, we know that we operate on a
+         given FILE * stream and we can actually attempt to rewind that
+         ourselves with fseek() */
+      if(data->state.fread_func == (curl_read_callback)fread) {
+        if(-1 != fseek(data->state.in, 0, SEEK_SET))
+          /* successful rewind */
+          return CURLE_OK;
+      }
+
+      /* no callback set or failure above, makes us fail at once */
+      failf(data, "necessary data rewind wasn't possible");
+      return CURLE_SEND_FAIL_REWIND;
+    }
+  }
+  return CURLE_OK;
 }
 
 /*
- * posttransfer() is called immediately after a transfer ends
+ * Curl_preconnect() is called immediately before a connect starts. When a
+ * redirect is followed, this is then called multiple times during a single
+ * transfer.
  */
-static void multi_posttransfer(struct Curl_easy *data)
+CURLcode Curl_preconnect(struct Curl_easy *data)
 {
-#if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
-  /* restore the signal handler for SIGPIPE before we get back */
-  if(!data->set.no_signal)
-    signal(SIGPIPE, data->state.prev_signal);
-#else
-  (void)data; /* unused parameter */
-#endif
+  if(!data->state.buffer) {
+    data->state.buffer = malloc(data->set.buffer_size + 1);
+    if(!data->state.buffer)
+      return CURLE_OUT_OF_MEMORY;
+  }
+
+  return CURLE_OK;
+}
+
+static void set_in_callback(struct Curl_multi *multi, bool value)
+{
+  multi->in_callback = value;
 }
 
 static CURLMcode multi_runsingle(struct Curl_multi *multi,
@@ -1820,6 +1836,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
   bool async;
   bool protocol_connected = FALSE;
   bool dophase_done = FALSE;
+  bool done = FALSE;
   CURLMcode rc;
   CURLcode result = CURLE_OK;
   timediff_t recv_timeout_ms;
@@ -1833,7 +1850,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
     /* a multi-level callback returned error before, meaning every individual
      transfer now has failed */
     result = CURLE_ABORTED_BY_CALLBACK;
-    multi_posttransfer(data);
+    Curl_posttransfer(data);
     multi_done(data, result, FALSE);
     multistate(data, MSTATE_COMPLETED);
   }
@@ -1859,63 +1876,74 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         return CURLM_INTERNAL_ERROR;
     }
 
-    /* Wait for the connect state as only then is the start time stored, but
-       we must not check already completed handles */
-    if((data->mstate >= MSTATE_CONNECT) && (data->mstate < MSTATE_COMPLETED) &&
-       multi_handle_timeout(data, nowp, &stream_error, &result, FALSE))
-      /* Skip the statemachine and go directly to error handling section. */
-      goto statemachine_end;
+    if(data->conn &&
+       (data->mstate >= MSTATE_CONNECT) &&
+       (data->mstate < MSTATE_COMPLETED)) {
+      /* Check for overall operation timeout here but defer handling the
+       * connection timeout to later, to allow for a connection to be set up
+       * in the window since we last checked timeout. This prevents us
+       * tearing down a completed connection in the case where we were slow
+       * to check the timeout (e.g. process descheduled during this loop).
+       * We set connect_timeout=FALSE to do this. */
+
+      /* we need to wait for the connect state as only then is the start time
+         stored, but we must not check already completed handles */
+      if(multi_handle_timeout(data, nowp, &stream_error, &result, FALSE)) {
+        /* Skip the statemachine and go directly to error handling section. */
+        goto statemachine_end;
+      }
+    }
 
     switch(data->mstate) {
     case MSTATE_INIT:
-      /* Transitional state. init this transfer. A handle never comes
-         back to this state. */
+      /* init this transfer. */
       result = Curl_pretransfer(data);
+
+      if(!result) {
+        /* after init, go CONNECT */
+        multistate(data, MSTATE_CONNECT);
+        *nowp = Curl_pgrsTime(data, TIMER_STARTOP);
+        rc = CURLM_CALL_MULTI_PERFORM;
+      }
+      break;
+
+    case MSTATE_CONNECT:
+      /* Connect. We want to get a connection identifier filled in. */
+      /* init this transfer. */
+      result = Curl_preconnect(data);
       if(result)
         break;
 
-      /* after init, go SETUP */
-      multistate(data, MSTATE_SETUP);
-      (void)Curl_pgrsTime(data, TIMER_STARTOP);
-      FALLTHROUGH();
-
-    case MSTATE_SETUP:
-      /* Transitional state. Setup things for a new transfer. The handle
-         can come back to this state on a redirect. */
       *nowp = Curl_pgrsTime(data, TIMER_STARTSINGLE);
       if(data->set.timeout)
         Curl_expire(data, data->set.timeout, EXPIRE_TIMEOUT);
+
       if(data->set.connecttimeout)
-        /* Since a connection might go to pending and back to CONNECT several
-           times before it actually takes off, we need to set the timeout once
-           in SETUP before we enter CONNECT the first time. */
         Curl_expire(data, data->set.connecttimeout, EXPIRE_CONNECTTIMEOUT);
 
-      multistate(data, MSTATE_CONNECT);
-      FALLTHROUGH();
-
-    case MSTATE_CONNECT:
-      /* Connect. We want to get a connection identifier filled in. This state
-         can be entered from SETUP and from PENDING. */
       result = Curl_connect(data, &async, &connected);
       if(CURLE_NO_CONNECTION_AVAILABLE == result) {
         /* There was no connection available. We will go to the pending
            state and wait for an available connection. */
         multistate(data, MSTATE_PENDING);
-        /* unlink from process list */
-        Curl_node_remove(&data->multi_queue);
-        /* add handle to pending list */
-        Curl_llist_append(&multi->pending, data, &data->multi_queue);
+
+        /* add this handle to the list of connect-pending handles */
+        Curl_llist_insert_next(&multi->pending, multi->pending.tail, data,
+                               &data->connect_queue);
+        /* unlink from the main list */
+        unlink_easy(multi, data);
         result = CURLE_OK;
         break;
       }
-      else
+      else if(data->state.previouslypending) {
+        /* this transfer comes from the pending queue so try move another */
+        infof(data, "Transfer was pending, now try another");
         process_pending_handles(data->multi);
+      }
 
       if(!result) {
-        *nowp = Curl_pgrsTime(data, TIMER_POSTQUEUE);
         if(async)
-          /* We are now waiting for an asynchronous name lookup */
+          /* We're now waiting for an asynchronous name lookup */
           multistate(data, MSTATE_RESOLVING);
         else {
           /* after the connect has been sent off, go WAITCONNECT unless the
@@ -1923,14 +1951,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
              WAITDO or DO! */
           rc = CURLM_CALL_MULTI_PERFORM;
 
-          if(connected) {
-            if(!data->conn->bits.reuse &&
-               Curl_conn_is_multiplex(data->conn, FIRSTSOCKET)) {
-              /* new connection, can multiplex, wake pending handles */
-              process_pending_handles(data->multi);
-            }
+          if(connected)
             multistate(data, MSTATE_PROTOCONNECT);
-          }
           else {
             multistate(data, MSTATE_CONNECTING);
           }
@@ -1957,12 +1979,12 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         hostname = conn->host.name;
 
       /* check if we have the name resolved by now */
-      dns = Curl_fetch_addr(data, hostname, conn->primary.remote_port);
+      dns = Curl_fetch_addr(data, hostname, (int)conn->port);
 
       if(dns) {
 #ifdef CURLRES_ASYNCH
-        data->state.async.dns = dns;
-        data->state.async.done = TRUE;
+        conn->resolve_async.dns = dns;
+        conn->resolve_async.done = TRUE;
 #endif
         result = CURLE_OK;
         infof(data, "Hostname '%s' was found in DNS cache", hostname);
@@ -1974,7 +1996,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       /* Update sockets here, because the socket(s) may have been
          closed and the application thus needs to be told, even if it
          is likely that the same socket(s) will again be used further
-         down. If the name has not yet been resolved, it is likely
+         down.  If the name has not yet been resolved, it is likely
          that new sockets have been opened in an attempt to contact
          another resolver. */
       rc = singlesocket(multi, data);
@@ -2014,12 +2036,22 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       /* this is HTTP-specific, but sending CONNECT to a proxy is HTTP... */
       DEBUGASSERT(data->conn);
       result = Curl_http_connect(data, &protocol_connected);
-      if(!result) {
+#ifndef CURL_DISABLE_PROXY
+      if(data->conn->bits.proxy_connect_closed) {
         rc = CURLM_CALL_MULTI_PERFORM;
-        /* initiate protocol connect phase */
-        multistate(data, MSTATE_PROTOCONNECT);
+        /* connect back to proxy again */
+        result = CURLE_OK;
+        multi_done(data, CURLE_OK, FALSE);
+        multistate(data, MSTATE_CONNECT);
       }
       else
+#endif
+        if(!result) {
+          rc = CURLM_CALL_MULTI_PERFORM;
+          /* initiate protocol connect phase */
+          multistate(data, MSTATE_PROTOCONNECT);
+        }
+      else
         stream_error = TRUE;
       break;
 #endif
@@ -2029,17 +2061,12 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       DEBUGASSERT(data->conn);
       result = Curl_conn_connect(data, FIRSTSOCKET, FALSE, &connected);
       if(connected && !result) {
-        if(!data->conn->bits.reuse &&
-           Curl_conn_is_multiplex(data->conn, FIRSTSOCKET)) {
-          /* new connection, can multiplex, wake pending handles */
-          process_pending_handles(data->multi);
-        }
         rc = CURLM_CALL_MULTI_PERFORM;
         multistate(data, MSTATE_PROTOCONNECT);
       }
       else if(result) {
         /* failure detected */
-        multi_posttransfer(data);
+        Curl_posttransfer(data);
         multi_done(data, result, TRUE);
         stream_error = TRUE;
         break;
@@ -2047,6 +2074,9 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       break;
 
     case MSTATE_PROTOCONNECT:
+      if(data->state.rewindbeforesend)
+        result = readrewind(data);
+
       if(!result && data->conn->bits.reuse) {
         /* ftp seems to hang when protoconnect on reused connection
          * since we handle PROTOCONNECT in general inside the filers, it
@@ -2069,7 +2099,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       }
       else {
         /* failure detected */
-        multi_posttransfer(data);
+        Curl_posttransfer(data);
         multi_done(data, result, TRUE);
         stream_error = TRUE;
       }
@@ -2085,7 +2115,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       }
       else if(result) {
         /* failure detected */
-        multi_posttransfer(data);
+        Curl_posttransfer(data);
         multi_done(data, result, TRUE);
         stream_error = TRUE;
       }
@@ -2098,17 +2128,16 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         /* call the prerequest callback function */
         Curl_set_in_callback(data, true);
         prereq_rc = data->set.fprereq(data->set.prereq_userp,
-                                      data->info.primary.remote_ip,
-                                      data->info.primary.local_ip,
-                                      data->info.primary.remote_port,
-                                      data->info.primary.local_port);
+                                      data->info.conn_primary_ip,
+                                      data->info.conn_local_ip,
+                                      data->info.conn_primary_port,
+                                      data->info.conn_local_port);
         Curl_set_in_callback(data, false);
         if(prereq_rc != CURL_PREREQFUNC_OK) {
           failf(data, "operation aborted by pre-request callback");
-          /* failure in pre-request callback - do not do any other
-             processing */
+          /* failure in pre-request callback - don't do any other processing */
           result = CURLE_ABORTED_BY_CALLBACK;
-          multi_posttransfer(data);
+          Curl_posttransfer(data);
           multi_done(data, result, FALSE);
           stream_error = TRUE;
           break;
@@ -2138,7 +2167,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
                 /* skip some states if it is important */
                 multi_done(data, CURLE_OK, FALSE);
 
-                /* if there is no connection left, skip the DONE state */
+                /* if there's no connection left, skip the DONE state */
                 multistate(data, data->conn ?
                            MSTATE_DONE : MSTATE_COMPLETED);
                 rc = CURLM_CALL_MULTI_PERFORM;
@@ -2154,13 +2183,13 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
 
           /* after DO, go DO_DONE... or DO_MORE */
           else if(data->conn->bits.do_more) {
-            /* we are supposed to do more, but we need to sit down, relax
+            /* we're supposed to do more, but we need to sit down, relax
                and wait a little while first */
             multistate(data, MSTATE_DOING_MORE);
             rc = CURLM_CALL_MULTI_PERFORM;
           }
           else {
-            /* we are done with the DO, now DID */
+            /* we're done with the DO, now DID */
             multistate(data, MSTATE_DID);
             rc = CURLM_CALL_MULTI_PERFORM;
           }
@@ -2169,7 +2198,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
                 data->conn->bits.reuse) {
           /*
            * In this situation, a connection that we were trying to use
-           * may have unexpectedly died. If possible, send the connection
+           * may have unexpectedly died.  If possible, send the connection
            * back to the CONNECT phase so we can try again.
            */
           char *newurl = NULL;
@@ -2183,7 +2212,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
             stream_error = TRUE;
           }
 
-          multi_posttransfer(data);
+          Curl_posttransfer(data);
           drc = multi_done(data, result, FALSE);
 
           /* When set to retry the connection, we must go back to the CONNECT
@@ -2193,7 +2222,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
               follow = FOLLOW_RETRY;
               drc = Curl_follow(data, newurl, follow);
               if(!drc) {
-                multistate(data, MSTATE_SETUP);
+                multistate(data, MSTATE_CONNECT);
                 rc = CURLM_CALL_MULTI_PERFORM;
                 result = CURLE_OK;
               }
@@ -2203,19 +2232,19 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
               }
             }
             else {
-              /* done did not return OK or SEND_ERROR */
+              /* done didn't return OK or SEND_ERROR */
               result = drc;
             }
           }
           else {
-            /* Have error handler disconnect conn if we cannot retry */
+            /* Have error handler disconnect conn if we can't retry */
             stream_error = TRUE;
           }
           free(newurl);
         }
         else {
           /* failure detected */
-          multi_posttransfer(data);
+          Curl_posttransfer(data);
           if(data->conn)
             multi_done(data, result, FALSE);
           stream_error = TRUE;
@@ -2237,7 +2266,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       }
       else {
         /* failure detected */
-        multi_posttransfer(data);
+        Curl_posttransfer(data);
         multi_done(data, result, FALSE);
         stream_error = TRUE;
       }
@@ -2263,7 +2292,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       }
       else {
         /* failure detected */
-        multi_posttransfer(data);
+        Curl_posttransfer(data);
         multi_done(data, result, FALSE);
         stream_error = TRUE;
       }
@@ -2275,7 +2304,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         /* Check if we can move pending requests to send pipe */
         process_pending_handles(multi); /*  multiplexed */
 
-      /* Only perform the transfer if there is a good socket to work with.
+      /* Only perform the transfer if there's a good socket to work with.
          Having both BAD is a signal to skip immediately to DONE */
       if((data->conn->sockfd != CURL_SOCKET_BAD) ||
          (data->conn->writesockfd != CURL_SOCKET_BAD))
@@ -2305,29 +2334,31 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
            result != CURLE_HTTP2_STREAM)
           streamclose(data->conn, "Transfer returned error");
 
-        multi_posttransfer(data);
+        Curl_posttransfer(data);
         multi_done(data, result, TRUE);
       }
       else {
         send_timeout_ms = 0;
         if(data->set.max_send_speed)
           send_timeout_ms =
-            Curl_pgrsLimitWaitTime(&data->progress.ul,
+            Curl_pgrsLimitWaitTime(data->progress.uploaded,
+                                   data->progress.ul_limit_size,
                                    data->set.max_send_speed,
+                                   data->progress.ul_limit_start,
                                    *nowp);
 
         recv_timeout_ms = 0;
         if(data->set.max_recv_speed)
           recv_timeout_ms =
-            Curl_pgrsLimitWaitTime(&data->progress.dl,
+            Curl_pgrsLimitWaitTime(data->progress.downloaded,
+                                   data->progress.dl_limit_size,
                                    data->set.max_recv_speed,
+                                   data->progress.dl_limit_start,
                                    *nowp);
 
         if(!send_timeout_ms && !recv_timeout_ms) {
           multistate(data, MSTATE_PERFORMING);
           Curl_ratelimit(data, *nowp);
-          /* start performing again right away */
-          rc = CURLM_CALL_MULTI_PERFORM;
         }
         else if(send_timeout_ms >= recv_timeout_ms)
           Curl_expire(data, send_timeout_ms, EXPIRE_TOOFAST);
@@ -2340,18 +2371,24 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
     {
       char *newurl = NULL;
       bool retry = FALSE;
+      bool comeback = FALSE;
+      DEBUGASSERT(data->state.buffer);
       /* check if over send speed */
       send_timeout_ms = 0;
       if(data->set.max_send_speed)
-        send_timeout_ms = Curl_pgrsLimitWaitTime(&data->progress.ul,
+        send_timeout_ms = Curl_pgrsLimitWaitTime(data->progress.uploaded,
+                                                 data->progress.ul_limit_size,
                                                  data->set.max_send_speed,
+                                                 data->progress.ul_limit_start,
                                                  *nowp);
 
       /* check if over recv speed */
       recv_timeout_ms = 0;
       if(data->set.max_recv_speed)
-        recv_timeout_ms = Curl_pgrsLimitWaitTime(&data->progress.dl,
+        recv_timeout_ms = Curl_pgrsLimitWaitTime(data->progress.downloaded,
+                                                 data->progress.dl_limit_size,
                                                  data->set.max_recv_speed,
+                                                 data->progress.dl_limit_start,
                                                  *nowp);
 
       if(send_timeout_ms || recv_timeout_ms) {
@@ -2365,9 +2402,9 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       }
 
       /* read/write data if it is ready to do so */
-      result = Curl_sendrecv(data, nowp);
+      result = Curl_readwrite(data->conn, data, &done, &comeback);
 
-      if(data->req.done || (result == CURLE_RECV_ERROR)) {
+      if(done || (result == CURLE_RECV_ERROR)) {
         /* If CURLE_RECV_ERROR happens early enough, we assume it was a race
          * condition and the server closed the reused connection exactly when
          * we wanted to use it, so figure out if that is indeed the case.
@@ -2382,7 +2419,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
           /* if we are to retry, set the result to OK and consider the
              request as done */
           result = CURLE_OK;
-          data->req.done = TRUE;
+          done = TRUE;
         }
       }
       else if((CURLE_HTTP2_STREAM == result) &&
@@ -2402,7 +2439,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
              as done */
           retry = TRUE;
           result = CURLE_OK;
-          data->req.done = TRUE;
+          done = TRUE;
         }
         else
           result = ret;
@@ -2411,8 +2448,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       if(result) {
         /*
          * The transfer phase returned error, we mark the connection to get
-         * closed to prevent being reused. This is because we cannot possibly
-         * know if the connection is in a good shape or not now. Unless it is
+         * closed to prevent being reused. This is because we can't possibly
+         * know if the connection is in a good shape or not now.  Unless it is
          * a protocol which uses two "channels" like FTP, as then the error
          * happened in the data connection.
          */
@@ -2421,13 +2458,13 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
            result != CURLE_HTTP2_STREAM)
           streamclose(data->conn, "Transfer returned error");
 
-        multi_posttransfer(data);
+        Curl_posttransfer(data);
         multi_done(data, result, TRUE);
       }
-      else if(data->req.done && !Curl_cwriter_is_paused(data)) {
+      else if(done) {
 
         /* call this even if the readwrite function returned error */
-        multi_posttransfer(data);
+        Curl_posttransfer(data);
 
         /* When we follow redirects or is set to retry the connection, we must
            to go back to the CONNECT state */
@@ -2447,20 +2484,22 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
           /* multi_done() might return CURLE_GOT_NOTHING */
           result = Curl_follow(data, newurl, follow);
           if(!result) {
-            multistate(data, MSTATE_SETUP);
+            multistate(data, MSTATE_CONNECT);
             rc = CURLM_CALL_MULTI_PERFORM;
           }
+          free(newurl);
         }
         else {
           /* after the transfer is done, go DONE */
 
-          /* but first check to see if we got a location info even though we
-             are not following redirects */
+          /* but first check to see if we got a location info even though we're
+             not following redirects */
           if(data->req.location) {
             free(newurl);
             newurl = data->req.location;
             data->req.location = NULL;
             result = Curl_follow(data, newurl, FOLLOW_FAKE);
+            free(newurl);
             if(result) {
               stream_error = TRUE;
               result = multi_done(data, result, TRUE);
@@ -2473,13 +2512,12 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
           }
         }
       }
-      else if(data->state.select_bits && !Curl_xfer_is_blocked(data)) {
+      else if(comeback) {
         /* This avoids CURLM_CALL_MULTI_PERFORM so that a very fast transfer
-           will not get stuck on this transfer at the expense of other
-           concurrent transfers */
+           won't get stuck on this transfer at the expense of other concurrent
+           transfers */
         Curl_expire(data, 0, EXPIRE_RUN_NOW);
       }
-      free(newurl);
       break;
     }
 
@@ -2490,6 +2528,10 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       if(data->conn) {
         CURLcode res;
 
+        if(data->conn->bits.multiplex)
+          /* Check if we can move pending requests to connection */
+          process_pending_handles(multi); /* multiplexing */
+
         /* post-transfer command */
         res = multi_done(data, result, FALSE);
 
@@ -2508,8 +2550,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         }
       }
 #endif
-      /* after we have DONE what we are supposed to do, go COMPLETED, and
-         it does not matter what the multi_done() returned! */
+      /* after we have DONE what we're supposed to do, go COMPLETED, and
+         it doesn't matter what the multi_done() returned! */
       multistate(data, MSTATE_COMPLETED);
       break;
 
@@ -2526,7 +2568,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       return CURLM_INTERNAL_ERROR;
     }
 
-    if(data->mstate >= MSTATE_CONNECT &&
+    if(data->conn &&
+       data->mstate >= MSTATE_CONNECT &&
        data->mstate < MSTATE_DO &&
        rc != CURLM_CALL_MULTI_PERFORM &&
        !multi_ischanged(multi, false)) {
@@ -2536,7 +2579,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
        * (i.e. CURLM_CALL_MULTI_PERFORM == TRUE) then we should do that before
        * declaring the connection timed out as we may almost have a completed
        * connection. */
-      multi_handle_timeout(data, nowp, &stream_error, &result, FALSE);
+      multi_handle_timeout(data, nowp, &stream_error, &result, TRUE);
     }
 
 statemachine_end:
@@ -2544,7 +2587,7 @@ statemachine_end:
     if(data->mstate < MSTATE_COMPLETED) {
       if(result) {
         /*
-         * If an error was returned, and we are not in completed state now,
+         * If an error was returned, and we aren't in completed state now,
          * then we go to completed and consider this transfer aborted.
          */
 
@@ -2556,27 +2599,31 @@ statemachine_end:
 
         if(data->conn) {
           if(stream_error) {
-            /* Do not attempt to send data over a connection that timed out */
+            /* Don't attempt to send data over a connection that timed out */
             bool dead_connection = result == CURLE_OPERATION_TIMEDOUT;
             struct connectdata *conn = data->conn;
 
             /* This is where we make sure that the conn pointer is reset.
-               We do not have to do this in every case block above where a
+               We don't have to do this in every case block above where a
                failure is detected */
             Curl_detach_connection(data);
-            Curl_cpool_disconnect(data, conn, dead_connection);
+
+            /* remove connection from cache */
+            Curl_conncache_remove_conn(data, conn, TRUE);
+
+            /* disconnect properly */
+            Curl_disconnect(data, conn, dead_connection);
           }
         }
         else if(data->mstate == MSTATE_CONNECT) {
           /* Curl_connect() failed */
-          multi_posttransfer(data);
-          Curl_pgrsUpdate_nometer(data);
+          (void)Curl_posttransfer(data);
         }
 
         multistate(data, MSTATE_COMPLETED);
         rc = CURLM_CALL_MULTI_PERFORM;
       }
-      /* if there is still a connection to use, call the progress function */
+      /* if there's still a connection to use, call the progress function */
       else if(data->conn && Curl_pgrsUpdate(data)) {
         /* aborted due to progress callback return code must close the
            connection */
@@ -2608,10 +2655,11 @@ statemachine_end:
       }
       multistate(data, MSTATE_MSGSENT);
 
-      /* unlink from the process list */
-      Curl_node_remove(&data->multi_queue);
-      /* add this handle msgsent list */
-      Curl_llist_append(&multi->msgsent, data, &data->multi_queue);
+      /* add this handle to the list of msgsent handles */
+      Curl_llist_insert_next(&multi->msgsent, multi->msgsent.tail, data,
+                             &data->connect_queue);
+      /* unlink from the main list */
+      unlink_easy(multi, data);
       return CURLM_OK;
     }
   } while((rc == CURLM_CALL_MULTI_PERFORM) || multi_ischanged(multi, FALSE));
@@ -2623,12 +2671,10 @@ statemachine_end:
 
 CURLMcode curl_multi_perform(struct Curl_multi *multi, int *running_handles)
 {
+  struct Curl_easy *data;
   CURLMcode returncode = CURLM_OK;
-  struct Curl_tree *t = NULL;
+  struct Curl_tree *t;
   struct curltime now = Curl_now();
-  struct Curl_llist_node *e;
-  struct Curl_llist_node *n = NULL;
-  SIGPIPE_VARIABLE(pipe_st);
 
   if(!GOOD_MULTI_HANDLE(multi))
     return CURLM_BAD_HANDLE;
@@ -2636,31 +2682,31 @@ CURLMcode curl_multi_perform(struct Curl_multi *multi, int *running_handles)
   if(multi->in_callback)
     return CURLM_RECURSIVE_API_CALL;
 
-  sigpipe_init(&pipe_st);
-  for(e = Curl_llist_head(&multi->process); e; e = n) {
-    struct Curl_easy *data = Curl_node_elem(e);
+  data = multi->easyp;
+  if(data) {
     CURLMcode result;
+    bool nosig = data->set.no_signal;
+    SIGPIPE_VARIABLE(pipe_st);
+    sigpipe_ignore(data, &pipe_st);
     /* Do the loop and only alter the signal ignore state if the next handle
        has a different NO_SIGNAL state than the previous */
-
-    /* the current node might be unlinked in multi_runsingle(), get the next
-       pointer now */
-    n = Curl_node_next(e);
-
-    if(data != multi->cpool.idata) {
-      /* connection pool handle is processed below */
-      sigpipe_apply(data, &pipe_st);
+    do {
+      /* the current node might be unlinked in multi_runsingle(), get the next
+         pointer now */
+      struct Curl_easy *datanext = data->next;
+      if(data->set.no_signal != nosig) {
+        sigpipe_restore(&pipe_st);
+        sigpipe_ignore(data, &pipe_st);
+        nosig = data->set.no_signal;
+      }
       result = multi_runsingle(multi, &now, data);
       if(result)
         returncode = result;
-    }
+      data = datanext; /* operate on next handle */
+    } while(data);
+    sigpipe_restore(&pipe_st);
   }
 
-  sigpipe_apply(multi->cpool.idata, &pipe_st);
-  Curl_cpool_multi_perform(multi);
-
-  sigpipe_restore(&pipe_st);
-
   /*
    * Simply remove all expired timers from the splay since handles are dealt
    * with unconditionally by this function and curl_multi_timeout() requires
@@ -2673,24 +2719,13 @@ CURLMcode curl_multi_perform(struct Curl_multi *multi, int *running_handles)
    */
   do {
     multi->timetree = Curl_splaygetbest(now, multi->timetree, &t);
-    if(t) {
+    if(t)
       /* the removed may have another timeout in queue */
-      struct Curl_easy *data = Curl_splayget(t);
-      if(data->mstate == MSTATE_PENDING) {
-        bool stream_unused;
-        CURLcode result_unused;
-        if(multi_handle_timeout(data, &now, &stream_unused, &result_unused,
-                                FALSE)) {
-          infof(data, "PENDING handle timeout");
-          move_pending_to_connect(multi, data);
-        }
-      }
-      (void)add_next_timeout(now, multi, Curl_splayget(t));
-    }
+      (void)add_next_timeout(now, multi, t->payload);
+
   } while(t);
 
-  if(running_handles)
-    *running_handles = (int)multi->num_alive;
+  *running_handles = multi->num_alive;
 
   if(CURLM_OK >= returncode)
     returncode = Curl_update_timer(multi);
@@ -2698,45 +2733,35 @@ CURLMcode curl_multi_perform(struct Curl_multi *multi, int *running_handles)
   return returncode;
 }
 
-/* unlink_all_msgsent_handles() moves all nodes back from the msgsent list to
-   the process list */
+/* unlink_all_msgsent_handles() detaches all those easy handles from this
+   multi handle */
 static void unlink_all_msgsent_handles(struct Curl_multi *multi)
 {
-  struct Curl_llist_node *e;
-  for(e = Curl_llist_head(&multi->msgsent); e; e = Curl_node_next(e)) {
-    struct Curl_easy *data = Curl_node_elem(e);
-    if(data) {
-      DEBUGASSERT(data->mstate == MSTATE_MSGSENT);
-      Curl_node_remove(&data->multi_queue);
-      /* put it into the process list */
-      Curl_llist_append(&multi->process, data, &data->multi_queue);
-    }
+  struct Curl_llist_element *e = multi->msgsent.head;
+  if(e) {
+    struct Curl_easy *data = e->ptr;
+    DEBUGASSERT(data->mstate == MSTATE_MSGSENT);
+    data->multi = NULL;
   }
 }
 
 CURLMcode curl_multi_cleanup(struct Curl_multi *multi)
 {
+  struct Curl_easy *data;
+  struct Curl_easy *nextdata;
+
   if(GOOD_MULTI_HANDLE(multi)) {
-    struct Curl_llist_node *e;
-    struct Curl_llist_node *n;
     if(multi->in_callback)
       return CURLM_RECURSIVE_API_CALL;
 
     multi->magic = 0; /* not good anymore */
 
-    /* move the pending and msgsent entries back to process
-       so that there is just one list to iterate over */
     unlink_all_msgsent_handles(multi);
     process_pending_handles(multi);
-
     /* First remove all remaining easy handles */
-    for(e = Curl_llist_head(&multi->process); e; e = n) {
-      struct Curl_easy *data = Curl_node_elem(e);
-
-      if(!GOOD_EASY_HANDLE(data))
-        return CURLM_BAD_HANDLE;
-
-      n = Curl_node_next(e);
+    data = multi->easyp;
+    while(data) {
+      nextdata = data->next;
       if(!data->state.done && data->conn)
         /* if DONE was never called for this handle */
         (void)multi_done(data, CURLE_OK, TRUE);
@@ -2747,18 +2772,23 @@ CURLMcode curl_multi_cleanup(struct Curl_multi *multi)
         data->dns.hostcachetype = HCACHE_NONE;
       }
 
+      /* Clear the pointer to the connection cache */
+      data->state.conn_cache = NULL;
       data->multi = NULL; /* clear the association */
 
 #ifdef USE_LIBPSL
       if(data->psl == &multi->psl)
         data->psl = NULL;
 #endif
+
+      data = nextdata;
     }
 
-    Curl_cpool_destroy(&multi->cpool);
+    /* Close all the connections in the connection cache */
+    Curl_conncache_close_all_connections(&multi->conn_cache);
 
     sockhash_destroy(&multi->sockhash);
-    Curl_hash_destroy(&multi->proto_hash);
+    Curl_conncache_destroy(&multi->conn_cache);
     Curl_hash_destroy(&multi->hostcache);
     Curl_psl_destroy(&multi->psl);
 
@@ -2767,13 +2797,14 @@ CURLMcode curl_multi_cleanup(struct Curl_multi *multi)
 #else
 #ifdef ENABLE_WAKEUP
     wakeup_close(multi->wakeup_pair[0]);
-#ifndef USE_EVENTFD
     wakeup_close(multi->wakeup_pair[1]);
 #endif
 #endif
+
+#ifdef USE_SSL
+    Curl_free_multi_ssl_backend_data(multi->ssl_backend_data);
 #endif
 
-    multi_xfer_bufs_free(multi);
     free(multi);
 
     return CURLM_OK;
@@ -2801,15 +2832,15 @@ CURLMsg *curl_multi_info_read(struct Curl_multi *multi, int *msgs_in_queue)
      !multi->in_callback &&
      Curl_llist_count(&multi->msglist)) {
     /* there is one or more messages in the list */
-    struct Curl_llist_node *e;
+    struct Curl_llist_element *e;
 
     /* extract the head of the list to return */
-    e = Curl_llist_head(&multi->msglist);
+    e = multi->msglist.head;
 
-    msg = Curl_node_elem(e);
+    msg = e->ptr;
 
     /* remove the extracted entry */
-    Curl_node_remove(e);
+    Curl_llist_remove(&multi->msglist, e, NULL);
 
     *msgs_in_queue = curlx_uztosi(Curl_llist_count(&multi->msglist));
 
@@ -2827,54 +2858,41 @@ static CURLMcode singlesocket(struct Curl_multi *multi,
                               struct Curl_easy *data)
 {
   struct easy_pollset cur_poll;
-  CURLMcode mresult;
-
-  /* Fill in the 'current' struct with the state as it is now: what sockets to
-     supervise and for what actions */
-  multi_getsock(data, &cur_poll);
-  mresult = Curl_multi_pollset_ev(multi, data, &cur_poll, &data->last_poll);
-
-  if(!mresult) /* Remember for next time */
-    memcpy(&data->last_poll, &cur_poll, sizeof(cur_poll));
-  return mresult;
-}
-
-CURLMcode Curl_multi_pollset_ev(struct Curl_multi *multi,
-                                struct Curl_easy *data,
-                                struct easy_pollset *ps,
-                                struct easy_pollset *last_ps)
-{
   unsigned int i;
   struct Curl_sh_entry *entry;
   curl_socket_t s;
   int rc;
 
+  /* Fill in the 'current' struct with the state as it is now: what sockets to
+     supervise and for what actions */
+  multi_getsock(data, &cur_poll);
+
   /* We have 0 .. N sockets already and we get to know about the 0 .. M
      sockets we should have from now on. Detect the differences, remove no
      longer supervised ones and add new ones */
 
   /* walk over the sockets we got right now */
-  for(i = 0; i < ps->num; i++) {
-    unsigned char cur_action = ps->actions[i];
+  for(i = 0; i < cur_poll.num; i++) {
+    unsigned char cur_action = cur_poll.actions[i];
     unsigned char last_action = 0;
     int comboaction;
 
-    s = ps->sockets[i];
+    s = cur_poll.sockets[i];
 
     /* get it from the hash */
     entry = sh_getentry(&multi->sockhash, s);
     if(entry) {
       /* check if new for this transfer */
       unsigned int j;
-      for(j = 0; j< last_ps->num; j++) {
-        if(s == last_ps->sockets[j]) {
-          last_action = last_ps->actions[j];
+      for(j = 0; j< data->last_poll.num; j++) {
+        if(s == data->last_poll.sockets[j]) {
+          last_action = data->last_poll.actions[j];
           break;
         }
       }
     }
     else {
-      /* this is a socket we did not have before, add it to the hash! */
+      /* this is a socket we didn't have before, add it to the hash! */
       entry = sh_addentry(&multi->sockhash, s);
       if(!entry)
         /* fatal */
@@ -2882,30 +2900,23 @@ CURLMcode Curl_multi_pollset_ev(struct Curl_multi *multi,
     }
     if(last_action && (last_action != cur_action)) {
       /* Socket was used already, but different action now */
-      if(last_action & CURL_POLL_IN) {
-        DEBUGASSERT(entry->readers);
+      if(last_action & CURL_POLL_IN)
         entry->readers--;
-      }
-      if(last_action & CURL_POLL_OUT) {
-        DEBUGASSERT(entry->writers);
+      if(last_action & CURL_POLL_OUT)
         entry->writers--;
-      }
-      if(cur_action & CURL_POLL_IN) {
+      if(cur_action & CURL_POLL_IN)
         entry->readers++;
-      }
       if(cur_action & CURL_POLL_OUT)
         entry->writers++;
     }
-    else if(!last_action &&
-            !Curl_hash_pick(&entry->transfers, (char *)&data, /* hash key */
-                            sizeof(struct Curl_easy *))) {
-      DEBUGASSERT(entry->users < 100000); /* detect weird values */
+    else if(!last_action) {
       /* a new transfer using this socket */
       entry->users++;
       if(cur_action & CURL_POLL_IN)
         entry->readers++;
       if(cur_action & CURL_POLL_OUT)
         entry->writers++;
+
       /* add 'data' to the transfer hash on this socket! */
       if(!Curl_hash_add(&entry->transfers, (char *)&data, /* hash key */
                         sizeof(struct Curl_easy *), data)) {
@@ -2934,19 +2945,18 @@ CURLMcode Curl_multi_pollset_ev(struct Curl_multi *multi,
       }
     }
 
-    /* store the current action state */
-    entry->action = (unsigned int)comboaction;
+    entry->action = comboaction; /* store the current action state */
   }
 
-  /* Check for last_poll.sockets that no longer appear in ps->sockets.
+  /* Check for last_poll.sockets that no longer appear in cur_poll.sockets.
    * Need to remove the easy handle from the multi->sockhash->transfers and
    * remove multi->sockhash entry when this was the last transfer */
-  for(i = 0; i < last_ps->num; i++) {
+  for(i = 0; i< data->last_poll.num; i++) {
     unsigned int j;
     bool stillused = FALSE;
-    s = last_ps->sockets[i];
-    for(j = 0; j < ps->num; j++) {
-      if(s == ps->sockets[j]) {
+    s = data->last_poll.sockets[i];
+    for(j = 0; j < cur_poll.num; j++) {
+      if(s == cur_poll.sockets[j]) {
         /* this is still supervised */
         stillused = TRUE;
         break;
@@ -2959,29 +2969,25 @@ CURLMcode Curl_multi_pollset_ev(struct Curl_multi *multi,
     /* if this is NULL here, the socket has been closed and notified so
        already by Curl_multi_closed() */
     if(entry) {
-      unsigned char oldactions = last_ps->actions[i];
+      unsigned char oldactions = data->last_poll.actions[i];
       /* this socket has been removed. Decrease user count */
-      DEBUGASSERT(entry->users);
       entry->users--;
       if(oldactions & CURL_POLL_OUT)
         entry->writers--;
       if(oldactions & CURL_POLL_IN)
         entry->readers--;
       if(!entry->users) {
-        bool dead = FALSE;
         if(multi->socket_cb) {
           set_in_callback(multi, TRUE);
           rc = multi->socket_cb(data, s, CURL_POLL_REMOVE,
                                 multi->socket_userp, entry->socketp);
           set_in_callback(multi, FALSE);
-          if(rc == -1)
-            dead = TRUE;
+          if(rc == -1) {
+            multi->dead = TRUE;
+            return CURLM_ABORTED_BY_CALLBACK;
+          }
         }
         sh_delentry(entry, &multi->sockhash, s);
-        if(dead) {
-          multi->dead = TRUE;
-          return CURLM_ABORTED_BY_CALLBACK;
-        }
       }
       else {
         /* still users, but remove this handle as a user of this socket */
@@ -2993,6 +2999,8 @@ CURLMcode Curl_multi_pollset_ev(struct Curl_multi *multi,
     }
   } /* for loop over num */
 
+  /* Remember for next time */
+  memcpy(&data->last_poll, &cur_poll, sizeof(data->last_poll));
   return CURLM_OK;
 }
 
@@ -3008,7 +3016,7 @@ CURLcode Curl_updatesocket(struct Curl_easy *data)
  * Curl_multi_closed()
  *
  * Used by the connect code to tell the multi_socket code that one of the
- * sockets we were using is about to be closed. This function will then
+ * sockets we were using is about to be closed.  This function will then
  * remove it from the sockethash for this handle to make the multi_socket API
  * behave properly, especially for the case when libcurl will create another
  * socket again and it gets the same file descriptor number.
@@ -3017,17 +3025,13 @@ CURLcode Curl_updatesocket(struct Curl_easy *data)
 void Curl_multi_closed(struct Curl_easy *data, curl_socket_t s)
 {
   if(data) {
-    /* if there is still an easy handle associated with this connection */
+    /* if there's still an easy handle associated with this connection */
     struct Curl_multi *multi = data->multi;
-    DEBUGF(infof(data, "Curl_multi_closed, fd=%" FMT_SOCKET_T
-                 " multi is %p", s, (void *)multi));
     if(multi) {
       /* this is set if this connection is part of a handle that is added to
          a multi handle, and only then this is necessary */
       struct Curl_sh_entry *entry = sh_getentry(&multi->sockhash, s);
 
-      DEBUGF(infof(data, "Curl_multi_closed, fd=%" FMT_SOCKET_T
-                   " entry is %p", s, (void *)entry));
       if(entry) {
         int rc = 0;
         if(multi->socket_cb) {
@@ -3067,24 +3071,26 @@ static CURLMcode add_next_timeout(struct curltime now,
 {
   struct curltime *tv = &d->state.expiretime;
   struct Curl_llist *list = &d->state.timeoutlist;
-  struct Curl_llist_node *e;
+  struct Curl_llist_element *e;
+  struct time_node *node = NULL;
 
   /* move over the timeout list for this specific handle and remove all
      timeouts that are now passed tense and store the next pending
      timeout in *tv */
-  for(e = Curl_llist_head(list); e;) {
-    struct Curl_llist_node *n = Curl_node_next(e);
-    struct time_node *node = Curl_node_elem(e);
-    timediff_t diff = Curl_timediff_us(node->time, now);
+  for(e = list->head; e;) {
+    struct Curl_llist_element *n = e->next;
+    timediff_t diff;
+    node = (struct time_node *)e->ptr;
+    diff = Curl_timediff_us(node->time, now);
     if(diff <= 0)
       /* remove outdated entry */
-      Curl_node_remove(e);
+      Curl_llist_remove(list, e, NULL);
     else
       /* the list is sorted so get out on the first mismatch */
       break;
     e = n;
   }
-  e = Curl_llist_head(list);
+  e = list->head;
   if(!e) {
     /* clear the expire times within the handles that we remove from the
        splay tree */
@@ -3092,11 +3098,10 @@ static CURLMcode add_next_timeout(struct curltime now,
     tv->tv_usec = 0;
   }
   else {
-    struct time_node *node = Curl_node_elem(e);
     /* copy the first entry to 'tv' */
     memcpy(tv, &node->time, sizeof(*tv));
 
-    /* Insert this node again into the splay. Keep the timer in the list in
+    /* Insert this node again into the splay.  Keep the timer in the list in
        case we need to recompute future timers. */
     multi->timetree = Curl_splayinsert(*tv, multi->timetree,
                                        &d->state.timenode);
@@ -3104,59 +3109,6 @@ static CURLMcode add_next_timeout(struct curltime now,
   return CURLM_OK;
 }
 
-struct multi_run_ctx {
-  struct Curl_multi *multi;
-  struct curltime now;
-  size_t run_xfers;
-  SIGPIPE_MEMBER(pipe_st);
-  bool run_cpool;
-};
-
-static CURLMcode multi_run_expired(struct multi_run_ctx *mrc)
-{
-  struct Curl_multi *multi = mrc->multi;
-  struct Curl_easy *data = NULL;
-  struct Curl_tree *t = NULL;
-  CURLMcode result = CURLM_OK;
-
-  /*
-   * The loop following here will go on as long as there are expire-times left
-   * to process (compared to mrc->now) in the splay and 'data' will be
-   * re-assigned for every expired handle we deal with.
-   */
-  while(1) {
-    /* Check if there is one (more) expired timer to deal with! This function
-       extracts a matching node if there is one */
-    multi->timetree = Curl_splaygetbest(mrc->now, multi->timetree, &t);
-    if(!t)
-      goto out;
-
-    data = Curl_splayget(t); /* assign this for next loop */
-    if(!data)
-      continue;
-
-    (void)add_next_timeout(mrc->now, multi, data);
-    if(data == multi->cpool.idata) {
-      mrc->run_cpool = TRUE;
-      continue;
-    }
-
-    mrc->run_xfers++;
-    sigpipe_apply(data, &mrc->pipe_st);
-    result = multi_runsingle(multi, &mrc->now, data);
-
-    if(CURLM_OK >= result) {
-      /* get the socket(s) and check if the state has been changed since
-         last */
-      result = singlesocket(multi, data);
-      if(result)
-        goto out;
-    }
-  }
-
-out:
-  return result;
-}
 static CURLMcode multi_socket(struct Curl_multi *multi,
                               bool checkall,
                               curl_socket_t s,
@@ -3165,44 +3117,39 @@ static CURLMcode multi_socket(struct Curl_multi *multi,
 {
   CURLMcode result = CURLM_OK;
   struct Curl_easy *data = NULL;
-  struct multi_run_ctx mrc;
-
-  (void)ev_bitmask;
-  memset(&mrc, 0, sizeof(mrc));
-  mrc.multi = multi;
-  mrc.now = Curl_now();
-  sigpipe_init(&mrc.pipe_st);
+  struct Curl_tree *t;
+  struct curltime now = Curl_now();
+  bool first = FALSE;
+  bool nosig = FALSE;
+  SIGPIPE_VARIABLE(pipe_st);
 
   if(checkall) {
-    struct Curl_llist_node *e;
     /* *perform() deals with running_handles on its own */
     result = curl_multi_perform(multi, running_handles);
 
     /* walk through each easy handle and do the socket state change magic
        and callbacks */
     if(result != CURLM_BAD_HANDLE) {
-      for(e = Curl_llist_head(&multi->process); e && !result;
-          e = Curl_node_next(e)) {
-        result = singlesocket(multi, Curl_node_elem(e));
+      data = multi->easyp;
+      while(data && !result) {
+        result = singlesocket(multi, data);
+        data = data->next;
       }
     }
-    mrc.run_cpool = TRUE;
-    goto out;
-  }
 
+    /* or should we fall-through and do the timer-based stuff? */
+    return result;
+  }
   if(s != CURL_SOCKET_TIMEOUT) {
     struct Curl_sh_entry *entry = sh_getentry(&multi->sockhash, s);
 
-    if(!entry) {
-      /* Unmatched socket, we cannot act on it but we ignore this fact. In
+    if(!entry)
+      /* Unmatched socket, we can't act on it but we ignore this fact.  In
          real-world tests it has been proved that libevent can in fact give
          the application actions even though the socket was just previously
          asked to get removed, so thus we better survive stray socket actions
          and just move on. */
-      /* The socket might come from a connection that is being shut down
-       * by the multi's connection pool. */
-      Curl_cpool_multi_socket(multi, s, ev_bitmask);
-    }
+      ;
     else {
       struct Curl_hash_iterator iter;
       struct Curl_hash_element *he;
@@ -3215,43 +3162,75 @@ static CURLMcode multi_socket(struct Curl_multi *multi,
         DEBUGASSERT(data);
         DEBUGASSERT(data->magic == CURLEASY_MAGIC_NUMBER);
 
-        if(data == multi->cpool.idata)
-          mrc.run_cpool = TRUE;
-        else {
-          /* Expire with out current now, so we will get it below when
-           * asking the splaytree for expired transfers. */
-          Curl_expire_ex(data, &mrc.now, 0, EXPIRE_RUN_NOW);
-        }
+        if(data->conn && !(data->conn->handler->flags & PROTOPT_DIRLOCK))
+          /* set socket event bitmask if they're not locked */
+          data->conn->cselect_bits = (unsigned char)ev_bitmask;
+
+        Curl_expire(data, 0, EXPIRE_RUN_NOW);
       }
+
+      /* Now we fall-through and do the timer-based stuff, since we don't want
+         to force the user to have to deal with timeouts as long as at least
+         one connection in fact has traffic. */
+
+      data = NULL; /* set data to NULL again to avoid calling
+                      multi_runsingle() in case there's no need to */
+      now = Curl_now(); /* get a newer time since the multi_runsingle() loop
+                           may have taken some time */
     }
   }
+  else {
+    /* Asked to run due to time-out. Clear the 'lastcall' variable to force
+       Curl_update_timer() to trigger a callback to the app again even if the
+       same timeout is still the one to run after this call. That handles the
+       case when the application asks libcurl to run the timeout
+       prematurely. */
+    memset(&multi->timer_lastcall, 0, sizeof(multi->timer_lastcall));
+  }
 
-  result = multi_run_expired(&mrc);
-  if(result)
-    goto out;
+  /*
+   * The loop following here will go on as long as there are expire-times left
+   * to process in the splay and 'data' will be re-assigned for every expired
+   * handle we deal with.
+   */
+  do {
+    /* the first loop lap 'data' can be NULL */
+    if(data) {
+      if(!first) {
+        first = TRUE;
+        nosig = data->set.no_signal; /* initial state */
+        sigpipe_ignore(data, &pipe_st);
+      }
+      else if(data->set.no_signal != nosig) {
+        sigpipe_restore(&pipe_st);
+        sigpipe_ignore(data, &pipe_st);
+        nosig = data->set.no_signal; /* remember new state */
+      }
+      result = multi_runsingle(multi, &now, data);
 
-  if(mrc.run_xfers) {
-    /* Running transfers takes time. With a new timestamp, we might catch
-     * other expires which are due now. Instead of telling the application
-     * to set a 0 timeout and call us again, we run them here.
-     * Do that only once or it might be unfair to transfers on other
-     * sockets. */
-    mrc.now = Curl_now();
-    result = multi_run_expired(&mrc);
-  }
+      if(CURLM_OK >= result) {
+        /* get the socket(s) and check if the state has been changed since
+           last */
+        result = singlesocket(multi, data);
+        if(result)
+          break;
+      }
+    }
 
-out:
-  if(mrc.run_cpool) {
-    sigpipe_apply(multi->cpool.idata, &mrc.pipe_st);
-    Curl_cpool_multi_perform(multi);
-  }
-  sigpipe_restore(&mrc.pipe_st);
+    /* Check if there's one (more) expired timer to deal with! This function
+       extracts a matching node if there is one */
 
-  if(running_handles)
-    *running_handles = (int)multi->num_alive;
+    multi->timetree = Curl_splaygetbest(now, multi->timetree, &t);
+    if(t) {
+      data = t->payload; /* assign this for next loop */
+      (void)add_next_timeout(now, multi, t->payload);
+    }
 
-  if(CURLM_OK >= result)
-    result = Curl_update_timer(multi);
+  } while(t);
+  if(first)
+    sigpipe_restore(&pipe_st);
+
+  *running_handles = multi->num_alive;
   return result;
 }
 
@@ -3303,9 +3282,6 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi,
     break;
   case CURLMOPT_MAX_TOTAL_CONNECTIONS:
     multi->max_total_connections = va_arg(param, long);
-    /* for now, let this also decide the max number of connections
-     * in shutdown handling */
-    multi->max_shutdown_connections = va_arg(param, long);
     break;
     /* options formerly used for pipelining */
   case CURLMOPT_MAX_PIPELINE_LENGTH:
@@ -3340,28 +3316,39 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi,
 CURLMcode curl_multi_socket(struct Curl_multi *multi, curl_socket_t s,
                             int *running_handles)
 {
+  CURLMcode result;
   if(multi->in_callback)
     return CURLM_RECURSIVE_API_CALL;
-  return multi_socket(multi, FALSE, s, 0, running_handles);
+  result = multi_socket(multi, FALSE, s, 0, running_handles);
+  if(CURLM_OK >= result)
+    result = Curl_update_timer(multi);
+  return result;
 }
 
 CURLMcode curl_multi_socket_action(struct Curl_multi *multi, curl_socket_t s,
                                    int ev_bitmask, int *running_handles)
 {
+  CURLMcode result;
   if(multi->in_callback)
     return CURLM_RECURSIVE_API_CALL;
-  return multi_socket(multi, FALSE, s, ev_bitmask, running_handles);
+  result = multi_socket(multi, FALSE, s, ev_bitmask, running_handles);
+  if(CURLM_OK >= result)
+    result = Curl_update_timer(multi);
+  return result;
 }
 
 CURLMcode curl_multi_socket_all(struct Curl_multi *multi, int *running_handles)
 {
+  CURLMcode result;
   if(multi->in_callback)
     return CURLM_RECURSIVE_API_CALL;
-  return multi_socket(multi, TRUE, CURL_SOCKET_BAD, 0, running_handles);
+  result = multi_socket(multi, TRUE, CURL_SOCKET_BAD, 0, running_handles);
+  if(CURLM_OK >= result)
+    result = Curl_update_timer(multi);
+  return result;
 }
 
 static CURLMcode multi_timeout(struct Curl_multi *multi,
-                               struct curltime *expire_time,
                                long *timeout_ms)
 {
   static const struct curltime tv_zero = {0, 0};
@@ -3377,29 +3364,20 @@ static CURLMcode multi_timeout(struct Curl_multi *multi,
 
     /* splay the lowest to the bottom */
     multi->timetree = Curl_splay(tv_zero, multi->timetree);
-    /* this will not return NULL from a non-emtpy tree, but some compilers
-     * are not convinced of that. Analyzers are hard. */
-    *expire_time = multi->timetree? multi->timetree->key : tv_zero;
-
-    /* 'multi->timetree' will be non-NULL here but the compilers sometimes
-       yell at us if we assume so */
-    if(multi->timetree &&
-       Curl_timediff_us(multi->timetree->key, now) > 0) {
+
+    if(Curl_splaycomparekeys(multi->timetree->key, now) > 0) {
       /* some time left before expiration */
       timediff_t diff = Curl_timediff_ceil(multi->timetree->key, now);
-      /* this should be safe even on 32-bit archs, as we do not use that
+      /* this should be safe even on 32 bit archs, as we don't use that
          overly long timeouts */
       *timeout_ms = (long)diff;
     }
-    else {
+    else
       /* 0 means immediately */
       *timeout_ms = 0;
-    }
   }
-  else {
-    *expire_time = tv_zero;
+  else
     *timeout_ms = -1;
-  }
 
   return CURLM_OK;
 }
@@ -3407,8 +3385,6 @@ static CURLMcode multi_timeout(struct Curl_multi *multi,
 CURLMcode curl_multi_timeout(struct Curl_multi *multi,
                              long *timeout_ms)
 {
-  struct curltime expire_time;
-
   /* First, make some basic checks that the CURLM handle is a good handle */
   if(!GOOD_MULTI_HANDLE(multi))
     return CURLM_BAD_HANDLE;
@@ -3416,79 +3392,56 @@ CURLMcode curl_multi_timeout(struct Curl_multi *multi,
   if(multi->in_callback)
     return CURLM_RECURSIVE_API_CALL;
 
-  return multi_timeout(multi, &expire_time, timeout_ms);
+  return multi_timeout(multi, timeout_ms);
 }
 
-#define DEBUG_UPDATE_TIMER    0
-
 /*
  * Tell the application it should update its timers, if it subscribes to the
  * update timer callback.
  */
 CURLMcode Curl_update_timer(struct Curl_multi *multi)
 {
-  struct curltime expire_ts;
   long timeout_ms;
   int rc;
-  bool set_value = FALSE;
 
   if(!multi->timer_cb || multi->dead)
     return CURLM_OK;
-  if(multi_timeout(multi, &expire_ts, &timeout_ms)) {
+  if(multi_timeout(multi, &timeout_ms)) {
     return CURLM_OK;
   }
-
-  if(timeout_ms < 0 && multi->last_timeout_ms < 0) {
-#if DEBUG_UPDATE_TIMER
-    fprintf(stderr, "Curl_update_timer(), still no timeout, no change\n");
-#endif
-  }
-  else if(timeout_ms < 0) {
-    /* there is no timeout now but there was one previously */
-#if DEBUG_UPDATE_TIMER
-    fprintf(stderr, "Curl_update_timer(), remove timeout, "
-        " last_timeout=%ldms\n", multi->last_timeout_ms);
-#endif
-    timeout_ms = -1; /* normalize */
-    set_value = TRUE;
-  }
-  else if(multi->last_timeout_ms < 0) {
-#if DEBUG_UPDATE_TIMER
-    fprintf(stderr, "Curl_update_timer(), had no timeout, set now\n");
-#endif
-    set_value = TRUE;
-  }
-  else if(Curl_timediff_us(multi->last_expire_ts, expire_ts)) {
-    /* We had a timeout before and have one now, the absolute timestamp
-     * differs. The relative timeout_ms may be the same, but the starting
-     * point differs. Let the application restart its timer. */
-#if DEBUG_UPDATE_TIMER
-    fprintf(stderr, "Curl_update_timer(), expire timestamp changed\n");
-#endif
-    set_value = TRUE;
-  }
-  else {
-    /* We have same expire time as previously. Our relative 'timeout_ms'
-     * may be different now, but the application has the timer running
-     * and we do not to tell it to start this again. */
-#if DEBUG_UPDATE_TIMER
-    fprintf(stderr, "Curl_update_timer(), same expire timestamp, no change\n");
-#endif
+  if(timeout_ms < 0) {
+    static const struct curltime none = {0, 0};
+    if(Curl_splaycomparekeys(none, multi->timer_lastcall)) {
+      multi->timer_lastcall = none;
+      /* there's no timeout now but there was one previously, tell the app to
+         disable it */
+      set_in_callback(multi, TRUE);
+      rc = multi->timer_cb(multi, -1, multi->timer_userp);
+      set_in_callback(multi, FALSE);
+      if(rc == -1) {
+        multi->dead = TRUE;
+        return CURLM_ABORTED_BY_CALLBACK;
+      }
+      return CURLM_OK;
+    }
+    return CURLM_OK;
   }
 
-  if(set_value) {
-#if DEBUG_UPDATE_TIMER
-    fprintf(stderr, "Curl_update_timer(), set timeout %ldms\n", timeout_ms);
-#endif
-    multi->last_expire_ts = expire_ts;
-    multi->last_timeout_ms = timeout_ms;
-    set_in_callback(multi, TRUE);
-    rc = multi->timer_cb(multi, timeout_ms, multi->timer_userp);
-    set_in_callback(multi, FALSE);
-    if(rc == -1) {
-      multi->dead = TRUE;
-      return CURLM_ABORTED_BY_CALLBACK;
-    }
+  /* When multi_timeout() is done, multi->timetree points to the node with the
+   * timeout we got the (relative) time-out time for. We can thus easily check
+   * if this is the same (fixed) time as we got in a previous call and then
+   * avoid calling the callback again. */
+  if(Curl_splaycomparekeys(multi->timetree->key, multi->timer_lastcall) == 0)
+    return CURLM_OK;
+
+  multi->timer_lastcall = multi->timetree->key;
+
+  set_in_callback(multi, TRUE);
+  rc = multi->timer_cb(multi, timeout_ms, multi->timer_userp);
+  set_in_callback(multi, FALSE);
+  if(rc == -1) {
+    multi->dead = TRUE;
+    return CURLM_ABORTED_BY_CALLBACK;
   }
   return CURLM_OK;
 }
@@ -3501,13 +3454,13 @@ CURLMcode Curl_update_timer(struct Curl_multi *multi)
 static void
 multi_deltimeout(struct Curl_easy *data, expire_id eid)
 {
-  struct Curl_llist_node *e;
+  struct Curl_llist_element *e;
   struct Curl_llist *timeoutlist = &data->state.timeoutlist;
   /* find and remove the specific node from the list */
-  for(e = Curl_llist_head(timeoutlist); e; e = Curl_node_next(e)) {
-    struct time_node *n = Curl_node_elem(e);
+  for(e = timeoutlist->head; e; e = e->next) {
+    struct time_node *n = (struct time_node *)e->ptr;
     if(n->eid == eid) {
-      Curl_node_remove(e);
+      Curl_llist_remove(timeoutlist, e, NULL);
       return;
     }
   }
@@ -3525,9 +3478,9 @@ multi_addtimeout(struct Curl_easy *data,
                  struct curltime *stamp,
                  expire_id eid)
 {
-  struct Curl_llist_node *e;
+  struct Curl_llist_element *e;
   struct time_node *node;
-  struct Curl_llist_node *prev = NULL;
+  struct Curl_llist_element *prev = NULL;
   size_t n;
   struct Curl_llist *timeoutlist = &data->state.timeoutlist;
 
@@ -3540,8 +3493,8 @@ multi_addtimeout(struct Curl_easy *data,
   n = Curl_llist_count(timeoutlist);
   if(n) {
     /* find the correct spot in the list */
-    for(e = Curl_llist_head(timeoutlist); e; e = Curl_node_next(e)) {
-      struct time_node *check = Curl_node_elem(e);
+    for(e = timeoutlist->head; e; e = e->next) {
+      struct time_node *check = (struct time_node *)e->ptr;
       timediff_t diff = Curl_timediff(check->time, node->time);
       if(diff > 0)
         break;
@@ -3567,12 +3520,10 @@ multi_addtimeout(struct Curl_easy *data,
  *
  * Expire replaces a former timeout using the same id if already set.
  */
-static void Curl_expire_ex(struct Curl_easy *data,
-                           const struct curltime *nowp,
-                           timediff_t milli, expire_id id)
+void Curl_expire(struct Curl_easy *data, timediff_t milli, expire_id id)
 {
   struct Curl_multi *multi = data->multi;
-  struct curltime *curr_expire = &data->state.expiretime;
+  struct curltime *nowp = &data->state.expiretime;
   struct curltime set;
 
   /* this is only interesting while there is still an associated multi struct
@@ -3582,9 +3533,9 @@ static void Curl_expire_ex(struct Curl_easy *data,
 
   DEBUGASSERT(id < EXPIRE_LAST);
 
-  set = *nowp;
-  set.tv_sec += (time_t)(milli/1000); /* might be a 64 to 32 bits conversion */
-  set.tv_usec += (int)(milli%1000)*1000;
+  set = Curl_now();
+  set.tv_sec += (time_t)(milli/1000); /* might be a 64 to 32 bit conversion */
+  set.tv_usec += (unsigned int)(milli%1000)*1000;
 
   if(set.tv_usec >= 1000000) {
     set.tv_sec++;
@@ -3594,20 +3545,20 @@ static void Curl_expire_ex(struct Curl_easy *data,
   /* Remove any timer with the same id just in case. */
   multi_deltimeout(data, id);
 
-  /* Add it to the timer list. It must stay in the list until it has expired
+  /* Add it to the timer list.  It must stay in the list until it has expired
      in case we need to recompute the minimum timer later. */
   multi_addtimeout(data, &set, id);
 
-  if(curr_expire->tv_sec || curr_expire->tv_usec) {
+  if(nowp->tv_sec || nowp->tv_usec) {
     /* This means that the struct is added as a node in the splay tree.
        Compare if the new time is earlier, and only remove-old/add-new if it
        is. */
-    timediff_t diff = Curl_timediff(set, *curr_expire);
+    timediff_t diff = Curl_timediff(set, *nowp);
     int rc;
 
     if(diff > 0) {
       /* The current splay tree entry is sooner than this new expiry time.
-         We do not need to update our splay tree entry. */
+         We don't need to update our splay tree entry. */
       return;
     }
 
@@ -3621,18 +3572,12 @@ static void Curl_expire_ex(struct Curl_easy *data,
 
   /* Indicate that we are in the splay tree and insert the new timer expiry
      value since it is our local minimum. */
-  *curr_expire = set;
-  Curl_splayset(&data->state.timenode, data);
-  multi->timetree = Curl_splayinsert(*curr_expire, multi->timetree,
+  *nowp = set;
+  data->state.timenode.payload = data;
+  multi->timetree = Curl_splayinsert(*nowp, multi->timetree,
                                      &data->state.timenode);
 }
 
-void Curl_expire(struct Curl_easy *data, timediff_t milli, expire_id id)
-{
-  struct curltime now = Curl_now();
-  Curl_expire_ex(data, &now, milli, id);
-}
-
 /*
  * Curl_expire_done()
  *
@@ -3650,7 +3595,7 @@ void Curl_expire_done(struct Curl_easy *data, expire_id id)
  *
  * Clear ALL timeout values for this handle.
  */
-bool Curl_expire_clear(struct Curl_easy *data)
+void Curl_expire_clear(struct Curl_easy *data)
 {
   struct Curl_multi *multi = data->multi;
   struct curltime *nowp = &data->state.expiretime;
@@ -3658,7 +3603,7 @@ bool Curl_expire_clear(struct Curl_easy *data)
   /* this is only interesting while there is still an associated multi struct
      remaining! */
   if(!multi)
-    return FALSE;
+    return;
 
   if(nowp->tv_sec || nowp->tv_usec) {
     /* Since this is an cleared time, we must remove the previous entry from
@@ -3671,25 +3616,26 @@ bool Curl_expire_clear(struct Curl_easy *data)
     if(rc)
       infof(data, "Internal error clearing splay node = %d", rc);
 
-    /* clear the timeout list too */
-    Curl_llist_destroy(list, NULL);
+    /* flush the timeout list too */
+    while(list->size > 0) {
+      Curl_llist_remove(list, list->tail, NULL);
+    }
 
 #ifdef DEBUGBUILD
     infof(data, "Expire cleared");
 #endif
     nowp->tv_sec = 0;
     nowp->tv_usec = 0;
-    return TRUE;
   }
-  return FALSE;
 }
 
+
+
+
 CURLMcode curl_multi_assign(struct Curl_multi *multi, curl_socket_t s,
                             void *hashp)
 {
   struct Curl_sh_entry *there = NULL;
-  if(!GOOD_MULTI_HANDLE(multi))
-    return CURLM_BAD_HANDLE;
 
   there = sh_getentry(&multi->sockhash, s);
 
@@ -3701,55 +3647,75 @@ CURLMcode curl_multi_assign(struct Curl_multi *multi, curl_socket_t s,
   return CURLM_OK;
 }
 
-static void move_pending_to_connect(struct Curl_multi *multi,
-                                    struct Curl_easy *data)
+size_t Curl_multi_max_host_connections(struct Curl_multi *multi)
 {
-  DEBUGASSERT(data->mstate == MSTATE_PENDING);
-
-  /* Remove this node from the pending list */
-  Curl_node_remove(&data->multi_queue);
-
-  /* put it into the process list */
-  Curl_llist_append(&multi->process, data, &data->multi_queue);
-
-  multistate(data, MSTATE_CONNECT);
+  return multi ? multi->max_host_connections : 0;
+}
 
-  /* Make sure that the handle will be processed soonish. */
-  Curl_expire(data, 0, EXPIRE_RUN_NOW);
+size_t Curl_multi_max_total_connections(struct Curl_multi *multi)
+{
+  return multi ? multi->max_total_connections : 0;
 }
 
-/* process_pending_handles() moves a handle from PENDING back into the process
-   list and change state to CONNECT.
+/*
+ * When information about a connection has appeared, call this!
+ */
 
-   We do not move all transfers because that can be a significant amount.
-   Since this is tried every now and then doing too many too often becomes a
-   performance problem.
+void Curl_multiuse_state(struct Curl_easy *data,
+                         int bundlestate) /* use BUNDLE_* defines */
+{
+  struct connectdata *conn;
+  DEBUGASSERT(data);
+  DEBUGASSERT(data->multi);
+  conn = data->conn;
+  DEBUGASSERT(conn);
+  DEBUGASSERT(conn->bundle);
 
-   When there is a change for connection limits like max host connections etc,
-   this likely only allows one new transfer. When there is a pipewait change,
-   it can potentially allow hundreds of new transfers.
+  conn->bundle->multiuse = bundlestate;
+  process_pending_handles(data->multi);
+}
 
-   We could consider an improvement where we store the queue reason and allow
-   more pipewait rechecks than others.
-*/
+/* process_pending_handles() moves all handles from PENDING
+   back into the main list and change state to CONNECT */
 static void process_pending_handles(struct Curl_multi *multi)
 {
-  struct Curl_llist_node *e = Curl_llist_head(&multi->pending);
+  struct Curl_llist_element *e = multi->pending.head;
   if(e) {
-    struct Curl_easy *data = Curl_node_elem(e);
-    move_pending_to_connect(multi, data);
+    struct Curl_easy *data = e->ptr;
+
+    DEBUGASSERT(data->mstate == MSTATE_PENDING);
+
+    /* put it back into the main list */
+    link_easy(multi, data);
+
+    multistate(data, MSTATE_CONNECT);
+
+    /* Remove this node from the list */
+    Curl_llist_remove(&multi->pending, e, NULL);
+
+    /* Make sure that the handle will be processed soonish. */
+    Curl_expire(data, 0, EXPIRE_RUN_NOW);
+
+    /* mark this as having been in the pending queue */
+    data->state.previouslypending = TRUE;
   }
 }
 
 void Curl_set_in_callback(struct Curl_easy *data, bool value)
 {
-  if(data && data->multi)
-    data->multi->in_callback = value;
+  /* might get called when there is no data pointer! */
+  if(data) {
+    if(data->multi_easy)
+      data->multi_easy->in_callback = value;
+    else if(data->multi)
+      data->multi->in_callback = value;
+  }
 }
 
-bool Curl_is_in_callback(struct Curl_easy *data)
+bool Curl_is_in_callback(struct Curl_easy *easy)
 {
-  return (data && data->multi && data->multi->in_callback);
+  return ((easy->multi && easy->multi->in_callback) ||
+          (easy->multi_easy && easy->multi_easy->in_callback));
 }
 
 unsigned int Curl_multi_max_concurrent_streams(struct Curl_multi *multi)
@@ -3764,160 +3730,14 @@ struct Curl_easy **curl_multi_get_handles(struct Curl_multi *multi)
                                 (multi->num_easy + 1));
   if(a) {
     unsigned int i = 0;
-    struct Curl_llist_node *e;
-    for(e = Curl_llist_head(&multi->process); e; e = Curl_node_next(e)) {
-      struct Curl_easy *data = Curl_node_elem(e);
+    struct Curl_easy *e = multi->easyp;
+    while(e) {
       DEBUGASSERT(i < multi->num_easy);
-      if(!data->state.internal)
-        a[i++] = data;
+      if(!e->state.internal)
+        a[i++] = e;
+      e = e->next;
     }
     a[i] = NULL; /* last entry is a NULL */
   }
   return a;
 }
-
-CURLcode Curl_multi_xfer_buf_borrow(struct Curl_easy *data,
-                                    char **pbuf, size_t *pbuflen)
-{
-  DEBUGASSERT(data);
-  DEBUGASSERT(data->multi);
-  *pbuf = NULL;
-  *pbuflen = 0;
-  if(!data->multi) {
-    failf(data, "transfer has no multi handle");
-    return CURLE_FAILED_INIT;
-  }
-  if(!data->set.buffer_size) {
-    failf(data, "transfer buffer size is 0");
-    return CURLE_FAILED_INIT;
-  }
-  if(data->multi->xfer_buf_borrowed) {
-    failf(data, "attempt to borrow xfer_buf when already borrowed");
-    return CURLE_AGAIN;
-  }
-
-  if(data->multi->xfer_buf &&
-     data->set.buffer_size > data->multi->xfer_buf_len) {
-    /* not large enough, get a new one */
-    free(data->multi->xfer_buf);
-    data->multi->xfer_buf = NULL;
-    data->multi->xfer_buf_len = 0;
-  }
-
-  if(!data->multi->xfer_buf) {
-    data->multi->xfer_buf = malloc((size_t)data->set.buffer_size);
-    if(!data->multi->xfer_buf) {
-      failf(data, "could not allocate xfer_buf of %zu bytes",
-            (size_t)data->set.buffer_size);
-      return CURLE_OUT_OF_MEMORY;
-    }
-    data->multi->xfer_buf_len = data->set.buffer_size;
-  }
-
-  data->multi->xfer_buf_borrowed = TRUE;
-  *pbuf = data->multi->xfer_buf;
-  *pbuflen = data->multi->xfer_buf_len;
-  return CURLE_OK;
-}
-
-void Curl_multi_xfer_buf_release(struct Curl_easy *data, char *buf)
-{
-  (void)buf;
-  DEBUGASSERT(data);
-  DEBUGASSERT(data->multi);
-  DEBUGASSERT(!buf || data->multi->xfer_buf == buf);
-  data->multi->xfer_buf_borrowed = FALSE;
-}
-
-CURLcode Curl_multi_xfer_ulbuf_borrow(struct Curl_easy *data,
-                                      char **pbuf, size_t *pbuflen)
-{
-  DEBUGASSERT(data);
-  DEBUGASSERT(data->multi);
-  *pbuf = NULL;
-  *pbuflen = 0;
-  if(!data->multi) {
-    failf(data, "transfer has no multi handle");
-    return CURLE_FAILED_INIT;
-  }
-  if(!data->set.upload_buffer_size) {
-    failf(data, "transfer upload buffer size is 0");
-    return CURLE_FAILED_INIT;
-  }
-  if(data->multi->xfer_ulbuf_borrowed) {
-    failf(data, "attempt to borrow xfer_ulbuf when already borrowed");
-    return CURLE_AGAIN;
-  }
-
-  if(data->multi->xfer_ulbuf &&
-     data->set.upload_buffer_size > data->multi->xfer_ulbuf_len) {
-    /* not large enough, get a new one */
-    free(data->multi->xfer_ulbuf);
-    data->multi->xfer_ulbuf = NULL;
-    data->multi->xfer_ulbuf_len = 0;
-  }
-
-  if(!data->multi->xfer_ulbuf) {
-    data->multi->xfer_ulbuf = malloc((size_t)data->set.upload_buffer_size);
-    if(!data->multi->xfer_ulbuf) {
-      failf(data, "could not allocate xfer_ulbuf of %zu bytes",
-            (size_t)data->set.upload_buffer_size);
-      return CURLE_OUT_OF_MEMORY;
-    }
-    data->multi->xfer_ulbuf_len = data->set.upload_buffer_size;
-  }
-
-  data->multi->xfer_ulbuf_borrowed = TRUE;
-  *pbuf = data->multi->xfer_ulbuf;
-  *pbuflen = data->multi->xfer_ulbuf_len;
-  return CURLE_OK;
-}
-
-void Curl_multi_xfer_ulbuf_release(struct Curl_easy *data, char *buf)
-{
-  (void)buf;
-  DEBUGASSERT(data);
-  DEBUGASSERT(data->multi);
-  DEBUGASSERT(!buf || data->multi->xfer_ulbuf == buf);
-  data->multi->xfer_ulbuf_borrowed = FALSE;
-}
-
-static void multi_xfer_bufs_free(struct Curl_multi *multi)
-{
-  DEBUGASSERT(multi);
-  Curl_safefree(multi->xfer_buf);
-  multi->xfer_buf_len = 0;
-  multi->xfer_buf_borrowed = FALSE;
-  Curl_safefree(multi->xfer_ulbuf);
-  multi->xfer_ulbuf_len = 0;
-  multi->xfer_ulbuf_borrowed = FALSE;
-}
-
-struct Curl_easy *Curl_multi_get_handle(struct Curl_multi *multi,
-                                        curl_off_t mid)
-{
-
-  if(mid >= 0) {
-    struct Curl_easy *data;
-    struct Curl_llist_node *e;
-
-    for(e = Curl_llist_head(&multi->process); e; e = Curl_node_next(e)) {
-      data = Curl_node_elem(e);
-      if(data->mid == mid)
-        return data;
-    }
-    /* may be in msgsent queue */
-    for(e = Curl_llist_head(&multi->msgsent); e; e = Curl_node_next(e)) {
-      data = Curl_node_elem(e);
-      if(data->mid == mid)
-        return data;
-    }
-    /* may be in pending queue */
-    for(e = Curl_llist_head(&multi->pending); e; e = Curl_node_next(e)) {
-      data = Curl_node_elem(e);
-      if(data->mid == mid)
-        return data;
-    }
-  }
-  return NULL;
-}