Why is a non-blocking TCP connect() occasionally so slow on Linux?

I was trying to measure the speed of a TCP server I'm writing, and I've noticed that there might be a fundamental problem of measuring the speed of the connect() calls: if I connect in a non-blocking way, connect() operations become very slow after a few seconds. Here is the example code in Python: ``` #! /usr/bin/python2.4 import errno import os import select import socket import sys import time def NonBlockingConnect(sock, addr): #time.sleep(0.0001) # Fixes the problem. while True: try: return sock.connect(addr) except socket.error, e: if e.args[0] not in (errno.EINPROGRESS, errno.EALREADY): raise os.write(2, '^') if not select.select((), (sock,), (), 0.5)[1]: os.write(2, 'P') def InfiniteClient(addr): while True: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) sock.setblocking(0) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # sock.connect(addr) NonBlockingConnect(sock, addr) sock.close() os.write(2, '.') def InfiniteServer(server_socket): while True: sock, addr = server_socket.accept() sock.close() server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind(('127.0.0.1', 45454)) server_socket.listen(128) if os.fork(): # Parent. InfiniteServer(server_socket) else: addr = server_socket.getsockname() server_socket.close() InfiniteClient(addr) ``` With `NonBlockingConnect`, most connect() operations are fast, but in every few seconds there happens to be one connect() operation which takes at least 2 seconds (as indicated by 5 consecutive `P` letters on the output). By using `sock.connect` instead of `NonBlockingConnect` all connect operations seem to be fast. How is it possible to get rid of these slow connect()s? I'm running Ubuntu Karmic desktop with the standard PAE kernel: ``` Linux narancs 2.6.31-20-generic-pae #57-Ubuntu SMP Mon Feb 8 10:23:59 UTC 2010 i686 GNU/Linux ``` It's strange that there are no delays with `strace -f ./conn.py`. It's strange that there are no delays if I uncomment the very fast `time.sleep`. It's strange that there are no delays on my Ubuntu Hardy system: All these systems are affected (running Ubuntu Karmic, Ubuntu Hardy, Debian Etch): ``` Linux narancs 2.6.31-20-generic-pae #57-Ubuntu SMP Mon Feb 8 10:23:59 UTC 2010 i686 GNU/Linux Linux t 2.6.24-grsec #1 SMP Thu Apr 24 14:15:58 CEST 2008 x86_64 GNU/Linux Linux geekpad 2.6.24-24-generic #1 SMP Fri Sep 18 16:49:39 UTC 2009 i686 GNU/Linux ``` It's strange that the following Debian Lenny system is not affected: ``` Linux t 2.6.31.5 #2 SMP Thu Nov 5 15:33:05 CET 2009 i686 GNU/Linux ``` FYI There are no delays if I use an AF_UNIX socket. FYI I get the same behavior if I implement the client in C: ``` /* by pts@fazekas.hu at Sun Apr 25 20:47:24 CEST 2010 */ #include <arpa/inet.h> #include <errno.h> #include <fcntl.h> #include <netinet/in.h> #include <stdio.h> #include <string.h> #include <sys/select.h> #include <sys/socket.h> #include <unistd.h> static int work(void) { fd_set rset; fd_set wset; fd_set eset; socklen_t sl; struct timeval timeout; struct sockaddr_in sa; int sd, i, j; long l; sd = socket(AF_INET, SOCK_STREAM, 0); if (sd < 0) { perror("socket"); return 2; } l = fcntl(sd, F_GETFL, 0); if (l < 0) { perror("fcntl-getfl"); close(sd); return 2; } if (0 != fcntl(sd, F_SETFL, l | O_NONBLOCK)) { perror("fcntl-setfl"); close(sd); return 2; } memset(&sa, '\0', sizeof(sa)); sa.sin_family = AF_INET; sa.sin_port = htons(45454); sa.sin_addr.s_addr = inet_addr("127.0.0.1"); while (0 != connect(sd, (struct sockaddr*)&sa, sizeof sa)) { if (errno != EAGAIN && errno != EINPROGRESS && errno != EALREADY) { perror("connect"); close(sd); return 2; } FD_ZERO(&rset); FD_ZERO(&wset); FD_ZERO(&eset); j = 0; do { timeout.tv_sec = 0; timeout.tv_usec = 100 * 1000; /* 0.1 sec */ FD_SET(sd, &wset); FD_SET(sd, &eset); i = select(sd + 1, &rset, &wset, &eset, &timeout); if (i < 0) { perror("select"); close(sd); return 2; } if (++j == 5) { (void)write(2, "P", 1); j = 0; } } while (i == 0); sl = sizeof i; if (0 != getsockopt(sd, SOL_SOCKET, SO_ERROR, &i, &sl)) { perror("getsockopt"); close(sd); return 2; } if (i != 0) { if (i == ECONNRESET) { (void)write(2, "R", 1); close(sd); return -3; } fprintf(stderr, "connect-SO_ERROR: %s\n", strerror(i)); close(sd); return 2; } } close(sd); return 0; } int main(int argc, char**argv) { int i; (void)argc; (void)argv; while ((i = work()) <= 0) (void)write(2, ".", 1); return i; } ```