Prefork: Use i % total to cycle between ready_fds instead of shuffle

ask · ask · commit f5086b38fe2b · 2014-06-05T20:16:28.000+01:00
diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py
@@ -677,14 +677,25 @@ def on_inqueue_close(fd, proc):
                 pass
         self.on_inqueue_close = on_inqueue_close
 
-        def schedule_writes(ready_fds, shuffle=random.shuffle):
+        def schedule_writes(ready_fds, shuffle=random.shuffle, curindex=[0]):
             # Schedule write operation to ready file descriptor.
             # The file descriptor is writeable, but that does not
             # mean the process is currently reading from the socket.
             # The socket is buffered so writeable simply means that
             # the buffer can accept at least 1 byte of data.
-            shuffle(ready_fds)
-            for ready_fd in ready_fds:
+
+            # This means we have to cycle between the ready fds.
+            # the first version used shuffle, but using i % total
+            # is about 30% faster with many processes.  The latter
+            # also shows more fairness in write stats when used with
+            # many processes [XXX On OS X, this may vary depending
+            # on event loop implementation (i.e select vs epoll), so
+            # have to test further]
+            total = len(ready_fds)
+
+            for i in range(total):
+                ready_fd = ready_fds[curindex[0] % total]
+                curindex[0] += 1
                 if ready_fd in active_writes:
                     # already writing to this fd
                     continue
