shithub: sox

Info  •  Files  •  Log  •  Branches

speed up pipe inputs on Linux 2.6.35+

Linux 2.6.35+ allows pipe buffer resizing via fcntl(2).   When
running multi-threaded SoX invocations with large buffers, the
default pipe size (64K) can be too small and become a bottleneck
for IPC.  Increasing the pipe to the maximum allowed size
reduces the amount of stalls in data flow between processes (SoX
or otherwise).

When using SOX_OPTS="--multi-thread --buffer 131072" on a 4-core
system, a command like:

	sox -M "|sox $< -p $(lft_fx)" "|sox $< -p $(rgt_fx)" $@

..can run significantly faster (10-80%) depending on the
processing chain, file sizes/formats and effects in use.

Before this patch, using "--buffer 131072" could be hugely
detrimental to performance due to the pipe being only half
the size (64K) of the SoX buffer.

--- a/src/formats.c

+++ b/src/formats.c

@@ -18,6 +18,7 @@

  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

  */

+#define _GNU_SOURCE

 #include "sox_i.h"

 #include <assert.h>

@@ -37,6 +38,10 @@

   #include <magic.h>

 #endif

+#ifdef HAVE_UNISTD_H

+#  include <unistd.h>

+#endif

+

 #define PIPE_AUTO_DETECT_SIZE 256 /* Only as much as we can rewind a pipe */

 #define AUTO_DETECT_SIZE 4096     /* For seekable file, so no restriction */

@@ -370,6 +375,50 @@

     fclose(file);

 }

+static void incr_pipe_size(FILE *f)

+{

+/*

+ * Linux 2.6.35 and later has the ability to expand the pipe buffer

+ * Try to get it as big as possible to avoid stalls when SoX itself

+ * is using big buffers

+ */

+#if defined(F_GETPIPE_SZ) && defined(F_SETPIPE_SZ)

+  static long max_pipe_size;

+

+  /* read the maximum size of the pipe the first time this is called */

+  if (max_pipe_size == 0) {

+    const char path[] = "/proc/sys/fs/pipe-max-size";

+    int fd = open(path, O_RDONLY);

+

+    max_pipe_size = -1;

+    if (fd >= 0) {

+      char buf[80];

+      ssize_t r = read(fd, buf, sizeof(buf) - 1);

+

+      if (r > 0) {

+        buf[r] = 0;

+        max_pipe_size = strtol(buf, NULL, 10);

+

+        /* guard against obviously wrong values on messed up systems */

+        if (max_pipe_size <= PIPE_BUF || max_pipe_size > INT_MAX)

+          max_pipe_size = -1;

+      }

+      close(fd);

+    }

+  }

+

+  if (max_pipe_size > PIPE_BUF) {

+    int fd = fileno(f);

+

+    if (fcntl(fd, F_SETPIPE_SZ, max_pipe_size) >= 0)

+      lsx_debug("got pipe %ld bytes\n", max_pipe_size);

+    else

+      lsx_warn("couldn't set pipe size to %ld bytes: %s\n",

+               max_pipe_size, strerror(errno));

+  }

+#endif /* do nothing for platforms without F_{GET,SET}PIPE_SZ */

+}

+

 static FILE * xfopen(char const * identifier, char const * mode, lsx_io_type * io_type)

 {

   *io_type = lsx_io_file;

@@ -382,6 +431,7 @@

 #endif

     f = popen(identifier + 1, POPEN_MODE);

     *io_type = lsx_io_pipe;

+    incr_pipe_size(f);

 #else

     lsx_fail("this build of SoX cannot open pipes");

 #endif

@@ -394,6 +444,7 @@

     char * command = lsx_malloc(strlen(command_format) + strlen(identifier));

     sprintf(command, command_format, identifier);

     f = popen(command, POPEN_MODE);

+    incr_pipe_size(f);

     free(command);

     *io_type = lsx_io_url;

 #else