IO copy API for stream copying

.github/workflows/unit-tests.yml

@@ -3,13 +3,17 @@ on:
   push:
     paths:
       - 'main/network.c'
+      - 'main/io/**'
+      - 'main/php_io.h'
       - 'tests/unit/**'
       - '.github/workflows/unit-tests.yml'
     branches:
       - master
   pull_request:
     paths:
       - 'main/network.c'
+      - 'main/io/**'
+      - 'main/php_io.h'
       - 'tests/unit/**'
       - '.github/workflows/unit-tests.yml'
     branches:

configure.ac

@@ -369,6 +369,9 @@ AC_SEARCH_LIBS([Pgrab], [proc])
 dnl Haiku does not have network api in libc.
 AC_SEARCH_LIBS([setsockopt], [network])
 
+dnl Solaris/illumos provide sendfile() in libsendfile; libc on Linux/FreeBSD.
+AC_SEARCH_LIBS([sendfile], [sendfile])
+
 dnl Check for openpty. It may require linking against libutil or libbsd.
 AC_CHECK_FUNCS([openpty],,
   [AC_SEARCH_LIBS([openpty], [util bsd], [AC_DEFINE([HAVE_OPENPTY], [1])])])
@@ -588,10 +591,12 @@ AC_CHECK_FUNCS(m4_normalize([
   putenv
   reallocarray
   scandir
+  sendfile
   setenv
   setitimer
   shutdown
   sigprocmask
+  splice
   statfs
   statvfs
   std_syslog
@@ -1685,6 +1690,15 @@ PHP_ADD_SOURCES_X([main],
   [PHP_FASTCGI_OBJS],
   [no])
 
+PHP_ADD_SOURCES([main/io], m4_normalize([
+    php_io.c
+    php_io_copy_linux.c
+    php_io_copy_freebsd.c
+    php_io_copy_solaris.c
+    php_io_copy_macos.c
+  ]),
+  [-DZEND_ENABLE_STATIC_TSRMLS_CACHE=1])
+
 PHP_ADD_SOURCES([main/poll], m4_normalize([
     poll_backend_epoll.c
     poll_backend_eventport.c

ext/openssl/tests/stream_copy_to_stream_ssl_to_file.phpt

@@ -0,0 +1,71 @@
+--TEST--
+stream_copy_to_stream() from a TLS stream copies decrypted data (no fd fast-path)
+--EXTENSIONS--
+openssl
+--SKIPIF--
+<?php
+if (!function_exists("proc_open")) die("skip no proc_open");
+?>
+--FILE--
+<?php
+
+$certFile = __DIR__ . DIRECTORY_SEPARATOR . 'stream_copy_ssl.pem.tmp';
+$cacertFile = __DIR__ . DIRECTORY_SEPARATOR . 'stream_copy_ssl-ca.pem.tmp';
+
+$serverCode = <<<'CODE'
+    $serverCtx = stream_context_create(['ssl' => [
+        'local_cert' => '%s',
+    ]]);
+    $flags = STREAM_SERVER_BIND | STREAM_SERVER_LISTEN;
+    $server = stream_socket_server("ssl://127.0.0.1:0", $errno, $errstr, $flags, $serverCtx);
+    phpt_notify_server_start($server);
+
+    $conn = stream_socket_accept($server, 5);
+    fwrite($conn, str_repeat("secret-", 1000));
+    fclose($conn);
+    fclose($server);
+CODE;
+$serverCode = sprintf($serverCode, $certFile);
+
+$peerName = 'stream_copy_ssl_peer';
+$clientCode = <<<'CODE'
+    $clientCtx = stream_context_create(['ssl' => [
+        'verify_peer' => true,
+        'cafile' => '%s',
+        'peer_name' => '%s',
+    ]]);
+    $client = stream_socket_client("ssl://{{ ADDR }}", $errno, $errstr, 5, STREAM_CLIENT_CONNECT, $clientCtx);
+
+    $tmp = tmpfile();
+    /* If the copy offloaded the raw socket fd it would write ciphertext; the
+     * decrypted plaintext proves it correctly fell back to the userspace loop. */
+    $copied = stream_copy_to_stream($client, $tmp);
+    var_dump($copied);
+
+    fseek($tmp, 0, SEEK_SET);
+    $content = stream_get_contents($tmp);
+    var_dump(strlen($content));
+    var_dump($content === str_repeat("secret-", 1000));
+
+    fclose($tmp);
+    fclose($client);
+CODE;
+$clientCode = sprintf($clientCode, $cacertFile, $peerName);
+
+include 'CertificateGenerator.inc';
+$certificateGenerator = new CertificateGenerator();
+$certificateGenerator->saveCaCert($cacertFile);
+$certificateGenerator->saveNewCertAsFileWithKey($peerName, $certFile);
+
+include 'ServerClientTestCase.inc';
+ServerClientTestCase::getInstance()->run($clientCode, $serverCode);
+?>
+--CLEAN--
+<?php
+@unlink(__DIR__ . DIRECTORY_SEPARATOR . 'stream_copy_ssl.pem.tmp');
+@unlink(__DIR__ . DIRECTORY_SEPARATOR . 'stream_copy_ssl-ca.pem.tmp');
+?>
+--EXPECT--
+int(7000)
+int(7000)
+bool(true)

ext/openssl/xp_ssl.c

@@ -27,7 +27,7 @@
 #include "zend_exceptions.h"
 #include "php_openssl.h"
 #include "php_openssl_backend.h"
-#include "php_network.h"
+#include "php_io.h"
 #include <openssl/ssl.h>
 #include <openssl/rsa.h>
 #include <openssl/x509.h>
@@ -3627,6 +3627,18 @@ static int php_openssl_sockop_cast(php_stream *stream, int castas, void **ret)
 				*(php_socket_t *)ret = sslsock->s.socket;
 			}
 			return SUCCESS;
+		case PHP_STREAM_AS_FD_FOR_COPY:
+			if (sslsock->ssl_active) {
+				return FAILURE;
+			}
+			if (ret) {
+				php_io_fd *copy_fd = (php_io_fd *) ret;
+				copy_fd->socket = sslsock->s.socket;
+				copy_fd->fd_type = PHP_IO_FD_SOCKET;
+				copy_fd->timeout = sslsock->s.timeout;
+				copy_fd->is_blocked = sslsock->s.is_blocked;
+			}
+			return SUCCESS;
 		default:
 			return FAILURE;
 	}

ext/standard/tests/streams/stream_copy_to_stream_file_to_file_append.phpt

@@ -0,0 +1,34 @@
+--TEST--
+stream_copy_to_stream() file to file with an append-mode destination
+--FILE--
+<?php
+
+$srcFile = __DIR__ . '/stream_copy_append_src.txt';
+$dstFile = __DIR__ . '/stream_copy_append_dst.txt';
+
+file_put_contents($srcFile, str_repeat("b", 3000));
+file_put_contents($dstFile, "PREFIX-");
+
+$src = fopen($srcFile, 'r');
+/* O_APPEND must disable the fd-level copy fast-path and still append correctly. */
+$dst = fopen($dstFile, 'a');
+
+$copied = stream_copy_to_stream($src, $dst);
+var_dump($copied);
+
+fclose($src);
+fclose($dst);
+
+$result = file_get_contents($dstFile);
+var_dump(strlen($result));
+var_dump($result === "PREFIX-" . str_repeat("b", 3000));
+?>
+--CLEAN--
+<?php
+@unlink(__DIR__ . '/stream_copy_append_src.txt');
+@unlink(__DIR__ . '/stream_copy_append_dst.txt');
+?>
+--EXPECT--
+int(3000)
+int(3007)
+bool(true)

ext/standard/tests/streams/stream_copy_to_stream_file_to_file_dest_read_ahead.phpt

@@ -0,0 +1,38 @@
+--TEST--
+stream_copy_to_stream() file to file with a partially read destination
+--FILE--
+<?php
+
+$srcFile = __DIR__ . '/stream_copy_dest_read_ahead_src.txt';
+$dstFile = __DIR__ . '/stream_copy_dest_read_ahead_dst.txt';
+
+file_put_contents($srcFile, str_repeat("N", 50));
+file_put_contents($dstFile, str_repeat("O", 3000));
+
+$src = fopen($srcFile, 'r');
+$dst = fopen($dstFile, 'r+');
+/* Buffered read-ahead moves the fd offset past the stream position; the copy
+ * must land at the stream position. */
+fread($dst, 10);
+
+$copied = stream_copy_to_stream($src, $dst);
+var_dump($copied);
+var_dump(ftell($dst));
+
+fclose($src);
+fclose($dst);
+
+$result = file_get_contents($dstFile);
+var_dump(strlen($result));
+var_dump($result === str_repeat("O", 10) . str_repeat("N", 50) . str_repeat("O", 2940));
+?>
+--CLEAN--
+<?php
+@unlink(__DIR__ . '/stream_copy_dest_read_ahead_src.txt');
+@unlink(__DIR__ . '/stream_copy_dest_read_ahead_dst.txt');
+?>
+--EXPECT--
+int(50)
+int(60)
+int(3000)
+bool(true)

ext/standard/tests/streams/stream_copy_to_stream_file_to_file_over_2gb.phpt

@@ -0,0 +1,44 @@
+--TEST--
+stream_copy_to_stream() copies files larger than 2GB in full
+--SKIPIF--
+<?php
+if (!getenv('RUN_RESOURCE_HEAVY_TESTS')) die('skip resource-heavy test');
+if (PHP_INT_SIZE < 8) die('skip 64-bit only');
+if (getenv('SKIP_SLOW_TESTS')) die('skip slow test');
+$dir = sys_get_temp_dir();
+if (disk_free_space($dir) < 4 * 1024 * 1024 * 1024) {
+    die('skip Reason: Insufficient disk space (less than 4GB)');
+}
+?>
+--FILE--
+<?php
+$size = 3 * 1024 * 1024 * 1024; // exceeds the ~2GB per-call kernel copy limit
+$src = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "stream_copy_over_2gb_src.bin";
+$dst = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "stream_copy_over_2gb_dst.bin";
+
+// Create a sparse 3GB source so the copy loop runs without using 3GB of data.
+$fh = fopen($src, "wb");
+fseek($fh, $size - 1);
+fwrite($fh, "\0");
+fclose($fh);
+
+$in = fopen($src, "rb");
+$out = fopen($dst, "wb");
+$copied = stream_copy_to_stream($in, $out);
+fclose($in);
+fclose($out);
+
+var_dump($copied === $size);
+var_dump(filesize($dst) === $size);
+
+unlink($src);
+unlink($dst);
+?>
+--EXPECT--
+bool(true)
+bool(true)
+--CLEAN--
+<?php
+@unlink(sys_get_temp_dir() . DIRECTORY_SEPARATOR . "stream_copy_over_2gb_src.bin");
+@unlink(sys_get_temp_dir() . DIRECTORY_SEPARATOR . "stream_copy_over_2gb_dst.bin");
+?>

ext/standard/tests/streams/stream_copy_to_stream_file_to_socket_maxlen.phpt

@@ -0,0 +1,57 @@
+--TEST--
+stream_copy_to_stream() file to socket with a maxlength shorter than the file (bounded sendfile + source offset)
+--SKIPIF--
+<?php
+if (!function_exists("proc_open")) die("skip no proc_open");
+?>
+--FILE--
+<?php
+
+$serverCode = <<<'CODE'
+    $server = stream_socket_server("tcp://127.0.0.1:0", $errno, $errstr);
+    phpt_notify_server_start($server);
+
+    $conn = stream_socket_accept($server);
+    $result = stream_get_contents($conn);
+
+    phpt_notify(message: strlen($result));
+    phpt_notify(message: $result === str_repeat("A", 8192) ? "match" : "mismatch");
+
+    fclose($conn);
+    fclose($server);
+CODE;
+
+$clientCode = <<<'CODE'
+    $src = tmpfile();
+    fwrite($src, str_repeat("A", 8192) . str_repeat("B", 8192));
+    rewind($src);
+
+    $dest = stream_socket_client("tcp://{{ ADDR }}", $errno, $errstr, 10);
+
+    /* Only the first 8192 bytes must be sent: the bounded sendfile path has to
+     * stop at maxlen rather than streaming to EOF. */
+    $copied = stream_copy_to_stream($src, $dest, 8192);
+    var_dump($copied);
+
+    /* The source position must have advanced by exactly maxlen, so the kernel
+     * offload restored the descriptor offset to the maxlen boundary. */
+    $rest = fread($src, 8192);
+    var_dump(strlen($rest));
+    var_dump($rest === str_repeat("B", 8192));
+
+    fclose($dest);
+    fclose($src);
+
+    var_dump((int) trim(phpt_wait()));
+    var_dump(trim(phpt_wait()) === "match");
+CODE;
+
+include sprintf("%s/../../../openssl/tests/ServerClientTestCase.inc", __DIR__);
+ServerClientTestCase::getInstance()->run($clientCode, $serverCode);
+?>
+--EXPECT--
+int(8192)
+int(8192)
+bool(true)
+int(8192)
+bool(true)

ext/standard/tests/streams/stream_copy_to_stream_file_to_socket_medium.phpt

@@ -0,0 +1,48 @@
+--TEST--
+stream_copy_to_stream() 16k with file as $source and socket as $dest
+--SKIPIF--
+<?php
+if (!function_exists("proc_open")) die("skip no proc_open");
+?>
+--FILE--
+<?php
+
+$serverCode = <<<'CODE'
+    $server = stream_socket_server("tcp://127.0.0.1:0", $errno, $errstr);
+    phpt_notify_server_start($server);
+
+    $conn = stream_socket_accept($server);
+    $data = str_repeat('data', 4096);
+    $result = stream_get_contents($conn);
+
+    phpt_notify(message: strlen($result));
+    phpt_notify(message: $result === $data ? "match" : "mismatch");
+
+    fclose($conn);
+    fclose($server);
+CODE;
+
+$clientCode = <<<'CODE'
+    $src = tmpfile();
+    $data = str_repeat('data', 4096);
+    fwrite($src, $data);
+    rewind($src);
+
+    $dest = stream_socket_client("tcp://{{ ADDR }}", $errno, $errstr, 10);
+    $copied = stream_copy_to_stream($src, $dest);
+    var_dump($copied);
+
+    fclose($dest);
+    fclose($src);
+
+    var_dump((int) trim(phpt_wait()));
+    var_dump(trim(phpt_wait()) === "match");
+CODE;
+
+include sprintf("%s/../../../openssl/tests/ServerClientTestCase.inc", __DIR__);
+ServerClientTestCase::getInstance()->run($clientCode, $serverCode);
+?>
+--EXPECT--
+int(16384)
+int(16384)
+bool(true)