From 1a72c0190af0cf6706be9e1813ece18266844184 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Tue, 22 May 2018 14:31:52 +0200
Subject: [PATCH 01/44] have shutil.copyfileobj use sendfile() if possible

---
 Lib/shutil.py           | 57 ++++++++++++++++++++++++++++++++++++++++-
 Lib/test/test_shutil.py | 55 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 111 insertions(+), 1 deletion(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 3c02776a406551..42e7fae713e1f2 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -10,6 +10,7 @@
 import fnmatch
 import collections
 import errno
+import io
 
 try:
     import zlib
@@ -42,6 +43,9 @@
 except ImportError:
     getgrnam = None
 
+_HAS_SENDFILE = hasattr(os, "sendfile")
+COPY_BUFSIZE = 16 * 1024
+
 __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
            "copytree", "move", "rmtree", "Error", "SpecialFileError",
            "ExecError", "make_archive", "get_archive_formats",
@@ -72,9 +76,60 @@ class RegistryError(Exception):
     """Raised when a registry operation with the archiving
     and unpacking registries fails"""
 
+class _GiveupOnSendfile(Exception):
+    """Raised when os.sendfile() cannot be used"""
+
+
+def _copyfileobj_sendfile(fsrc, fdst):
+    """Copy data from one file object to another one by using
+    zero-copy sendfile() method (faster).
+    """
+    try:
+        infd = fsrc.fileno()
+        outfd = fdst.fileno()
+    except (AttributeError, io.UnsupportedOperation) as err:
+        raise _GiveupOnSendfile(err)  # not a regular file
+
+    try:
+        blocksize = os.fstat(infd).st_size
+    except OSError:
+        blocksize = COPY_BUFSIZE
+    else:
+        if blocksize <= 0:
+            blocksize = COPY_BUFSIZE
+
+    try:
+        offset = fsrc.tell()
+    except (AttributeError, io.UnsupportedOperation) as err:
+        offset = 0
 
-def copyfileobj(fsrc, fdst, length=16*1024):
+    total_sent = 0
+    while True:
+        try:
+            sent = os.sendfile(outfd, infd, offset, blocksize)
+        except OSError as err:
+            if total_sent == 0:
+                # We can get here for different reasons, the main
+                # one being a fd is not a regular mmap(2)-like
+                # fd, in which case we'll fall back on using plain
+                # read()/write() copy.
+                raise _GiveupOnSendfile(err)
+            else:
+                raise err from None
+        else:
+            if sent == 0:
+                break  # EOF
+            offset += sent
+            total_sent += sent
+
+def copyfileobj(fsrc, fdst, length=COPY_BUFSIZE):
     """copy data from file-like object fsrc to file-like object fdst"""
+    if _HAS_SENDFILE:
+        try:
+            return _copyfileobj_sendfile(fsrc, fdst)
+        except _GiveupOnSendfile:
+            pass
+
     while 1:
         buf = fsrc.read(length)
         if not buf:
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 2cb2f14643e1b3..1f990536223ea6 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -12,6 +12,8 @@
 import functools
 import pathlib
 import subprocess
+import random
+import string
 from shutil import (make_archive,
                     register_archive_format, unregister_archive_format,
                     get_archive_formats, Error, unpack_archive,
@@ -1829,6 +1831,59 @@ def test_move_dir_caseinsensitive(self):
         finally:
             os.rmdir(dst_dir)
 
+
+@unittest.skipIf(not hasattr(os, "sendfile"), 'needs os.sendfile()')
+class TestCopyFileObjSendfile(unittest.TestCase):
+    FILESIZE = (10 * 1024 * 1024)  # 10 MiB
+    BUFSIZE = 8192
+    FILEDATA = b""
+
+    @classmethod
+    def setUpClass(cls):
+        def chunks(total, step):
+            assert total >= step
+            while total > step:
+                yield step
+                total -= step
+            if total:
+                yield total
+
+        chunk = b"".join([random.choice(string.ascii_letters).encode()
+                          for i in range(cls.BUFSIZE)])
+        with open(TESTFN, 'wb') as f:
+            for csize in chunks(cls.FILESIZE, cls.BUFSIZE):
+                f.write(chunk)
+        with open(TESTFN, 'rb') as f:
+            cls.FILEDATA = f.read()
+            assert len(cls.FILEDATA) == cls.FILESIZE
+
+    @classmethod
+    def tearDownClass(cls):
+        support.unlink(TESTFN)
+
+    def tearDown(self):
+        support.unlink(TESTFN2)
+
+    def get_files(self):
+        src = open(TESTFN, "rb")
+        self.addCleanup(src.close)
+        dst = open(TESTFN2, "wb")
+        self.addCleanup(dst.close)
+        return src, dst
+
+    def test_regular_copy(self):
+        src, dst = self.get_files()
+        shutil.copyfileobj(src, dst)
+        with open(TESTFN2, "rb") as f:
+            self.assertEqual(f.read(), self.FILEDATA)
+
+    def test_unhandled_exception(self):
+        src, dst = self.get_files()
+        with unittest.mock.patch('os.sendfile',
+                                 side_effect=ZeroDivisionError):
+            self.assertRaises(ZeroDivisionError, shutil.copyfileobj, src, dst)
+
+
 class TermsizeTests(unittest.TestCase):
     def test_does_not_crash(self):
         """Check if get_terminal_size() returns a meaningful value.

From 77c4bfae6e33718d4fcb27a055bfd5e1f82a1a56 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Tue, 22 May 2018 14:34:15 +0200
Subject: [PATCH 02/44] refactoring: use ctx manager

---
 Lib/test/test_shutil.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 1f990536223ea6..ead520b0f0e70c 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -14,6 +14,7 @@
 import subprocess
 import random
 import string
+import contextlib
 from shutil import (make_archive,
                     register_archive_format, unregister_archive_format,
                     get_archive_formats, Error, unpack_archive,
@@ -1864,24 +1865,24 @@ def tearDownClass(cls):
     def tearDown(self):
         support.unlink(TESTFN2)
 
+    @contextlib.contextmanager
     def get_files(self):
-        src = open(TESTFN, "rb")
-        self.addCleanup(src.close)
-        dst = open(TESTFN2, "wb")
-        self.addCleanup(dst.close)
-        return src, dst
+        with open(TESTFN, "rb") as src:
+            with open(TESTFN2, "wb") as dst:
+                yield (src, dst)
 
     def test_regular_copy(self):
-        src, dst = self.get_files()
-        shutil.copyfileobj(src, dst)
+        with self.get_files() as (src, dst):
+            shutil.copyfileobj(src, dst)
         with open(TESTFN2, "rb") as f:
             self.assertEqual(f.read(), self.FILEDATA)
 
     def test_unhandled_exception(self):
-        src, dst = self.get_files()
         with unittest.mock.patch('os.sendfile',
                                  side_effect=ZeroDivisionError):
-            self.assertRaises(ZeroDivisionError, shutil.copyfileobj, src, dst)
+            with self.get_files() as (src, dst):
+                self.assertRaises(ZeroDivisionError,
+                                  shutil.copyfileobj, src, dst)
 
 
 class TermsizeTests(unittest.TestCase):

From 2afa04ac3a42955fb674156d000d543ece032e18 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Tue, 22 May 2018 14:40:10 +0200
Subject: [PATCH 03/44] add test with non-regular file obj

---
 Lib/test/test_shutil.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index ead520b0f0e70c..2a8356080982a6 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -15,6 +15,7 @@
 import random
 import string
 import contextlib
+import io
 from shutil import (make_archive,
                     register_archive_format, unregister_archive_format,
                     get_archive_formats, Error, unpack_archive,
@@ -1877,6 +1878,13 @@ def test_regular_copy(self):
         with open(TESTFN2, "rb") as f:
             self.assertEqual(f.read(), self.FILEDATA)
 
+    def test_non_regular_file(self):
+        with io.BytesIO(self.FILEDATA) as src:
+            with open(TESTFN2, "wb") as dst:
+                shutil.copyfileobj(src, dst)
+        with open(TESTFN2, "rb") as f:
+            self.assertEqual(f.read(), self.FILEDATA)
+
     def test_unhandled_exception(self):
         with unittest.mock.patch('os.sendfile',
                                  side_effect=ZeroDivisionError):

From 542cd17739348dc5f01a45628c8055c7792e7034 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Tue, 22 May 2018 14:44:53 +0200
Subject: [PATCH 04/44] emulate case where file size can't be determined

---
 Lib/test/test_shutil.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 2a8356080982a6..37776435de372f 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1892,6 +1892,12 @@ def test_unhandled_exception(self):
                 self.assertRaises(ZeroDivisionError,
                                   shutil.copyfileobj, src, dst)
 
+    def test_cant_get_size(self):
+        with unittest.mock.patch('os.fstat', side_effect=OSError) as m:
+            with self.get_files() as (src, dst):
+                shutil.copyfileobj(src, dst)
+                assert m.called
+
 
 class TermsizeTests(unittest.TestCase):
     def test_does_not_crash(self):

From 3520c6c4a8b86f0bad21189223eb33ac3ee8e868 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Tue, 22 May 2018 15:09:33 +0200
Subject: [PATCH 05/44] reference _copyfileobj_sendfile directly

---
 Lib/test/test_shutil.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 37776435de372f..1112e965f9a75c 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -21,7 +21,7 @@
                     get_archive_formats, Error, unpack_archive,
                     register_unpack_format, RegistryError,
                     unregister_unpack_format, get_unpack_formats,
-                    SameFileError)
+                    SameFileError, _GiveupOnSendfile)
 import tarfile
 import zipfile
 
@@ -1874,14 +1874,17 @@ def get_files(self):
 
     def test_regular_copy(self):
         with self.get_files() as (src, dst):
-            shutil.copyfileobj(src, dst)
+            shutil._copyfileobj_sendfile(src, dst)
         with open(TESTFN2, "rb") as f:
             self.assertEqual(f.read(), self.FILEDATA)
 
     def test_non_regular_file(self):
         with io.BytesIO(self.FILEDATA) as src:
             with open(TESTFN2, "wb") as dst:
+                with self.assertRaises(_GiveupOnSendfile):
+                    shutil._copyfileobj_sendfile(src, dst)
                 shutil.copyfileobj(src, dst)
+
         with open(TESTFN2, "rb") as f:
             self.assertEqual(f.read(), self.FILEDATA)
 
@@ -1895,7 +1898,7 @@ def test_unhandled_exception(self):
     def test_cant_get_size(self):
         with unittest.mock.patch('os.fstat', side_effect=OSError) as m:
             with self.get_files() as (src, dst):
-                shutil.copyfileobj(src, dst)
+                shutil._copyfileobj_sendfile(src, dst)
                 assert m.called
 
 

From 050a7222fe5ea76efe5e32389ac6f599a17c4c83 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Tue, 22 May 2018 15:27:09 +0200
Subject: [PATCH 06/44] add test for offset() at certain position

---
 Lib/test/test_shutil.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 1112e965f9a75c..47a3e87cc8b484 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1888,6 +1888,14 @@ def test_non_regular_file(self):
         with open(TESTFN2, "rb") as f:
             self.assertEqual(f.read(), self.FILEDATA)
 
+    def test_start_offset(self):
+        # Modify src file position.
+        with self.get_files() as (src, dst):
+            src.seek(666)
+            shutil._copyfileobj_sendfile(src, dst)
+        with open(TESTFN2, "rb") as f:
+            self.assertEqual(f.read(), self.FILEDATA[666:])
+
     def test_unhandled_exception(self):
         with unittest.mock.patch('os.sendfile',
                                  side_effect=ZeroDivisionError):
@@ -1895,7 +1903,20 @@ def test_unhandled_exception(self):
                 self.assertRaises(ZeroDivisionError,
                                   shutil.copyfileobj, src, dst)
 
+    def test_exception_on_first_call(self):
+        # Emulate a case where the first call to sendfile() raises
+        # an exception in which case the function is supposed to
+        # give up immediately.
+        with unittest.mock.patch('os.sendfile',
+                                 side_effect=OSError):
+            with self.get_files() as (src, dst):
+                with self.assertRaises(_GiveupOnSendfile):
+                    shutil._copyfileobj_sendfile(src, dst)
+
     def test_cant_get_size(self):
+        # Emulate a case where src file size cannot be determined.
+        # Internally bufsize will be set to a small value and
+        # sendfile() will be called repeatedly.
         with unittest.mock.patch('os.fstat', side_effect=OSError) as m:
             with self.get_files() as (src, dst):
                 shutil._copyfileobj_sendfile(src, dst)

From c1fd38af066bbc9287a714e00b2b4d18857feccf Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Tue, 22 May 2018 15:32:47 +0200
Subject: [PATCH 07/44] add test for empty file

---
 Lib/test/test_shutil.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 47a3e87cc8b484..7fa05c74f9b0ce 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1888,8 +1888,22 @@ def test_non_regular_file(self):
         with open(TESTFN2, "rb") as f:
             self.assertEqual(f.read(), self.FILEDATA)
 
-    def test_start_offset(self):
-        # Modify src file position.
+    def test_empty_file(self):
+        srcname = TESTFN + 'src'
+        dstname = TESTFN + 'dst'
+        self.addCleanup(lambda: support.unlink(srcname))
+        self.addCleanup(lambda: support.unlink(dstname))
+        with open(srcname, "wb"):
+            pass
+
+        with open(srcname, "rb") as src:
+            with open(dstname, "wb") as dst:
+                shutil._copyfileobj_sendfile(src, dst)
+
+        with open(dstname, "rb") as f:
+            self.assertEqual(f.read(), b"")
+
+    def test_start_position(self):
         with self.get_files() as (src, dst):
             src.seek(666)
             shutil._copyfileobj_sendfile(src, dst)

From 2ab63171f3340a66fcd668a856b858487d522afd Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Tue, 22 May 2018 15:37:27 +0200
Subject: [PATCH 08/44] add test for non regular file dst

---
 Lib/test/test_shutil.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 7fa05c74f9b0ce..0d248f48ea49cf 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1878,7 +1878,7 @@ def test_regular_copy(self):
         with open(TESTFN2, "rb") as f:
             self.assertEqual(f.read(), self.FILEDATA)
 
-    def test_non_regular_file(self):
+    def test_non_regular_file_src(self):
         with io.BytesIO(self.FILEDATA) as src:
             with open(TESTFN2, "wb") as dst:
                 with self.assertRaises(_GiveupOnSendfile):
@@ -1888,6 +1888,15 @@ def test_non_regular_file(self):
         with open(TESTFN2, "rb") as f:
             self.assertEqual(f.read(), self.FILEDATA)
 
+    def test_non_regular_file_dst(self):
+        with open(TESTFN, "rb") as src:
+            with io.BytesIO() as dst:
+                with self.assertRaises(_GiveupOnSendfile):
+                    shutil._copyfileobj_sendfile(src, dst)
+                shutil.copyfileobj(src, dst)
+                dst.seek(0)
+                self.assertEqual(dst.read(), self.FILEDATA)
+
     def test_empty_file(self):
         srcname = TESTFN + 'src'
         dstname = TESTFN + 'dst'

From dacc3b6ddeeb9c3f79abbb956eafd5bdf35700fe Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Tue, 22 May 2018 19:14:54 +0200
Subject: [PATCH 09/44] small refactoring

---
 Lib/shutil.py           | 4 ++--
 Lib/test/test_shutil.py | 9 +++------
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 42e7fae713e1f2..bcaa4da59435a4 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -81,8 +81,8 @@ class _GiveupOnSendfile(Exception):
 
 
 def _copyfileobj_sendfile(fsrc, fdst):
-    """Copy data from one file object to another one by using
-    zero-copy sendfile() method (faster).
+    """Copy data from one file object to another by using zero-copy
+    sendfile() method (faster).
     """
     try:
         infd = fsrc.fileno()
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 0d248f48ea49cf..709ced093610d1 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1875,8 +1875,7 @@ def get_files(self):
     def test_regular_copy(self):
         with self.get_files() as (src, dst):
             shutil._copyfileobj_sendfile(src, dst)
-        with open(TESTFN2, "rb") as f:
-            self.assertEqual(f.read(), self.FILEDATA)
+        self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA)
 
     def test_non_regular_file_src(self):
         with io.BytesIO(self.FILEDATA) as src:
@@ -1885,8 +1884,7 @@ def test_non_regular_file_src(self):
                     shutil._copyfileobj_sendfile(src, dst)
                 shutil.copyfileobj(src, dst)
 
-        with open(TESTFN2, "rb") as f:
-            self.assertEqual(f.read(), self.FILEDATA)
+        self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA)
 
     def test_non_regular_file_dst(self):
         with open(TESTFN, "rb") as src:
@@ -1916,8 +1914,7 @@ def test_start_position(self):
         with self.get_files() as (src, dst):
             src.seek(666)
             shutil._copyfileobj_sendfile(src, dst)
-        with open(TESTFN2, "rb") as f:
-            self.assertEqual(f.read(), self.FILEDATA[666:])
+        self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA[666:])
 
     def test_unhandled_exception(self):
         with unittest.mock.patch('os.sendfile',

From 29d5881e78d8040d236e6240caeb1d7427cb544e Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 18:37:23 +0200
Subject: [PATCH 10/44] leave copyfileobj() alone in order to not introduce any
 incompatibility

---
 Lib/shutil.py           | 22 ++++++++++++++++------
 Lib/test/test_shutil.py |  5 ++---
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index bcaa4da59435a4..a1a828f291862b 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -124,17 +124,27 @@ def _copyfileobj_sendfile(fsrc, fdst):
 
 def copyfileobj(fsrc, fdst, length=COPY_BUFSIZE):
     """copy data from file-like object fsrc to file-like object fdst"""
+    while 1:
+        buf = fsrc.read(length)
+        if not buf:
+            break
+        fdst.write(buf)
+
+def _copyfileobj2(fsrc, fdst):
+    """Same as above but tries to use zero-copy sendfile(2) syscall
+    (faster). This is used by copyfile(), copy() and copy2() in order
+    to leave copyfileobj() alone and not introduce backward
+    incompatibilities.
+    E.g. by using sendfile() fdst.tell() is not updated() and fdst
+    cannot be opened in "a" mode.
+    """
     if _HAS_SENDFILE:
         try:
             return _copyfileobj_sendfile(fsrc, fdst)
         except _GiveupOnSendfile:
             pass
 
-    while 1:
-        buf = fsrc.read(length)
-        if not buf:
-            break
-        fdst.write(buf)
+    return copyfileobj(fsrc, fdst)
 
 def _samefile(src, dst):
     # Macintosh, Unix.
@@ -174,7 +184,7 @@ def copyfile(src, dst, *, follow_symlinks=True):
     else:
         with open(src, 'rb') as fsrc:
             with open(dst, 'wb') as fdst:
-                copyfileobj(fsrc, fdst)
+                _copyfileobj2(fsrc, fdst)
     return dst
 
 def copymode(src, dst, *, follow_symlinks=True):
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 709ced093610d1..03bb5a4133720d 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1919,9 +1919,8 @@ def test_start_position(self):
     def test_unhandled_exception(self):
         with unittest.mock.patch('os.sendfile',
                                  side_effect=ZeroDivisionError):
-            with self.get_files() as (src, dst):
-                self.assertRaises(ZeroDivisionError,
-                                  shutil.copyfileobj, src, dst)
+            self.assertRaises(ZeroDivisionError,
+                              shutil.copyfile, TESTFN, TESTFN2)
 
     def test_exception_on_first_call(self):
         # Emulate a case where the first call to sendfile() raises

From 114c4dec91206f74ad5b86a63f04cedf969a4b25 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 18:43:55 +0200
Subject: [PATCH 11/44] minor refactoring

---
 Lib/shutil.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index a1a828f291862b..cb5a7fa721f43d 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -98,17 +98,13 @@ def _copyfileobj_sendfile(fsrc, fdst):
         if blocksize <= 0:
             blocksize = COPY_BUFSIZE
 
-    try:
-        offset = fsrc.tell()
-    except (AttributeError, io.UnsupportedOperation) as err:
-        offset = 0
-
-    total_sent = 0
+    offset = 0
+    total_copied = 0
     while True:
         try:
             sent = os.sendfile(outfd, infd, offset, blocksize)
         except OSError as err:
-            if total_sent == 0:
+            if total_copied == 0:
                 # We can get here for different reasons, the main
                 # one being a fd is not a regular mmap(2)-like
                 # fd, in which case we'll fall back on using plain
@@ -120,7 +116,7 @@ def _copyfileobj_sendfile(fsrc, fdst):
             if sent == 0:
                 break  # EOF
             offset += sent
-            total_sent += sent
+            total_copied += sent
 
 def copyfileobj(fsrc, fdst, length=COPY_BUFSIZE):
     """copy data from file-like object fsrc to file-like object fdst"""
@@ -135,8 +131,8 @@ def _copyfileobj2(fsrc, fdst):
     (faster). This is used by copyfile(), copy() and copy2() in order
     to leave copyfileobj() alone and not introduce backward
     incompatibilities.
-    E.g. by using sendfile() fdst.tell() is not updated() and fdst
-    cannot be opened in "a" mode.
+    E.g. by using sendfile() fdst cannot be opened in "a"(ppend) mode
+    and its offset doesn't get updated.
     """
     if _HAS_SENDFILE:
         try:

From 501c0dd2f92e10016fa55254fce27bec396de79d Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 18:49:06 +0200
Subject: [PATCH 12/44] remove old test

---
 Lib/test/test_shutil.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 03bb5a4133720d..4ee5235272369f 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1910,12 +1910,6 @@ def test_empty_file(self):
         with open(dstname, "rb") as f:
             self.assertEqual(f.read(), b"")
 
-    def test_start_position(self):
-        with self.get_files() as (src, dst):
-            src.seek(666)
-            shutil._copyfileobj_sendfile(src, dst)
-        self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA[666:])
-
     def test_unhandled_exception(self):
         with unittest.mock.patch('os.sendfile',
                                  side_effect=ZeroDivisionError):

From 41b4506b3f637b3bb388e37daf08cee12156aec1 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 18:52:15 +0200
Subject: [PATCH 13/44] update docstring

---
 Lib/shutil.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index cb5a7fa721f43d..cdbd11d0a0f40c 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -129,10 +129,11 @@ def copyfileobj(fsrc, fdst, length=COPY_BUFSIZE):
 def _copyfileobj2(fsrc, fdst):
     """Same as above but tries to use zero-copy sendfile(2) syscall
     (faster). This is used by copyfile(), copy() and copy2() in order
-    to leave copyfileobj() alone and not introduce backward
-    incompatibilities.
+    to leave copyfileobj() alone and not introduce any backward
+    incompatibility.
     E.g. by using sendfile() fdst cannot be opened in "a"(ppend) mode
-    and its offset doesn't get updated.
+    and its offset doesn't get updated. Also, fsrc and fdst may be
+    opened in text mode.
     """
     if _HAS_SENDFILE:
         try:

From fdb0973ff46eaaba332b343b7b9331f9bca6b768 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 19:45:24 +0200
Subject: [PATCH 14/44] update docstring; rename exception class

---
 Lib/shutil.py           | 36 ++++++++++++++++++++----------------
 Lib/test/test_shutil.py |  8 ++++----
 2 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index cdbd11d0a0f40c..3f33221dc9b6c8 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -76,10 +76,18 @@ class RegistryError(Exception):
     """Raised when a registry operation with the archiving
     and unpacking registries fails"""
 
-class _GiveupOnSendfile(Exception):
+class _GiveupOnZeroCopy(Exception):
     """Raised when os.sendfile() cannot be used"""
 
 
+def copyfileobj(fsrc, fdst, length=COPY_BUFSIZE):
+    """copy data from file-like object fsrc to file-like object fdst"""
+    while 1:
+        buf = fsrc.read(length)
+        if not buf:
+            break
+        fdst.write(buf)
+
 def _copyfileobj_sendfile(fsrc, fdst):
     """Copy data from one file object to another by using zero-copy
     sendfile() method (faster).
@@ -88,7 +96,7 @@ def _copyfileobj_sendfile(fsrc, fdst):
         infd = fsrc.fileno()
         outfd = fdst.fileno()
     except (AttributeError, io.UnsupportedOperation) as err:
-        raise _GiveupOnSendfile(err)  # not a regular file
+        raise _GiveupOnZeroCopy(err)  # not a regular file
 
     try:
         blocksize = os.fstat(infd).st_size
@@ -109,7 +117,7 @@ def _copyfileobj_sendfile(fsrc, fdst):
                 # one being a fd is not a regular mmap(2)-like
                 # fd, in which case we'll fall back on using plain
                 # read()/write() copy.
-                raise _GiveupOnSendfile(err)
+                raise _GiveupOnZeroCopy(err)
             else:
                 raise err from None
         else:
@@ -118,27 +126,23 @@ def _copyfileobj_sendfile(fsrc, fdst):
             offset += sent
             total_copied += sent
 
-def copyfileobj(fsrc, fdst, length=COPY_BUFSIZE):
-    """copy data from file-like object fsrc to file-like object fdst"""
-    while 1:
-        buf = fsrc.read(length)
-        if not buf:
-            break
-        fdst.write(buf)
-
 def _copyfileobj2(fsrc, fdst):
-    """Same as above but tries to use zero-copy sendfile(2) syscall
+    """Copies 2 filesystem files by using zero-copy sendfile(2) syscall
     (faster). This is used by copyfile(), copy() and copy2() in order
     to leave copyfileobj() alone and not introduce any backward
     incompatibility.
-    E.g. by using sendfile() fdst cannot be opened in "a"(ppend) mode
-    and its offset doesn't get updated. Also, fsrc and fdst may be
-    opened in text mode.
+    Possible incompatibilities by using sendfile() are:
+    - fdst cannot be opened in "a"(ppend) mode
+    - fdst offset doesn't get updated
+    - fsrc and fdst may be opened in text mode
+    - fsrc may be a BufferedReader (which hides unread data in a buffer),
+      GzipFile (which decompresses data), HTTPResponse (which decodes
+      chunks), ...
     """
     if _HAS_SENDFILE:
         try:
             return _copyfileobj_sendfile(fsrc, fdst)
-        except _GiveupOnSendfile:
+        except _GiveupOnZeroCopy:
             pass
 
     return copyfileobj(fsrc, fdst)
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 4ee5235272369f..77054ea3b47f37 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -21,7 +21,7 @@
                     get_archive_formats, Error, unpack_archive,
                     register_unpack_format, RegistryError,
                     unregister_unpack_format, get_unpack_formats,
-                    SameFileError, _GiveupOnSendfile)
+                    SameFileError, _GiveupOnZeroCopy)
 import tarfile
 import zipfile
 
@@ -1880,7 +1880,7 @@ def test_regular_copy(self):
     def test_non_regular_file_src(self):
         with io.BytesIO(self.FILEDATA) as src:
             with open(TESTFN2, "wb") as dst:
-                with self.assertRaises(_GiveupOnSendfile):
+                with self.assertRaises(_GiveupOnZeroCopy):
                     shutil._copyfileobj_sendfile(src, dst)
                 shutil.copyfileobj(src, dst)
 
@@ -1889,7 +1889,7 @@ def test_non_regular_file_src(self):
     def test_non_regular_file_dst(self):
         with open(TESTFN, "rb") as src:
             with io.BytesIO() as dst:
-                with self.assertRaises(_GiveupOnSendfile):
+                with self.assertRaises(_GiveupOnZeroCopy):
                     shutil._copyfileobj_sendfile(src, dst)
                 shutil.copyfileobj(src, dst)
                 dst.seek(0)
@@ -1923,7 +1923,7 @@ def test_exception_on_first_call(self):
         with unittest.mock.patch('os.sendfile',
                                  side_effect=OSError):
             with self.get_files() as (src, dst):
-                with self.assertRaises(_GiveupOnSendfile):
+                with self.assertRaises(_GiveupOnZeroCopy):
                     shutil._copyfileobj_sendfile(src, dst)
 
     def test_cant_get_size(self):

From 64d2bc59358848c2c72132bf1a8a5898b0a8e717 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 20:05:08 +0200
Subject: [PATCH 15/44] detect platforms which only support file to socket zero
 copy

---
 Lib/shutil.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 3f33221dc9b6c8..cdd816e4768b74 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -77,7 +77,7 @@ class RegistryError(Exception):
     and unpacking registries fails"""
 
 class _GiveupOnZeroCopy(Exception):
-    """Raised when os.sendfile() cannot be used"""
+    """Raised when os.sendfile() cannot be used for copying files."""
 
 
 def copyfileobj(fsrc, fdst, length=COPY_BUFSIZE):
@@ -92,6 +92,7 @@ def _copyfileobj_sendfile(fsrc, fdst):
     """Copy data from one file object to another by using zero-copy
     sendfile() method (faster).
     """
+    global _HAS_SENDFILE
     try:
         infd = fsrc.fileno()
         outfd = fdst.fileno()
@@ -112,11 +113,13 @@ def _copyfileobj_sendfile(fsrc, fdst):
         try:
             sent = os.sendfile(outfd, infd, offset, blocksize)
         except OSError as err:
+            if err.errno == errno.ENOTSOCK:
+                # sendfile() on this platform does not support copies
+                # between regular files (only sockets).
+                _HAS_SENDFILE = False
             if total_copied == 0:
-                # We can get here for different reasons, the main
-                # one being a fd is not a regular mmap(2)-like
-                # fd, in which case we'll fall back on using plain
-                # read()/write() copy.
+                # Immediately give up on first call.
+                # Probably one of the fds is not regular mmap(2)-like fd.
                 raise _GiveupOnZeroCopy(err)
             else:
                 raise err from None

From 3a3c8efbbe91d9490fe7e0e1b233cd80a640e4ed Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 20:14:58 +0200
Subject: [PATCH 16/44] don't run test on platforms where file-to-file zero
 copy is not supported

---
 Lib/test/test_shutil.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 77054ea3b47f37..90a4f192e31c33 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -88,6 +88,28 @@ def rlistdir(path):
             res.append(name)
     return res
 
+def supports_file2file_sendfile():
+    if not hasattr(os, "sendfile"):
+        return False
+    try:
+        with open(TESTFN, "wb") as f:
+            f.write(b"0123456789")
+        with open(TESTFN, "rb") as src, open(TESTFN2, "wb") as dst:
+            infd = src.fileno()
+            outfd = dst.fileno()
+            try:
+                os.sendfile(outfd, infd, 0, 1024)
+            except OSError:
+                return False
+            else:
+                return True
+    finally:
+        support.unlink(TESTFN)
+        support.unlink(TESTFN2)
+
+
+SUPPORTS_SENDFILE = supports_file2file_sendfile()
+
 
 class TestShutil(unittest.TestCase):
 
@@ -1834,7 +1856,7 @@ def test_move_dir_caseinsensitive(self):
             os.rmdir(dst_dir)
 
 
-@unittest.skipIf(not hasattr(os, "sendfile"), 'needs os.sendfile()')
+@unittest.skipIf(not SUPPORTS_SENDFILE, 'os.sendfile() not supported')
 class TestCopyFileObjSendfile(unittest.TestCase):
     FILESIZE = (10 * 1024 * 1024)  # 10 MiB
     BUFSIZE = 8192

From 78617370c1e6df95be22ada5c03eebcee3c6c249 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 20:19:39 +0200
Subject: [PATCH 17/44] use tempfiles

---
 Lib/test/test_shutil.py | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 90a4f192e31c33..94bc82cf44f38e 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -89,23 +89,32 @@ def rlistdir(path):
     return res
 
 def supports_file2file_sendfile():
+    # ...apparently Linux is the only one.
     if not hasattr(os, "sendfile"):
         return False
+    srcname = None
+    dstname = None
     try:
-        with open(TESTFN, "wb") as f:
+        with tempfile.NamedTemporaryFile("wb", delete=False) as f:
+            srcname = f.name
             f.write(b"0123456789")
-        with open(TESTFN, "rb") as src, open(TESTFN2, "wb") as dst:
-            infd = src.fileno()
-            outfd = dst.fileno()
-            try:
-                os.sendfile(outfd, infd, 0, 1024)
-            except OSError:
-                return False
-            else:
-                return True
+
+        with open(srcname, "rb") as src:
+            with tempfile.NamedTemporaryFile("wb", delete=False) as dst:
+                dstname = f.name
+                infd = src.fileno()
+                outfd = dst.fileno()
+                try:
+                    os.sendfile(outfd, infd, 0, 2)
+                except OSError:
+                    return False
+                else:
+                    return True
     finally:
-        support.unlink(TESTFN)
-        support.unlink(TESTFN2)
+        if srcname is not None:
+            support.unlink(srcname)
+        if dstname is not None:
+            support.unlink(dstname)
 
 
 SUPPORTS_SENDFILE = supports_file2file_sendfile()
@@ -2055,4 +2064,4 @@ def test_module_all_attribute(self):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main(verbosity=2)

From f3eecfdad2c9edb115ca59f5b487cfc5c655a03e Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 20:20:02 +0200
Subject: [PATCH 18/44] reset verbosity

---
 Lib/test/test_shutil.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 94bc82cf44f38e..6290dcddec98c9 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -2064,4 +2064,4 @@ def test_module_all_attribute(self):
 
 
 if __name__ == '__main__':
-    unittest.main(verbosity=2)
+    unittest.main()

From f67ce578e85ec46824bde4eef9033eb473f982de Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 20:41:43 +0200
Subject: [PATCH 19/44] add test for smaller chunks

---
 Lib/test/test_shutil.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 6290dcddec98c9..67912068ef0f9b 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1966,6 +1966,17 @@ def test_cant_get_size(self):
                 shutil._copyfileobj_sendfile(src, dst)
                 assert m.called
 
+    def test_smaller_chunks(self):
+        # Force file size detection to be smaller than the actual file
+        # size, resulting in multiple calls to sendfile().
+        mock = unittest.mock.Mock()
+        mock.st_size = 65536 + 1
+        with unittest.mock.patch('os.fstat', return_value=mock) as m:
+            with self.get_files() as (src, dst):
+                shutil._copyfileobj_sendfile(src, dst)
+                assert m.called
+        self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA)
+
 
 class TermsizeTests(unittest.TestCase):
     def test_does_not_crash(self):

From d45725453e370979e7344f8770acea26784da952 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 20:51:03 +0200
Subject: [PATCH 20/44] add big file size test

---
 Lib/test/test_shutil.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 67912068ef0f9b..82fd3ef94c4ed6 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1966,9 +1966,11 @@ def test_cant_get_size(self):
                 shutil._copyfileobj_sendfile(src, dst)
                 assert m.called
 
-    def test_smaller_chunks(self):
-        # Force file size detection to be smaller than the actual file
-        # size, resulting in multiple calls to sendfile().
+    def test_small_chunks(self):
+        # Force internal file size detection to be smaller than the
+        # actual file size. We want to force sendfile() to be called
+        # multiple times, also in order to emulate a src fd which gets
+        # bigger while it is being copied.
         mock = unittest.mock.Mock()
         mock.st_size = 65536 + 1
         with unittest.mock.patch('os.fstat', return_value=mock) as m:
@@ -1977,6 +1979,19 @@ def test_smaller_chunks(self):
                 assert m.called
         self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA)
 
+    def test_big_chunk(self):
+        # Force internal file size detection to be +100MB bigger than
+        # the actual file size. Make sure sendfile() does not rely on
+        # file size value except for (maybe) a better throughput /
+        # performance.
+        mock = unittest.mock.Mock()
+        mock.st_size = self.FILESIZE + (100 * 1024 * 1024)
+        with unittest.mock.patch('os.fstat', return_value=mock) as m:
+            with self.get_files() as (src, dst):
+                shutil._copyfileobj_sendfile(src, dst)
+                assert m.called
+        self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA)
+
 
 class TermsizeTests(unittest.TestCase):
     def test_does_not_crash(self):

From 8eb211d4db6304ff97dab5ee2ad352169373c47c Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 20:59:02 +0200
Subject: [PATCH 21/44] add comment

---
 Lib/shutil.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index cdd816e4768b74..14dc7bed585b5d 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -53,7 +53,7 @@
            "get_unpack_formats", "register_unpack_format",
            "unregister_unpack_format", "unpack_archive",
            "ignore_patterns", "chown", "which", "get_terminal_size",
-           "SameFileError"]
+           "SameFileError", "COPY_BUFSIZE"]
            # disk_usage is added later, if available on the platform
 
 class Error(OSError):
@@ -80,8 +80,10 @@ class _GiveupOnZeroCopy(Exception):
     """Raised when os.sendfile() cannot be used for copying files."""
 
 
-def copyfileobj(fsrc, fdst, length=COPY_BUFSIZE):
+def copyfileobj(fsrc, fdst, length=None):
     """copy data from file-like object fsrc to file-like object fdst"""
+    if length is None:
+        length = COPY_BUFSIZE
     while 1:
         buf = fsrc.read(length)
         if not buf:
@@ -100,6 +102,10 @@ def _copyfileobj_sendfile(fsrc, fdst):
         raise _GiveupOnZeroCopy(err)  # not a regular file
 
     try:
+        # Hopefully the whole file will be copied in a single call.
+        # sendfile() is called in a loop 'till EOF is reached (0 return)
+        # so a bufsize smaller than the actual file size should be OK
+        # also in case the src file content changes while being copied.
         blocksize = os.fstat(infd).st_size
     except OSError:
         blocksize = COPY_BUFSIZE
@@ -118,8 +124,8 @@ def _copyfileobj_sendfile(fsrc, fdst):
                 # between regular files (only sockets).
                 _HAS_SENDFILE = False
             if total_copied == 0:
-                # Immediately give up on first call.
-                # Probably one of the fds is not regular mmap(2)-like fd.
+                # Immediately give up on first call. Probably one of the
+                # fds is not regular mmap(2)-like fd.
                 raise _GiveupOnZeroCopy(err)
             else:
                 raise err from None

From a0fe7036b1f4f1266602d208b69454152b07233d Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 21:14:05 +0200
Subject: [PATCH 22/44] update doc

---
 Doc/library/shutil.rst | 8 ++++++++
 Doc/whatsnew/3.8.rst   | 4 ++++
 2 files changed, 12 insertions(+)

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 1527deb167f1e3..f2e55be1795d25 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -74,6 +74,8 @@ Directory and files operations
       Raise :exc:`SameFileError` instead of :exc:`Error`.  Since the former is
       a subclass of the latter, this change is backward compatible.
 
+   .. versionchanged:: 3.8
+      Uses high-performance :func:`os.sendfile` if available.
 
 .. exception:: SameFileError
 
@@ -163,6 +165,9 @@ Directory and files operations
       Added *follow_symlinks* argument.
       Now returns path to the newly created file.
 
+   .. versionchanged:: 3.8
+      Uses high-performance :func:`os.sendfile` if available.
+
 .. function:: copy2(src, dst, *, follow_symlinks=True)
 
    Identical to :func:`~shutil.copy` except that :func:`copy2`
@@ -185,6 +190,9 @@ Directory and files operations
       file system attributes too (currently Linux only).
       Now returns path to the newly created file.
 
+   .. versionchanged:: 3.8
+      Uses high-performance :func:`os.sendfile` if available.
+
 .. function:: ignore_patterns(\*patterns)
 
    This factory function creates a function that can be used as a callable for
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 9aad908f927f84..0227e75550f282 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -98,6 +98,10 @@ Optimizations
   first introduced in Python 3.4.  It offers better performance and smaller
   size compared to Protocol 3 available since Python 3.0.
 
+* :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2` use
+  high-performance :func:`os.sendfile` if available resulting in roughly a
+  20% speedup.
+
 Build and C API Changes
 =======================
 

From 72961478b6b3288f3b5c7bba91cb7cc5ab31f98a Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 21:35:31 +0200
Subject: [PATCH 23/44] update whatsnew doc

---
 Doc/whatsnew/3.8.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 0227e75550f282..123566a73393ef 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -100,7 +100,8 @@ Optimizations
 
 * :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2` use
   high-performance :func:`os.sendfile` if available resulting in roughly a
-  20% speedup.
+  20%/25% speedup of the copying operation and a considerably lower CPU cycles
+  consumption.
 
 Build and C API Changes
 =======================

From d0c3bbac48f615ace60b0adac2824e6477539c45 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 22:15:39 +0200
Subject: [PATCH 24/44] update doc

---
 Doc/whatsnew/3.8.rst | 2 +-
 Lib/shutil.py        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 123566a73393ef..3cb9bbf02b53f6 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -99,7 +99,7 @@ Optimizations
   size compared to Protocol 3 available since Python 3.0.
 
 * :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2` use
-  high-performance :func:`os.sendfile` if available resulting in roughly a
+  high-performance :func:`os.sendfile` is available resulting in roughly a
   20%/25% speedup of the copying operation and a considerably lower CPU cycles
   consumption.
 
diff --git a/Lib/shutil.py b/Lib/shutil.py
index 14dc7bed585b5d..6c510329aee699 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -141,7 +141,7 @@ def _copyfileobj2(fsrc, fdst):
     to leave copyfileobj() alone and not introduce any backward
     incompatibility.
     Possible incompatibilities by using sendfile() are:
-    - fdst cannot be opened in "a"(ppend) mode
+    - fdst cannot be open in "a"(ppend) mode
     - fdst offset doesn't get updated
     - fsrc and fdst may be opened in text mode
     - fsrc may be a BufferedReader (which hides unread data in a buffer),

From 2cafd805162d0880f1d2b5fd135084bdeb2aab38 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 22:23:48 +0200
Subject: [PATCH 25/44] catch Exception

---
 Lib/shutil.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 6c510329aee699..1d799c6a555495 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -98,7 +98,7 @@ def _copyfileobj_sendfile(fsrc, fdst):
     try:
         infd = fsrc.fileno()
         outfd = fdst.fileno()
-    except (AttributeError, io.UnsupportedOperation) as err:
+    except Exception as err:
         raise _GiveupOnZeroCopy(err)  # not a regular file
 
     try:
@@ -107,7 +107,7 @@ def _copyfileobj_sendfile(fsrc, fdst):
         # so a bufsize smaller than the actual file size should be OK
         # also in case the src file content changes while being copied.
         blocksize = os.fstat(infd).st_size
-    except OSError:
+    except Exception:
         blocksize = COPY_BUFSIZE
     else:
         if blocksize <= 0:

From bb2a75f50c194f3badbf4bd1ae12968fa20573bc Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Thu, 24 May 2018 22:24:50 +0200
Subject: [PATCH 26/44] remove unused import

---
 Lib/shutil.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 1d799c6a555495..92118b03679b30 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -10,7 +10,6 @@
 import fnmatch
 import collections
 import errno
-import io
 
 try:
     import zlib

From e5025dce0d4bbccc1eca9da400643333225052f0 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Fri, 25 May 2018 00:25:40 +0200
Subject: [PATCH 27/44] add test case for error on second sendfile() call

---
 Doc/whatsnew/3.8.rst    |  2 +-
 Lib/test/test_shutil.py | 25 ++++++++++++++++++++++---
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 3cb9bbf02b53f6..a08cf45cbad6e0 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -100,7 +100,7 @@ Optimizations
 
 * :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2` use
   high-performance :func:`os.sendfile` is available resulting in roughly a
-  20%/25% speedup of the copying operation and a considerably lower CPU cycles
+  20-25% speedup of the copying operation and a considerably lower CPU cycles
   consumption.
 
 Build and C API Changes
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 82fd3ef94c4ed6..3b2d5d14577200 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1938,8 +1938,7 @@ def test_empty_file(self):
             with open(dstname, "wb") as dst:
                 shutil._copyfileobj_sendfile(src, dst)
 
-        with open(dstname, "rb") as f:
-            self.assertEqual(f.read(), b"")
+        self.assertEqual(read_file(dstname, binary=True), b"")
 
     def test_unhandled_exception(self):
         with unittest.mock.patch('os.sendfile',
@@ -1957,6 +1956,25 @@ def test_exception_on_first_call(self):
                 with self.assertRaises(_GiveupOnZeroCopy):
                     shutil._copyfileobj_sendfile(src, dst)
 
+    def test_exception_on_second_call(self):
+        # ...but on subsequent calls we expect the exception to bubble up.
+        def sendfile(*args, **kwargs):
+            if not flag:
+                flag.append(None)
+                return orig_sendfile(*args, **kwargs)
+            else:
+                raise OSError(errno.EBADF, "yo")
+
+        flag = []
+        orig_sendfile = os.sendfile
+        with unittest.mock.patch('os.sendfile', create=True,
+                                 side_effect=sendfile):
+            with self.get_files() as (src, dst):
+                with self.assertRaises(OSError) as cm:
+                    shutil._copyfileobj_sendfile(src, dst)
+        assert flag
+        self.assertEqual(cm.exception.errno, errno.EBADF)
+
     def test_cant_get_size(self):
         # Emulate a case where src file size cannot be determined.
         # Internally bufsize will be set to a small value and
@@ -1965,6 +1983,7 @@ def test_cant_get_size(self):
             with self.get_files() as (src, dst):
                 shutil._copyfileobj_sendfile(src, dst)
                 assert m.called
+        self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA)
 
     def test_small_chunks(self):
         # Force internal file size detection to be smaller than the
@@ -2083,7 +2102,7 @@ def test_module_all_attribute(self):
                       'unregister_archive_format', 'get_unpack_formats',
                       'register_unpack_format', 'unregister_unpack_format',
                       'unpack_archive', 'ignore_patterns', 'chown', 'which',
-                      'get_terminal_size', 'SameFileError']
+                      'get_terminal_size', 'SameFileError', 'COPY_BUFSIZE']
         if hasattr(os, 'statvfs') or os.name == 'nt':
             target_api.append('disk_usage')
         self.assertEqual(set(shutil.__all__), set(target_api))

From a36a534433471995ebe03475482b944fd8accedf Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Fri, 25 May 2018 00:36:56 +0200
Subject: [PATCH 28/44] turn docstring into comment

---
 Lib/shutil.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 92118b03679b30..1a00ec22b0c006 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -135,18 +135,17 @@ def _copyfileobj_sendfile(fsrc, fdst):
             total_copied += sent
 
 def _copyfileobj2(fsrc, fdst):
-    """Copies 2 filesystem files by using zero-copy sendfile(2) syscall
-    (faster). This is used by copyfile(), copy() and copy2() in order
-    to leave copyfileobj() alone and not introduce any backward
-    incompatibility.
-    Possible incompatibilities by using sendfile() are:
-    - fdst cannot be open in "a"(ppend) mode
-    - fdst offset doesn't get updated
-    - fsrc and fdst may be opened in text mode
-    - fsrc may be a BufferedReader (which hides unread data in a buffer),
-      GzipFile (which decompresses data), HTTPResponse (which decodes
-      chunks), ...
-    """
+    # Copies 2 filesystem files by using zero-copy sendfile(2) syscall
+    # (faster).  This is used by copyfile(), copy() and copy2() in order
+    # to leave copyfileobj() alone and not introduce any unexpected
+    # breakage. Possible risks by using sendfile() in copyfileobj() are:
+    # - fdst cannot be open in "a"(ppend) mode
+    # - fsrc and fdst may be opened in text mode
+    # - fdst offset doesn't get updated
+    # - fsrc may be a BufferedReader (which hides unread data in a buffer),
+    #   GzipFile (which decompresses data), HTTPResponse (which decodes
+    #   chunks).
+    # - possibly others...
     if _HAS_SENDFILE:
         try:
             return _copyfileobj_sendfile(fsrc, fdst)

From e9da3fa87056e7926082afdff60f50828f8f7f51 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Fri, 25 May 2018 01:16:37 +0200
Subject: [PATCH 29/44] add one more test

---
 Lib/shutil.py           | 27 ++++++++++++---------------
 Lib/test/test_shutil.py | 18 ++++++++++++++++++
 2 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 1a00ec22b0c006..c528950646c2f5 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -90,8 +90,8 @@ def copyfileobj(fsrc, fdst, length=None):
         fdst.write(buf)
 
 def _copyfileobj_sendfile(fsrc, fdst):
-    """Copy data from one file object to another by using zero-copy
-    sendfile() method (faster).
+    """Copy data from one file object to another by using
+    high-performance sendfile() method.
     """
     global _HAS_SENDFILE
     try:
@@ -100,20 +100,17 @@ def _copyfileobj_sendfile(fsrc, fdst):
     except Exception as err:
         raise _GiveupOnZeroCopy(err)  # not a regular file
 
+    # Hopefully the whole file will be copied in a single call.
+    # sendfile() is called in a loop 'till EOF is reached (0 return)
+    # so a bufsize smaller or bigger than the actual file size
+    # should not make any difference.
     try:
-        # Hopefully the whole file will be copied in a single call.
-        # sendfile() is called in a loop 'till EOF is reached (0 return)
-        # so a bufsize smaller than the actual file size should be OK
-        # also in case the src file content changes while being copied.
-        blocksize = os.fstat(infd).st_size
+        blocksize = max(os.fstat(infd).st_size, COPY_BUFSIZE, 16 * 1024)
     except Exception:
-        blocksize = COPY_BUFSIZE
-    else:
-        if blocksize <= 0:
-            blocksize = COPY_BUFSIZE
+        blocksize = max(COPY_BUFSIZE, 16 * 1024)
 
     offset = 0
-    total_copied = 0
+    total = 0
     while True:
         try:
             sent = os.sendfile(outfd, infd, offset, blocksize)
@@ -122,9 +119,9 @@ def _copyfileobj_sendfile(fsrc, fdst):
                 # sendfile() on this platform does not support copies
                 # between regular files (only sockets).
                 _HAS_SENDFILE = False
-            if total_copied == 0:
+            if total == 0:
                 # Immediately give up on first call. Probably one of the
-                # fds is not regular mmap(2)-like fd.
+                # fds is not a regular mmap(2)-like fd.
                 raise _GiveupOnZeroCopy(err)
             else:
                 raise err from None
@@ -132,7 +129,7 @@ def _copyfileobj_sendfile(fsrc, fdst):
             if sent == 0:
                 break  # EOF
             offset += sent
-            total_copied += sent
+            total += sent
 
 def _copyfileobj2(fsrc, fdst):
     # Copies 2 filesystem files by using zero-copy sendfile(2) syscall
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 3b2d5d14577200..6acb149cdc6d78 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -2011,6 +2011,24 @@ def test_big_chunk(self):
                 assert m.called
         self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA)
 
+    def test_blocksize_arg(self):
+        with unittest.mock.patch('os.sendfile',
+                                 side_effect=ZeroDivisionError) as m:
+            self.assertRaises(ZeroDivisionError,
+                              shutil.copyfile, TESTFN, TESTFN2)
+            blocksize = m.call_args[0][3]
+            # Make sure file size and the block size arg passed to
+            # sendfile() are the same.
+            self.assertEqual(blocksize, os.path.getsize(TESTFN))
+            # ...unless we're dealing with a small file.
+            support.unlink(TESTFN2)
+            write_file(TESTFN2, b"hello", binary=True)
+            self.addCleanup(support.unlink, TESTFN2 + '3')
+            self.assertRaises(ZeroDivisionError,
+                              shutil.copyfile, TESTFN2, TESTFN2 + '3')
+            blocksize = m.call_args[0][3]
+            self.assertEqual(blocksize, shutil.COPY_BUFSIZE)
+
 
 class TermsizeTests(unittest.TestCase):
     def test_does_not_crash(self):

From 9fcc2e7e52bd9efcc8ae913f6ac7622a398b15f5 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Fri, 25 May 2018 01:27:09 +0200
Subject: [PATCH 30/44] update comment

---
 Lib/shutil.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index c528950646c2f5..3a21fdfc95d7c7 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -103,7 +103,8 @@ def _copyfileobj_sendfile(fsrc, fdst):
     # Hopefully the whole file will be copied in a single call.
     # sendfile() is called in a loop 'till EOF is reached (0 return)
     # so a bufsize smaller or bigger than the actual file size
-    # should not make any difference.
+    # should not make any difference, also in case the file content
+    # changes while being copied.
     try:
         blocksize = max(os.fstat(infd).st_size, COPY_BUFSIZE, 16 * 1024)
     except Exception:

From 4f32242aa0cd8c92d1cc4bc4e955a368f5326467 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Fri, 25 May 2018 01:42:53 +0200
Subject: [PATCH 31/44] add Misc/NEWS entry

---
 Doc/whatsnew/3.8.rst                                        | 6 +++---
 .../next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst   | 4 ++++
 2 files changed, 7 insertions(+), 3 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index a08cf45cbad6e0..0e7883eada477a 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -98,10 +98,10 @@ Optimizations
   first introduced in Python 3.4.  It offers better performance and smaller
   size compared to Protocol 3 available since Python 3.0.
 
-* :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2` use
-  high-performance :func:`os.sendfile` is available resulting in roughly a
+* :func:`shutil.copyfile`, :func:`shutil.copy` and :func:`shutil.copy2` use
+  high-performance :func:`os.sendfile` if available resulting in roughly a
   20-25% speedup of the copying operation and a considerably lower CPU cycles
-  consumption.
+  consumption.  (Contributed by Giampaolo Rodola' and desbma in :issue:`33639`)
 
 Build and C API Changes
 =======================
diff --git a/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst b/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst
new file mode 100644
index 00000000000000..45b795fb8276a7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst
@@ -0,0 +1,4 @@
+shutil.copyfile(), shutil.copy() and shutil.copy2() use high-performance
+os.sendfile() if available resulting in roughly a 20-25% speedup of the
+copying operation and a considerably lower CPU cycles consumption.
+(Contributed by Giampaolo Rodola' and desbma in 33639)

From 24ad25acb8ce01ead720a4af48eb71a59dd98e6b Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Fri, 25 May 2018 12:06:02 +0200
Subject: [PATCH 32/44] get rid of COPY_BUFSIZE; it belongs to another PR

---
 Lib/shutil.py           | 11 ++++-------
 Lib/test/test_shutil.py |  4 ++--
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 3a21fdfc95d7c7..6311440791cc69 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -43,7 +43,6 @@
     getgrnam = None
 
 _HAS_SENDFILE = hasattr(os, "sendfile")
-COPY_BUFSIZE = 16 * 1024
 
 __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
            "copytree", "move", "rmtree", "Error", "SpecialFileError",
@@ -52,7 +51,7 @@
            "get_unpack_formats", "register_unpack_format",
            "unregister_unpack_format", "unpack_archive",
            "ignore_patterns", "chown", "which", "get_terminal_size",
-           "SameFileError", "COPY_BUFSIZE"]
+           "SameFileError"]
            # disk_usage is added later, if available on the platform
 
 class Error(OSError):
@@ -79,10 +78,8 @@ class _GiveupOnZeroCopy(Exception):
     """Raised when os.sendfile() cannot be used for copying files."""
 
 
-def copyfileobj(fsrc, fdst, length=None):
+def copyfileobj(fsrc, fdst, length=16*1024):
     """copy data from file-like object fsrc to file-like object fdst"""
-    if length is None:
-        length = COPY_BUFSIZE
     while 1:
         buf = fsrc.read(length)
         if not buf:
@@ -106,9 +103,9 @@ def _copyfileobj_sendfile(fsrc, fdst):
     # should not make any difference, also in case the file content
     # changes while being copied.
     try:
-        blocksize = max(os.fstat(infd).st_size, COPY_BUFSIZE, 16 * 1024)
+        blocksize = max(os.fstat(infd).st_size, 10 * 1024)
     except Exception:
-        blocksize = max(COPY_BUFSIZE, 16 * 1024)
+        blocksize = 100 * 1024
 
     offset = 0
     total = 0
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 6acb149cdc6d78..340b3a906eaaa1 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -2027,7 +2027,7 @@ def test_blocksize_arg(self):
             self.assertRaises(ZeroDivisionError,
                               shutil.copyfile, TESTFN2, TESTFN2 + '3')
             blocksize = m.call_args[0][3]
-            self.assertEqual(blocksize, shutil.COPY_BUFSIZE)
+            self.assertEqual(blocksize, 10 * 1024)
 
 
 class TermsizeTests(unittest.TestCase):
@@ -2120,7 +2120,7 @@ def test_module_all_attribute(self):
                       'unregister_archive_format', 'get_unpack_formats',
                       'register_unpack_format', 'unregister_unpack_format',
                       'unpack_archive', 'ignore_patterns', 'chown', 'which',
-                      'get_terminal_size', 'SameFileError', 'COPY_BUFSIZE']
+                      'get_terminal_size', 'SameFileError']
         if hasattr(os, 'statvfs') or os.name == 'nt':
             target_api.append('disk_usage')
         self.assertEqual(set(shutil.__all__), set(target_api))

From 24d20e629cf0674c86c72e677f9c42749e2aa8fc Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Fri, 25 May 2018 12:09:07 +0200
Subject: [PATCH 33/44] update doc

---
 Doc/library/shutil.rst                                   | 9 ++++++---
 Doc/whatsnew/3.8.rst                                     | 7 ++++---
 .../Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst     | 5 +++--
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index f2e55be1795d25..fad3e5ce6b6fe5 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -75,7 +75,8 @@ Directory and files operations
       a subclass of the latter, this change is backward compatible.
 
    .. versionchanged:: 3.8
-      Uses high-performance :func:`os.sendfile` if available.
+      Uses high-performance :func:`os.sendfile` if available and supports
+      file-to-file copy (namely Linux).
 
 .. exception:: SameFileError
 
@@ -166,7 +167,8 @@ Directory and files operations
       Now returns path to the newly created file.
 
    .. versionchanged:: 3.8
-      Uses high-performance :func:`os.sendfile` if available.
+      Uses high-performance :func:`os.sendfile` if available and supports
+      file-to-file copy (namely Linux).
 
 .. function:: copy2(src, dst, *, follow_symlinks=True)
 
@@ -191,7 +193,8 @@ Directory and files operations
       Now returns path to the newly created file.
 
    .. versionchanged:: 3.8
-      Uses high-performance :func:`os.sendfile` if available.
+      Uses high-performance :func:`os.sendfile` if available and supports
+      file-to-file copy (namely Linux).
 
 .. function:: ignore_patterns(\*patterns)
 
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 0e7883eada477a..42ec12988a2d69 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -99,9 +99,10 @@ Optimizations
   size compared to Protocol 3 available since Python 3.0.
 
 * :func:`shutil.copyfile`, :func:`shutil.copy` and :func:`shutil.copy2` use
-  high-performance :func:`os.sendfile` if available resulting in roughly a
-  20-25% speedup of the copying operation and a considerably lower CPU cycles
-  consumption.  (Contributed by Giampaolo Rodola' and desbma in :issue:`33639`)
+  high-performance :func:`os.sendfile` if available and supports file-to-file
+  copy (namely Linux) resulting in roughly a 20-25% speedup of the copying
+  operation and a considerably lower CPU cycles consumption.
+  (Contributed by Giampaolo Rodola' and desbma in :issue:`33639`)
 
 Build and C API Changes
 =======================
diff --git a/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst b/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst
index 45b795fb8276a7..b4b4a0c32db5d0 100644
--- a/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst
+++ b/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst
@@ -1,4 +1,5 @@
 shutil.copyfile(), shutil.copy() and shutil.copy2() use high-performance
-os.sendfile() if available resulting in roughly a 20-25% speedup of the
-copying operation and a considerably lower CPU cycles consumption.
+os.sendfile() if available and supports file-to-file copy (namely Linux)
+resulting in roughly a 20-25% speedup of the copying operation and a
+considerably lower CPU cycles consumption.
 (Contributed by Giampaolo Rodola' and desbma in 33639)

From 8380b9bf580bb504d4f579498bbf78cf92405b20 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Fri, 25 May 2018 12:14:47 +0200
Subject: [PATCH 34/44] set min bufsize to either 8MB or 128MB

---
 Lib/shutil.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 6311440791cc69..c940e85aa11680 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -103,9 +103,9 @@ def _copyfileobj_sendfile(fsrc, fdst):
     # should not make any difference, also in case the file content
     # changes while being copied.
     try:
-        blocksize = max(os.fstat(infd).st_size, 10 * 1024)
+        blocksize = max(os.fstat(infd).st_size, 2 ** 23)  # min 8MB
     except Exception:
-        blocksize = 100 * 1024
+        blocksize = 2 ** 27  # 128MB
 
     offset = 0
     total = 0

From 081f9074f313e808d2d705ccedce5d0c2d0c71ca Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Fri, 25 May 2018 12:16:11 +0200
Subject: [PATCH 35/44] fix test

---
 Lib/test/test_shutil.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 340b3a906eaaa1..6000ffa6d59dab 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -2027,7 +2027,7 @@ def test_blocksize_arg(self):
             self.assertRaises(ZeroDivisionError,
                               shutil.copyfile, TESTFN2, TESTFN2 + '3')
             blocksize = m.call_args[0][3]
-            self.assertEqual(blocksize, 10 * 1024)
+            self.assertEqual(blocksize, 2 ** 23)
 
 
 class TermsizeTests(unittest.TestCase):

From 5c242f35bfe575b2ad8a0207edcfb00374994168 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sat, 26 May 2018 13:11:54 +0200
Subject: [PATCH 36/44] make sendfile() raise immediately if filesystem is full
 on first call

---
 Lib/shutil.py           | 7 +++++--
 Lib/test/test_shutil.py | 7 +++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index c940e85aa11680..a7bd2c215025a4 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -118,8 +118,11 @@ def _copyfileobj_sendfile(fsrc, fdst):
                 # between regular files (only sockets).
                 _HAS_SENDFILE = False
             if total == 0:
-                # Immediately give up on first call. Probably one of the
-                # fds is not a regular mmap(2)-like fd.
+                if err.errno == errno.ENOSPC:
+                    # Filesystem is full.
+                    raise
+                # Immediately give up on first call. Probably one
+                # of the fds is not a regular mmap(2)-like fd.
                 raise _GiveupOnZeroCopy(err)
             else:
                 raise err from None
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 6000ffa6d59dab..510da441c5e25b 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -2029,6 +2029,13 @@ def test_blocksize_arg(self):
             blocksize = m.call_args[0][3]
             self.assertEqual(blocksize, 2 ** 23)
 
+    def test_filesystem_full(self):
+        # Emulate a case where filesystem is full and sendfile() fails
+        # on first call.
+        with unittest.mock.patch('os.sendfile',
+                                 side_effect=OSError(errno.ENOSPC, "yo")):
+            self.assertRaises(OSError, shutil.copyfile, TESTFN, TESTFN2)
+
 
 class TermsizeTests(unittest.TestCase):
     def test_does_not_crash(self):

From 470dba8c8616c798f6a76a0146d4c1fa200feea1 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sat, 26 May 2018 13:26:14 +0200
Subject: [PATCH 37/44] use sendfile() only on Linux

---
 Doc/library/shutil.rst                        |  9 +++------
 Doc/whatsnew/3.8.rst                          |  7 +++----
 Lib/shutil.py                                 | 19 +++++++++++--------
 .../2018-05-25-01-41-00.bpo-33639.xkY4tq.rst  |  5 ++---
 4 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index fad3e5ce6b6fe5..21b940b420962b 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -75,8 +75,7 @@ Directory and files operations
       a subclass of the latter, this change is backward compatible.
 
    .. versionchanged:: 3.8
-      Uses high-performance :func:`os.sendfile` if available and supports
-      file-to-file copy (namely Linux).
+      Uses high-performance :func:`os.sendfile` (Linux only).
 
 .. exception:: SameFileError
 
@@ -167,8 +166,7 @@ Directory and files operations
       Now returns path to the newly created file.
 
    .. versionchanged:: 3.8
-      Uses high-performance :func:`os.sendfile` if available and supports
-      file-to-file copy (namely Linux).
+      Uses high-performance :func:`os.sendfile` (Linux only).
 
 .. function:: copy2(src, dst, *, follow_symlinks=True)
 
@@ -193,8 +191,7 @@ Directory and files operations
       Now returns path to the newly created file.
 
    .. versionchanged:: 3.8
-      Uses high-performance :func:`os.sendfile` if available and supports
-      file-to-file copy (namely Linux).
+      Uses high-performance :func:`os.sendfile` (Linux only).
 
 .. function:: ignore_patterns(\*patterns)
 
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 42ec12988a2d69..64bf919908e059 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -99,10 +99,9 @@ Optimizations
   size compared to Protocol 3 available since Python 3.0.
 
 * :func:`shutil.copyfile`, :func:`shutil.copy` and :func:`shutil.copy2` use
-  high-performance :func:`os.sendfile` if available and supports file-to-file
-  copy (namely Linux) resulting in roughly a 20-25% speedup of the copying
-  operation and a considerably lower CPU cycles consumption.
-  (Contributed by Giampaolo Rodola' and desbma in :issue:`33639`)
+  high-performance :func:`os.sendfile` on Linux resulting in roughly a 20-25%
+  speedup of the copying operation and a considerably lower CPU cycles
+  consumption. (Contributed by Giampaolo Rodola' and desbma in :issue:`33639`)
 
 Build and C API Changes
 =======================
diff --git a/Lib/shutil.py b/Lib/shutil.py
index a7bd2c215025a4..88b0400e0e8ab3 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -42,7 +42,8 @@
 except ImportError:
     getgrnam = None
 
-_HAS_SENDFILE = hasattr(os, "sendfile")
+_HAS_LINUX_SENDFILE = hasattr(os, "sendfile") and \
+    sys.platform.startswith("linux")
 
 __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
            "copytree", "move", "rmtree", "Error", "SpecialFileError",
@@ -87,10 +88,11 @@ def copyfileobj(fsrc, fdst, length=16*1024):
         fdst.write(buf)
 
 def _copyfileobj_sendfile(fsrc, fdst):
-    """Copy data from one file object to another by using
-    high-performance sendfile() method.
+    """Copy data from one regular file object to another by using
+    high-performance sendfile() method. Linux >= 2.6.33 is apparently
+    the only platform able to do this.
     """
-    global _HAS_SENDFILE
+    global _HAS_LINUX_SENDFILE
     try:
         infd = fsrc.fileno()
         outfd = fdst.fileno()
@@ -114,9 +116,10 @@ def _copyfileobj_sendfile(fsrc, fdst):
             sent = os.sendfile(outfd, infd, offset, blocksize)
         except OSError as err:
             if err.errno == errno.ENOTSOCK:
-                # sendfile() on this platform does not support copies
-                # between regular files (only sockets).
-                _HAS_SENDFILE = False
+                # sendfile() on this platform (probably Linux < 2.6.33)
+                # does not support copies between regular files (only
+                # sockets).
+                _HAS_LINUX_SENDFILE = False
             if total == 0:
                 if err.errno == errno.ENOSPC:
                     # Filesystem is full.
@@ -144,7 +147,7 @@ def _copyfileobj2(fsrc, fdst):
     #   GzipFile (which decompresses data), HTTPResponse (which decodes
     #   chunks).
     # - possibly others...
-    if _HAS_SENDFILE:
+    if _HAS_LINUX_SENDFILE:
         try:
             return _copyfileobj_sendfile(fsrc, fdst)
         except _GiveupOnZeroCopy:
diff --git a/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst b/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst
index b4b4a0c32db5d0..2660b8dbf87970 100644
--- a/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst
+++ b/Misc/NEWS.d/next/Library/2018-05-25-01-41-00.bpo-33639.xkY4tq.rst
@@ -1,5 +1,4 @@
 shutil.copyfile(), shutil.copy() and shutil.copy2() use high-performance
-os.sendfile() if available and supports file-to-file copy (namely Linux)
-resulting in roughly a 20-25% speedup of the copying operation and a
-considerably lower CPU cycles consumption.
+os.sendfile() on Linux resulting in roughly a 20-25% speedup of the copying
+operation and a considerably lower CPU cycles consumption.
 (Contributed by Giampaolo Rodola' and desbma in 33639)

From cbc79e1d8d3b2464f4f5bb27d426529b4debb07b Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sat, 26 May 2018 13:37:14 +0200
Subject: [PATCH 38/44] raise err from None

---
 Lib/shutil.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 88b0400e0e8ab3..1a9efaf971f1f5 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -121,14 +121,12 @@ def _copyfileobj_sendfile(fsrc, fdst):
                 # sockets).
                 _HAS_LINUX_SENDFILE = False
             if total == 0:
-                if err.errno == errno.ENOSPC:
-                    # Filesystem is full.
-                    raise
+                if err.errno == errno.ENOSPC:  # filesystem is full
+                    raise err from None
                 # Immediately give up on first call. Probably one
                 # of the fds is not a regular mmap(2)-like fd.
                 raise _GiveupOnZeroCopy(err)
-            else:
-                raise err from None
+            raise err from None
         else:
             if sent == 0:
                 break  # EOF

From 5ac745a0c48438d5d6fc41e1f07aa9f4940def1e Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 27 May 2018 17:33:06 +0200
Subject: [PATCH 39/44] remove 'total' variable; it's not necessary

---
 Lib/shutil.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 1a9efaf971f1f5..9a879f426920f8 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -110,7 +110,6 @@ def _copyfileobj_sendfile(fsrc, fdst):
         blocksize = 2 ** 27  # 128MB
 
     offset = 0
-    total = 0
     while True:
         try:
             sent = os.sendfile(outfd, infd, offset, blocksize)
@@ -120,7 +119,7 @@ def _copyfileobj_sendfile(fsrc, fdst):
                 # does not support copies between regular files (only
                 # sockets).
                 _HAS_LINUX_SENDFILE = False
-            if total == 0:
+            if offset == 0:
                 if err.errno == errno.ENOSPC:  # filesystem is full
                     raise err from None
                 # Immediately give up on first call. Probably one
@@ -131,7 +130,6 @@ def _copyfileobj_sendfile(fsrc, fdst):
             if sent == 0:
                 break  # EOF
             offset += sent
-            total += sent
 
 def _copyfileobj2(fsrc, fdst):
     # Copies 2 filesystem files by using zero-copy sendfile(2) syscall

From eb1edd3ffd468b69987e5141f9c27cda6c9236ca Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 27 May 2018 17:39:56 +0200
Subject: [PATCH 40/44] check out file position as an extra method to determine
 whether some (partial) data was copied

---
 Lib/shutil.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 9a879f426920f8..0ae2bea2f77176 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -119,13 +119,16 @@ def _copyfileobj_sendfile(fsrc, fdst):
                 # does not support copies between regular files (only
                 # sockets).
                 _HAS_LINUX_SENDFILE = False
-            if offset == 0:
+            # Try hard to determine if no data was copied and give
+            # up only in that case.
+            if offset == 0 or os.lseek(outfd, 0, os.SEEK_CUR) == 0:
                 if err.errno == errno.ENOSPC:  # filesystem is full
                     raise err from None
                 # Immediately give up on first call. Probably one
                 # of the fds is not a regular mmap(2)-like fd.
                 raise _GiveupOnZeroCopy(err)
-            raise err from None
+            else:
+                raise err from None
         else:
             if sent == 0:
                 break  # EOF

From f83a99073b87a0a881bf4f911ee641ea3d28556c Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 27 May 2018 19:02:29 +0200
Subject: [PATCH 41/44] refactoring

---
 Lib/shutil.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 0ae2bea2f77176..e44ad4d50bb322 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -119,16 +119,15 @@ def _copyfileobj_sendfile(fsrc, fdst):
                 # does not support copies between regular files (only
                 # sockets).
                 _HAS_LINUX_SENDFILE = False
-            # Try hard to determine if no data was copied and give
-            # up only in that case.
-            if offset == 0 or os.lseek(outfd, 0, os.SEEK_CUR) == 0:
-                if err.errno == errno.ENOSPC:  # filesystem is full
-                    raise err from None
-                # Immediately give up on first call. Probably one
-                # of the fds is not a regular mmap(2)-like fd.
-                raise _GiveupOnZeroCopy(err)
-            else:
+
+            if err.errno == errno.ENOSPC:  # filesystem is full
                 raise err from None
+
+            # Give up on first call and if no data was copied.
+            if offset == 0 and os.lseek(outfd, 0, os.SEEK_CUR) == 0:
+                raise _GiveupOnZeroCopy(err)
+
+            raise err from None
         else:
             if sent == 0:
                 break  # EOF

From 7905991764fcb76bdb6c38ec8ba9d62157644e82 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 27 May 2018 22:03:26 +0200
Subject: [PATCH 42/44] test refactoring: move utility function out of test
 class

---
 Lib/test/test_shutil.py | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 510da441c5e25b..2818871727a9e5 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -64,6 +64,23 @@ def write_file(path, content, binary=False):
     with open(path, 'wb' if binary else 'w') as fp:
         fp.write(content)
 
+def write_test_file(path, size):
+    """Create a test file with an arbitrary size and random text content."""
+    def chunks(total, step):
+        assert total >= step
+        while total > step:
+            yield step
+            total -= step
+        if total:
+            yield total
+
+    bufsize = min(size, 8192)
+    chunk = b"".join([random.choice(string.ascii_letters).encode()
+                      for i in range(bufsize)])
+    with open(path, 'wb') as f:
+        for csize in chunks(size, bufsize):
+            f.write(chunk)
+
 def read_file(path, binary=False):
     """Return contents from a file located at *path*.
 
@@ -1865,27 +1882,14 @@ def test_move_dir_caseinsensitive(self):
             os.rmdir(dst_dir)
 
 
-@unittest.skipIf(not SUPPORTS_SENDFILE, 'os.sendfile() not supported')
-class TestCopyFileObjSendfile(unittest.TestCase):
+class _CopyFileTest(object):
     FILESIZE = (10 * 1024 * 1024)  # 10 MiB
     BUFSIZE = 8192
     FILEDATA = b""
 
     @classmethod
     def setUpClass(cls):
-        def chunks(total, step):
-            assert total >= step
-            while total > step:
-                yield step
-                total -= step
-            if total:
-                yield total
-
-        chunk = b"".join([random.choice(string.ascii_letters).encode()
-                          for i in range(cls.BUFSIZE)])
-        with open(TESTFN, 'wb') as f:
-            for csize in chunks(cls.FILESIZE, cls.BUFSIZE):
-                f.write(chunk)
+        write_test_file(TESTFN, cls.FILESIZE)
         with open(TESTFN, 'rb') as f:
             cls.FILEDATA = f.read()
             assert len(cls.FILEDATA) == cls.FILESIZE
@@ -1903,6 +1907,10 @@ def get_files(self):
             with open(TESTFN2, "wb") as dst:
                 yield (src, dst)
 
+
+@unittest.skipIf(not SUPPORTS_SENDFILE, 'os.sendfile() not supported')
+class TestCopyFileObjSendfile(_CopyFileTest, unittest.TestCase):
+
     def test_regular_copy(self):
         with self.get_files() as (src, dst):
             shutil._copyfileobj_sendfile(src, dst)

From 6ac06c1555d4a5d8e806f35966baf1f427daa270 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 27 May 2018 22:05:30 +0200
Subject: [PATCH 43/44] add assert

---
 Lib/test/test_shutil.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 2818871727a9e5..0ad07288874248 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -80,6 +80,7 @@ def chunks(total, step):
     with open(path, 'wb') as f:
         for csize in chunks(size, bufsize):
             f.write(chunk)
+    assert os.path.getsize(path) == size
 
 def read_file(path, binary=False):
     """Return contents from a file located at *path*.

From 9373b4c0810281c5b8d46fade341a1a3784bb26d Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 27 May 2018 22:08:25 +0200
Subject: [PATCH 44/44] remove unused class attribute

---
 Lib/test/test_shutil.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 0ad07288874248..289ded945a8df6 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1885,7 +1885,6 @@ def test_move_dir_caseinsensitive(self):
 
 class _CopyFileTest(object):
     FILESIZE = (10 * 1024 * 1024)  # 10 MiB
-    BUFSIZE = 8192
     FILEDATA = b""
 
     @classmethod