Fix Python3 stream encoding issues in _copy_data().
These issues were introduced in f8ccdc50
. Because we no longer convert
everything to an io.BytesIO in _encrypt() with _make_binary_stream(),
all io.StringIO()s which are passed through must take encoded strings
and io.BytesIO()s must take bytes (and there is actually a difference
with Python3).
Additionally, there appears to be an issue where the `outstream` passed
to _copy_data() is sometimes a _io.BufferedWriter and other times an
encodings.utf_8.StreamWriter. I am not sure yet where this problem was
introduced. For now, the workaround for dealing with the Python3
bytes/str io.BytesIO/io.StringIO problem also provides a workaround for
this issue.
* FIXES #88.
* FIXES #89 for Python3.
* FIXES #93.
fix/89-python3
parent
a7e772f10a
commit
43164fa7db
|
@ -167,7 +167,6 @@ def _copy_data(instream, outstream):
|
||||||
:param file outstream: The file descriptor of a tmpfile to write to.
|
:param file outstream: The file descriptor of a tmpfile to write to.
|
||||||
"""
|
"""
|
||||||
sent = 0
|
sent = 0
|
||||||
|
|
||||||
coder = find_encodings()
|
coder = find_encodings()
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
|
@ -179,17 +178,24 @@ def _copy_data(instream, outstream):
|
||||||
data = instream.read(1024)
|
data = instream.read(1024)
|
||||||
if len(data) == 0:
|
if len(data) == 0:
|
||||||
break
|
break
|
||||||
|
|
||||||
sent += len(data)
|
sent += len(data)
|
||||||
log.debug("Sending chunk %d bytes:\n%s"
|
log.debug("Sending chunk %d bytes:\n%s" % (sent, data))
|
||||||
% (sent, data))
|
|
||||||
|
if _py3k and isinstance(data, bytes):
|
||||||
|
encoded = coder.encode(data.decode(coder.name))[0]
|
||||||
|
elif _py3k and isinstance(data, str):
|
||||||
|
encoded = coder.encode(data)[0]
|
||||||
|
elif not _py3k and type(data) is not str:
|
||||||
|
encoded = coder.encode(data)[0]
|
||||||
|
else:
|
||||||
|
encoded = data
|
||||||
|
log.debug("Writing encoded data with type %s to outstream... "
|
||||||
|
% type(encoded))
|
||||||
|
|
||||||
|
if not _py3k:
|
||||||
try:
|
try:
|
||||||
outstream.write(data)
|
outstream.write(encoded)
|
||||||
except UnicodeError:
|
|
||||||
try:
|
|
||||||
outstream.write(coder.encode(data))
|
|
||||||
except IOError:
|
|
||||||
log.exception("Error sending data: Broken pipe")
|
|
||||||
break
|
|
||||||
except IOError as ioe:
|
except IOError as ioe:
|
||||||
# Can get 'broken pipe' errors even when all data was sent
|
# Can get 'broken pipe' errors even when all data was sent
|
||||||
if 'Broken pipe' in str(ioe):
|
if 'Broken pipe' in str(ioe):
|
||||||
|
@ -197,6 +203,48 @@ def _copy_data(instream, outstream):
|
||||||
else:
|
else:
|
||||||
log.exception(ioe)
|
log.exception(ioe)
|
||||||
break
|
break
|
||||||
|
else:
|
||||||
|
log.debug("Wrote data type <type 'str'> to outstream.")
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
outstream.write(bytes(encoded))
|
||||||
|
except TypeError as te:
|
||||||
|
# XXX FIXME This appears to happen because
|
||||||
|
# _threaded_copy_data() sometimes passes the `outstream` as an
|
||||||
|
# object with type <_io.BufferredWriter> and at other times
|
||||||
|
# with type <encodings.utf_8.StreamWriter>. We hit the
|
||||||
|
# following error when the `outstream` has type
|
||||||
|
# <encodings.utf_8.StreamWriter>.
|
||||||
|
if not "convert 'bytes' object to str implicitly" in str(te):
|
||||||
|
log.error(str(te))
|
||||||
|
try:
|
||||||
|
outstream.write(encoded.decode())
|
||||||
|
except TypeError as yate:
|
||||||
|
# We hit the "'str' does not support the buffer interface"
|
||||||
|
# error in Python3 when the `outstream` is an io.BytesIO and
|
||||||
|
# we try to write a str to it. We don't care about that
|
||||||
|
# error, we'll just try again with bytes.
|
||||||
|
if not "does not support the buffer interface" in str(yate):
|
||||||
|
log.error(str(yate))
|
||||||
|
except IOError as ioe:
|
||||||
|
# Can get 'broken pipe' errors even when all data was sent
|
||||||
|
if 'Broken pipe' in str(ioe):
|
||||||
|
log.error('Error sending data: Broken pipe')
|
||||||
|
else:
|
||||||
|
log.exception(ioe)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
log.debug("Wrote data type <class 'str'> outstream.")
|
||||||
|
except IOError as ioe:
|
||||||
|
# Can get 'broken pipe' errors even when all data was sent
|
||||||
|
if 'Broken pipe' in str(ioe):
|
||||||
|
log.error('Error sending data: Broken pipe')
|
||||||
|
else:
|
||||||
|
log.exception(ioe)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
log.debug("Wrote data type <class 'bytes'> to outstream.")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
outstream.close()
|
outstream.close()
|
||||||
except IOError as ioe:
|
except IOError as ioe:
|
||||||
|
|
Loading…
Reference in New Issue