Preserve carriage returns in quoted CSV/TSV fields (#895)

When reading CSV or TSV files through rows_from_file() or the CLI insert command, io.TextIOWrapper defaulted to universal-newline mode (newline=None), causing carriage returns (\r\n or lone \r) embedded inside quoted fields to be silently translated to \n before csv.reader could process them.

Setting newline='' in TextIOWrapper preserves carriage returns in quoted fields according to the Python standard library csv module specification.

Fixes #895

sqlite_utils/cli.py

@@ -1226,11 +1226,14 @@ def insert_upsert_implementation(
     # The --sniff option needs us to buffer the file to peek ahead
     sniff_buffer = None
     decoded_buffer = None
+    newline = "" if (csv or tsv) else None
     if sniff:
         sniff_buffer = io.BufferedReader(file, buffer_size=4096)
-        decoded_buffer = io.TextIOWrapper(sniff_buffer, encoding=encoding)
+        decoded_buffer = io.TextIOWrapper(
+            sniff_buffer, encoding=encoding, newline=newline
+        )
     else:
-        decoded_buffer = io.TextIOWrapper(file, encoding=encoding)
+        decoded_buffer = io.TextIOWrapper(file, encoding=encoding, newline=newline)
 
     tracker = None
     with file_progress(decoded_buffer, silent=silent) as decoded:

sqlite_utils/utils.py

@@ -346,7 +346,7 @@ def rows_from_file(
         return (json.loads(line) for line in fp if line.strip()), Format.NL
     elif format == Format.CSV:
         use_encoding: str = encoding or "utf-8-sig"
-        decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding)
+        decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding, newline="")
         if dialect is not None:
             reader = csv.DictReader(decoded_fp, dialect=dialect)
         else: