--encoding option for sqlite-utils memory, closes #280

Refs #272

docs/cli.rst

@@ -281,6 +281,12 @@ To read from standard input, use ``-`` as the filename - then use ``stdin`` or `
 
     $ cat example.csv | sqlite-utils memory - "select * from stdin"
 
+Incoming CSV data will be assumed to use ``utf-8``. If your data uses a different character encoding you can specify that with ``--encoding``::
+
+    $ cat example.csv | sqlite-utils memory - "select * from stdin" --encoding=latin-1
+
+If you are joining across multiple CSV files they must all use the same encoding.
+
 .. _cli_query_memory_attach:
 
 Joining in-memory data against existing databases using \-\-attach

sqlite_utils/cli.py

@@ -1138,6 +1138,10 @@ def query(
     type=(str, str),
     help="Named :parameters for SQL query",
 )
+@click.option(
+    "--encoding",
+    help="Character encoding for CSV input, defaults to utf-8",
+)
 @click.option("--dump", is_flag=True, help="Dump SQL for in-memory database")
 @click.option(
     "--save",
@@ -1159,6 +1163,7 @@ def memory(
     json_cols,
     raw,
     param,
+    encoding,
     dump,
     save,
     load_extension,
@@ -1171,13 +1176,17 @@ def memory(
         sql = None
     for i, path in enumerate(paths):
         if path == "-":
-            csv_fp = sys.stdin
+            csv_fp = sys.stdin.buffer
             csv_table = "stdin"
         else:
             csv_path = pathlib.Path(path)
             csv_table = csv_path.stem
             csv_fp = csv_path.open()
-        db[csv_table].insert_all(csv_std.DictReader(csv_fp))
+
+        encoding = encoding or "utf-8-sig"
+        decoded_fp = io.TextIOWrapper(csv_fp, encoding=encoding)
+
+        db[csv_table].insert_all(csv_std.DictReader(decoded_fp))
         # Add convenient t / t1 / t2 views
         view_names = ["t{}".format(i + 1)]
         if i == 0:

tests/test_cli_memory.py

@@ -1,6 +1,7 @@
 from sqlite_utils import cli, Database
 from click.testing import CliRunner
 import pytest
+import json
 
 
 def test_memory_basic():
@@ -34,6 +35,47 @@ def test_memory_csv(tmpdir, sql_from, use_stdin):
     )
 
 
+@pytest.mark.parametrize("use_stdin", (True, False))
+def test_memory_csv_encoding(tmpdir, use_stdin):
+    latin1_csv = (
+        b"date,name,latitude,longitude\n" b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n"
+    )
+    input = None
+    if use_stdin:
+        input = latin1_csv
+        csv_path = "-"
+        sql_from = "stdin"
+    else:
+        csv_path = str(tmpdir / "test.csv")
+        with open(csv_path, "wb") as fp:
+            fp.write(latin1_csv)
+        sql_from = "test"
+    # Without --encoding should error:
+    assert (
+        CliRunner()
+        .invoke(
+            cli.cli,
+            ["memory", csv_path, "select * from {}".format(sql_from), "--nl"],
+            input=input,
+        )
+        .exit_code
+        == 1
+    )
+    # With --encoding should work:
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-", "select * from stdin", "--encoding", "latin-1", "--nl"],
+        input=latin1_csv,
+    )
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.output.strip()) == {
+        "date": "2020-03-04",
+        "name": "S\u00e3o Paulo",
+        "latitude": "-23.561",
+        "longitude": "-46.645",
+    }
+
+
 @pytest.mark.parametrize("extra_args", ([], ["select 1"]))
 def test_memory_dump(extra_args):
     result = CliRunner().invoke(