diff --git a/docs/cli.rst b/docs/cli.rst index b7f0313..69d00a0 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -281,6 +281,12 @@ To read from standard input, use ``-`` as the filename - then use ``stdin`` or ` $ cat example.csv | sqlite-utils memory - "select * from stdin" +Incoming CSV data will be assumed to use ``utf-8``. If your data uses a different character encoding you can specify that with ``--encoding``:: + + $ cat example.csv | sqlite-utils memory - "select * from stdin" --encoding=latin-1 + +If you are joining across multiple CSV files they must all use the same encoding. + .. _cli_query_memory_attach: Joining in-memory data against existing databases using \-\-attach diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8fe37c8..eac76d4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1138,6 +1138,10 @@ def query( type=(str, str), help="Named :parameters for SQL query", ) +@click.option( + "--encoding", + help="Character encoding for CSV input, defaults to utf-8", +) @click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") @click.option( "--save", @@ -1159,6 +1163,7 @@ def memory( json_cols, raw, param, + encoding, dump, save, load_extension, @@ -1171,13 +1176,17 @@ def memory( sql = None for i, path in enumerate(paths): if path == "-": - csv_fp = sys.stdin + csv_fp = sys.stdin.buffer csv_table = "stdin" else: csv_path = pathlib.Path(path) csv_table = csv_path.stem csv_fp = csv_path.open() - db[csv_table].insert_all(csv_std.DictReader(csv_fp)) + + encoding = encoding or "utf-8-sig" + decoded_fp = io.TextIOWrapper(csv_fp, encoding=encoding) + + db[csv_table].insert_all(csv_std.DictReader(decoded_fp)) # Add convenient t / t1 / t2 views view_names = ["t{}".format(i + 1)] if i == 0: diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index f018837..f45abdf 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -1,6 +1,7 @@ from sqlite_utils import cli, Database from click.testing import CliRunner import pytest +import json def test_memory_basic(): @@ -34,6 +35,47 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): ) +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_csv_encoding(tmpdir, use_stdin): + latin1_csv = ( + b"date,name,latitude,longitude\n" b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n" + ) + input = None + if use_stdin: + input = latin1_csv + csv_path = "-" + sql_from = "stdin" + else: + csv_path = str(tmpdir / "test.csv") + with open(csv_path, "wb") as fp: + fp.write(latin1_csv) + sql_from = "test" + # Without --encoding should error: + assert ( + CliRunner() + .invoke( + cli.cli, + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + input=input, + ) + .exit_code + == 1 + ) + # With --encoding should work: + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin", "--encoding", "latin-1", "--nl"], + input=latin1_csv, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == { + "date": "2020-03-04", + "name": "S\u00e3o Paulo", + "latitude": "-23.561", + "longitude": "-46.645", + } + + @pytest.mark.parametrize("extra_args", ([], ["select 1"])) def test_memory_dump(extra_args): result = CliRunner().invoke(