diff --git a/docs/cli.rst b/docs/cli.rst index 974fe68..3ac5c4b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -474,6 +474,8 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite 207368,920 Kirkham St,37.760210314285,-122.47073935813 188702,1501 Evans Ave,37.7422086702947,-122.387293152263 +.. _cli_insert_csv_tsv: + Inserting CSV or TSV data ========================= @@ -483,14 +485,33 @@ If your data is in CSV format, you can insert it using the ``--csv`` option:: For tab-delimited data, use ``--tsv``:: - $ sqlite-utils insert dogs.db dogs docs.tsv --tsv + $ sqlite-utils insert dogs.db dogs dogs.tsv --tsv Data is expected to be encoded as Unicode UTF-8. If your data is an another character encoding you can specify it using the ``--encoding`` option:: - $ sqlite-utils insert dogs.db dogs docs.tsv --tsv --encoding=latin-1 + $ sqlite-utils insert dogs.db dogs dogs.tsv --tsv --encoding=latin-1 A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option. +.. _cli_insert_csv_tsv_delimiter: + +Alternative delimiters and quote characters +------------------------------------------- + +If your file uses a delimiter other than ``,`` or a quote character other than ``"`` you can specify them using the ``--delimiter`` and ``--quotechar`` options. + +Here's a CSV file that uses ``;`` for delimiters and the ``|`` symbol for quote characters:: + + name;description + Cleo;|Very fine; a friendly dog| + Pancakes;A local corgi + +You can import that using:: + + $ sqlite-utils insert dogs.db dogs dogs.csv --delimiter=";" --quotechar="|" + +Passing either ``--delimiter`` and ``--quotechar`` implies ``--csv``, so you can omit that option. + .. _cli_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0bc03fe..48bc737 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,4 +1,5 @@ import base64 +from tests.test_cli import test_query_memory_does_not_create_file import click import codecs from click_default_group import DefaultGroup @@ -601,6 +602,8 @@ def insert_upsert_options(fn): click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option("--tsv", is_flag=True, help="Expect TSV"), + click.option("--delimiter", help="Delimiter to use for CSV files"), + click.option("--quotechar", help="Quote character to use for CSV/TSV"), click.option( "--batch-size", type=int, default=100, help="Commit every X records" ), @@ -640,6 +643,8 @@ def insert_upsert_implementation( nl, csv, tsv, + delimiter, + quotechar, batch_size, alter, upsert, @@ -654,6 +659,8 @@ def insert_upsert_implementation( ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) + if delimiter or quotechar: + csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") if encoding and not (csv or tsv): @@ -665,7 +672,12 @@ def insert_upsert_implementation( if csv or tsv: dialect = "excel-tab" if tsv else "excel" with file_progress(json_file, silent=silent) as json_file: - reader = csv_std.reader(json_file, dialect=dialect) + csv_reader_args = {"dialect": dialect} + if delimiter: + csv_reader_args["delimiter"] = delimiter + if quotechar: + csv_reader_args["quotechar"] = quotechar + reader = csv_std.reader(json_file, **csv_reader_args) headers = next(reader) docs = (dict(zip(headers, row)) for row in reader) else: @@ -720,6 +732,8 @@ def insert( nl, csv, tsv, + delimiter, + quotechar, batch_size, alter, encoding, @@ -746,6 +760,8 @@ def insert( nl, csv, tsv, + delimiter, + quotechar, batch_size, alter=alter, upsert=False, @@ -773,6 +789,8 @@ def upsert( csv, tsv, batch_size, + delimiter, + quotechar, alter, not_null, default, @@ -794,6 +812,8 @@ def upsert( nl, csv, tsv, + delimiter, + quotechar, batch_size, alter=alter, upsert=True, diff --git a/tests/test_cli.py b/tests/test_cli.py index 381997b..52a80e9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -722,18 +722,28 @@ def test_insert_ignore(db_path, tmpdir): @pytest.mark.parametrize( - "content,option", - (("foo\tbar\tbaz\n1\t2\t3", "--tsv"), ("foo,bar,baz\n1,2,3", "--csv")), + "content,options", + [ + ("foo\tbar\tbaz\n1\t2\tcat,dog", ["--tsv"]), + ('foo,bar,baz\n1,2,"cat,dog"', ["--csv"]), + ('foo;bar;baz\n1;2;"cat,dog"', ["--csv", "--delimiter", ";"]), + # --delimiter implies --csv: + ('foo;bar;baz\n1;2;"cat,dog"', ["--delimiter", ";"]), + ("foo,bar,baz\n1,2,|cat,dog|", ["--csv", "--quotechar", "|"]), + ("foo,bar,baz\n1,2,|cat,dog|", ["--quotechar", "|"]), + ], ) -def test_insert_csv_tsv(content, option, db_path, tmpdir): +def test_insert_csv_tsv(content, options, db_path, tmpdir): db = Database(db_path) file_path = str(tmpdir / "insert.csv-tsv") open(file_path, "w").write(content) result = CliRunner().invoke( - cli.cli, ["insert", db_path, "data", file_path, option], catch_exceptions=False + cli.cli, + ["insert", db_path, "data", file_path] + options, + catch_exceptions=False, ) assert 0 == result.exit_code - assert [{"foo": "1", "bar": "2", "baz": "3"}] == list(db["data"].rows) + assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows) @pytest.mark.parametrize(