diff --git a/docs/changelog.rst b/docs/changelog.rst index 085d9d5..556e497 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _unreleased: + +Unreleased +---------- + +- JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) + .. _v4_0rc3: 4.0rc3 (2026-07-05) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 1eeaabf..49eba52 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -129,6 +129,8 @@ See :ref:`cli_query`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query @@ -198,6 +200,8 @@ See :ref:`cli_memory`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query @@ -435,6 +439,7 @@ See :ref:`cli_search`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -701,6 +706,7 @@ See :ref:`cli_tables`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --columns Include list of columns for each table --schema Include schema for each table --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -743,6 +749,7 @@ See :ref:`cli_views`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --columns Include list of columns for each view --schema Include schema for each view --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -790,6 +797,8 @@ See :ref:`cli_rows`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -830,6 +839,7 @@ See :ref:`cli_triggers`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -870,6 +880,7 @@ See :ref:`cli_indexes`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index 84a65d9..957ee64 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -111,6 +111,33 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +.. _cli_query_json_ascii: + +Unicode characters in JSON +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +JSON output includes unicode characters directly, without escaping them: + +.. code-block:: bash + + sqlite-utils dogs.db "select '日本語' as text" + +.. code-block:: output + + [{"text": "日本語"}] + +Use ``--ascii`` to escape non-ASCII characters as ``\uXXXX`` sequences instead: + +.. code-block:: bash + + sqlite-utils dogs.db "select '日本語' as text" --ascii + +.. code-block:: output + + [{"text": "\u65e5\u672c\u8a9e"}] + +The ``--ascii`` option can help on systems that cannot display or process UTF-8, such as Windows consoles using a legacy code page. On Windows, setting the ``PYTHONUTF8=1`` environment variable is an alternative fix for ``UnicodeEncodeError`` crashes when redirecting output to a file. + .. _cli_query_binary_json: Binary data in JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8918439..8ee2091 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -131,6 +131,13 @@ def output_options(fn): is_flag=True, default=False, ), + click.option( + "--ascii", + "ascii_", + help="Escape non-ASCII characters in JSON output as \\uXXXX", + is_flag=True, + default=False, + ), ) ): fn = decorator(fn) @@ -199,6 +206,7 @@ def tables( table, fmt, json_cols, + ascii_, columns, schema, load_extension, @@ -258,7 +266,7 @@ def tables( for row in _iter(): writer.writerow(row) else: - for line in output_rows(_iter(), headers, nl, arrays, json_cols): + for line in output_rows(_iter(), headers, nl, arrays, json_cols, ascii_): click.echo(line) @@ -296,6 +304,7 @@ def views( table, fmt, json_cols, + ascii_, columns, schema, load_extension, @@ -321,6 +330,7 @@ def views( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, columns=columns, schema=schema, load_extension=load_extension, @@ -1857,6 +1867,7 @@ def query( table, fmt, json_cols, + ascii_, raw, raw_lines, param, @@ -1895,6 +1906,7 @@ def query( nl, arrays, json_cols, + ascii_, ) @@ -1969,6 +1981,7 @@ def memory( table, fmt, json_cols, + ascii_, raw, raw_lines, param, @@ -2108,6 +2121,7 @@ def memory( nl, arrays, json_cols, + ascii_, ) @@ -2125,6 +2139,7 @@ def _execute_query( nl, arrays, json_cols, + ascii_, ): with db.conn: try: @@ -2167,7 +2182,7 @@ def _execute_query( for row in cursor: writer.writerow(row) else: - for line in output_rows(cursor, headers, nl, arrays, json_cols): + for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_): click.echo(line) @@ -2211,6 +2226,7 @@ def search( table, fmt, json_cols, + ascii_, load_extension, ): """Execute a full-text search against this table @@ -2257,6 +2273,7 @@ def search( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, param=[("query", q)], load_extension=load_extension, ) @@ -2317,6 +2334,7 @@ def rows( table, fmt, json_cols, + ascii_, load_extension, ): """Output all rows in the specified table @@ -2351,6 +2369,7 @@ def rows( fmt=fmt, param=param, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -2377,6 +2396,7 @@ def triggers( table, fmt, json_cols, + ascii_, load_extension, ): """Show triggers configured in this database @@ -2406,6 +2426,7 @@ def triggers( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -2434,6 +2455,7 @@ def indexes( table, fmt, json_cols, + ascii_, load_extension, ): """Show indexes for the whole database or specific tables @@ -2475,6 +2497,7 @@ def indexes( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -3112,7 +3135,7 @@ def convert( if multi: def preview(v): - return json.dumps(fn(v), default=repr) if v else v + return json.dumps(fn(v), default=repr, ensure_ascii=False) if v else v else: @@ -3458,7 +3481,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): @cli.command(name="plugins") def plugins_list(): "List installed plugins" - click.echo(json.dumps(get_plugins(), indent=2)) + click.echo(json.dumps(get_plugins(), indent=2, ensure_ascii=False)) ensure_plugins_loaded() @@ -3505,7 +3528,7 @@ FILE_COLUMNS = { } -def output_rows(iterator, headers, nl, arrays, json_cols): +def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False): # Duplicate column names would collide as dictionary keys, so rename # later occurrences id, id -> id, id_2 - CSV and table output keep # the original duplicate headers since they never build dictionaries @@ -3526,7 +3549,7 @@ def output_rows(iterator, headers, nl, arrays, json_cols): data = dict(zip(headers, data)) line = "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar=("[" if first else " ") if not nl else "", - serialized=json.dumps(data, default=json_binary), + serialized=json.dumps(data, default=json_binary, ensure_ascii=ascii_), maybecomma="," if (not nl and not is_last) else "", lastchar="]" if (is_last and not nl) else "", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index d11c42f..9e17969 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1047,6 +1047,34 @@ def test_query_json_with_json_cols(db_path): assert expected == result_rows.output.strip() +def test_query_json_unicode_not_escaped_by_default(db_path): + db = Database(db_path) + with db.conn: + db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text"]) + assert result.exit_code == 0 + assert result.output.strip() == '[{"id": 1, "text": "Japanese 日本語"}]' + # Same for --nl + result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text", "--nl"]) + assert result.exit_code == 0 + assert result.output.strip() == '{"id": 1, "text": "Japanese 日本語"}' + + +@pytest.mark.parametrize("command", ["query", "rows"]) +def test_query_json_ascii_option(db_path, command): + db = Database(db_path) + with db.conn: + db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + if command == "query": + args = [db_path, "select id, text from text", "--ascii"] + else: + args = ["rows", db_path, "text", "--ascii"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0 + expected = '[{"id": 1, "text": "Japanese ' + "\\u65e5\\u672c\\u8a9e" + '"}]' + assert result.output.strip() == expected + + @pytest.mark.parametrize( "content,is_binary", [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 443e72c..6c3f5c5 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -215,6 +215,25 @@ def test_convert_multi_dryrun(test_db_and_path): ) +def test_convert_multi_dryrun_unicode_not_escaped(test_db_and_path): + db_path = test_db_and_path[1] + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "{'text': 'Japanese 日本語'}", + "--dry-run", + "--multi", + ], + ) + assert result.exit_code == 0 + # Preview should match what jsonify_if_needed() would actually store + assert '{"text": "Japanese 日本語"}' in result.output + + @pytest.mark.parametrize("drop", (True, False)) def test_convert_output_column(test_db_and_path, drop): db, db_path = test_db_and_path