mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-07-26 19:04:32 +02:00
JSON output no longer escapes non-ASCII characters, new --ascii option (#777)
Closes #625 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JaHan1NhaTRAxJ9LQtSLf9
This commit is contained in:
Simon Willison
GitHub
parent
d516e58543
commit
815b6a7d3d
6 changed files with 121 additions and 6 deletions
|
|
@ -4,6 +4,13 @@
|
|||
Changelog
|
||||
===========
|
||||
|
||||
.. _unreleased:
|
||||
|
||||
Unreleased
|
||||
----------
|
||||
|
||||
- JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`)
|
||||
|
||||
.. _v4_0rc3:
|
||||
|
||||
4.0rc3 (2026-07-05)
|
||||
|
|
|
|||
|
|
@ -129,6 +129,8 @@ See :ref:`cli_query`.
|
|||
simple_outline, textile, tsv, unsafehtml, youtrack
|
||||
--json-cols Detect JSON cols and output them as JSON, not
|
||||
escaped strings
|
||||
--ascii Escape non-ASCII characters in JSON output as
|
||||
\uXXXX
|
||||
-r, --raw Raw output, first column of first row
|
||||
--raw-lines Raw output, first column of each row
|
||||
-p, --param <TEXT TEXT>... Named :parameters for SQL query
|
||||
|
|
@ -198,6 +200,8 @@ See :ref:`cli_memory`.
|
|||
simple_outline, textile, tsv, unsafehtml, youtrack
|
||||
--json-cols Detect JSON cols and output them as JSON, not
|
||||
escaped strings
|
||||
--ascii Escape non-ASCII characters in JSON output as
|
||||
\uXXXX
|
||||
-r, --raw Raw output, first column of first row
|
||||
--raw-lines Raw output, first column of each row
|
||||
-p, --param <TEXT TEXT>... Named :parameters for SQL query
|
||||
|
|
@ -435,6 +439,7 @@ See :ref:`cli_search`.
|
|||
youtrack
|
||||
--json-cols Detect JSON cols and output them as JSON, not escaped
|
||||
strings
|
||||
--ascii Escape non-ASCII characters in JSON output as \uXXXX
|
||||
--load-extension TEXT Path to SQLite extension, with optional :entrypoint
|
||||
-h, --help Show this message and exit.
|
||||
|
||||
|
|
@ -701,6 +706,7 @@ See :ref:`cli_tables`.
|
|||
youtrack
|
||||
--json-cols Detect JSON cols and output them as JSON, not escaped
|
||||
strings
|
||||
--ascii Escape non-ASCII characters in JSON output as \uXXXX
|
||||
--columns Include list of columns for each table
|
||||
--schema Include schema for each table
|
||||
--load-extension TEXT Path to SQLite extension, with optional :entrypoint
|
||||
|
|
@ -743,6 +749,7 @@ See :ref:`cli_views`.
|
|||
youtrack
|
||||
--json-cols Detect JSON cols and output them as JSON, not escaped
|
||||
strings
|
||||
--ascii Escape non-ASCII characters in JSON output as \uXXXX
|
||||
--columns Include list of columns for each view
|
||||
--schema Include schema for each view
|
||||
--load-extension TEXT Path to SQLite extension, with optional :entrypoint
|
||||
|
|
@ -790,6 +797,8 @@ See :ref:`cli_rows`.
|
|||
simple_outline, textile, tsv, unsafehtml, youtrack
|
||||
--json-cols Detect JSON cols and output them as JSON, not
|
||||
escaped strings
|
||||
--ascii Escape non-ASCII characters in JSON output as
|
||||
\uXXXX
|
||||
--load-extension TEXT Path to SQLite extension, with optional
|
||||
:entrypoint
|
||||
-h, --help Show this message and exit.
|
||||
|
|
@ -830,6 +839,7 @@ See :ref:`cli_triggers`.
|
|||
youtrack
|
||||
--json-cols Detect JSON cols and output them as JSON, not escaped
|
||||
strings
|
||||
--ascii Escape non-ASCII characters in JSON output as \uXXXX
|
||||
--load-extension TEXT Path to SQLite extension, with optional :entrypoint
|
||||
-h, --help Show this message and exit.
|
||||
|
||||
|
|
@ -870,6 +880,7 @@ See :ref:`cli_indexes`.
|
|||
youtrack
|
||||
--json-cols Detect JSON cols and output them as JSON, not escaped
|
||||
strings
|
||||
--ascii Escape non-ASCII characters in JSON output as \uXXXX
|
||||
--load-extension TEXT Path to SQLite extension, with optional :entrypoint
|
||||
-h, --help Show this message and exit.
|
||||
|
||||
|
|
|
|||
27
docs/cli.rst
27
docs/cli.rst
|
|
@ -111,6 +111,33 @@ If you want to pretty-print the output further, you can pipe it through ``python
|
|||
}
|
||||
]
|
||||
|
||||
.. _cli_query_json_ascii:
|
||||
|
||||
Unicode characters in JSON
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
JSON output includes unicode characters directly, without escaping them:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
sqlite-utils dogs.db "select '日本語' as text"
|
||||
|
||||
.. code-block:: output
|
||||
|
||||
[{"text": "日本語"}]
|
||||
|
||||
Use ``--ascii`` to escape non-ASCII characters as ``\uXXXX`` sequences instead:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
sqlite-utils dogs.db "select '日本語' as text" --ascii
|
||||
|
||||
.. code-block:: output
|
||||
|
||||
[{"text": "\u65e5\u672c\u8a9e"}]
|
||||
|
||||
The ``--ascii`` option can help on systems that cannot display or process UTF-8, such as Windows consoles using a legacy code page. On Windows, setting the ``PYTHONUTF8=1`` environment variable is an alternative fix for ``UnicodeEncodeError`` crashes when redirecting output to a file.
|
||||
|
||||
.. _cli_query_binary_json:
|
||||
|
||||
Binary data in JSON
|
||||
|
|
|
|||
|
|
@ -131,6 +131,13 @@ def output_options(fn):
|
|||
is_flag=True,
|
||||
default=False,
|
||||
),
|
||||
click.option(
|
||||
"--ascii",
|
||||
"ascii_",
|
||||
help="Escape non-ASCII characters in JSON output as \\uXXXX",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
),
|
||||
)
|
||||
):
|
||||
fn = decorator(fn)
|
||||
|
|
@ -199,6 +206,7 @@ def tables(
|
|||
table,
|
||||
fmt,
|
||||
json_cols,
|
||||
ascii_,
|
||||
columns,
|
||||
schema,
|
||||
load_extension,
|
||||
|
|
@ -258,7 +266,7 @@ def tables(
|
|||
for row in _iter():
|
||||
writer.writerow(row)
|
||||
else:
|
||||
for line in output_rows(_iter(), headers, nl, arrays, json_cols):
|
||||
for line in output_rows(_iter(), headers, nl, arrays, json_cols, ascii_):
|
||||
click.echo(line)
|
||||
|
||||
|
||||
|
|
@ -296,6 +304,7 @@ def views(
|
|||
table,
|
||||
fmt,
|
||||
json_cols,
|
||||
ascii_,
|
||||
columns,
|
||||
schema,
|
||||
load_extension,
|
||||
|
|
@ -321,6 +330,7 @@ def views(
|
|||
table=table,
|
||||
fmt=fmt,
|
||||
json_cols=json_cols,
|
||||
ascii_=ascii_,
|
||||
columns=columns,
|
||||
schema=schema,
|
||||
load_extension=load_extension,
|
||||
|
|
@ -1857,6 +1867,7 @@ def query(
|
|||
table,
|
||||
fmt,
|
||||
json_cols,
|
||||
ascii_,
|
||||
raw,
|
||||
raw_lines,
|
||||
param,
|
||||
|
|
@ -1895,6 +1906,7 @@ def query(
|
|||
nl,
|
||||
arrays,
|
||||
json_cols,
|
||||
ascii_,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1969,6 +1981,7 @@ def memory(
|
|||
table,
|
||||
fmt,
|
||||
json_cols,
|
||||
ascii_,
|
||||
raw,
|
||||
raw_lines,
|
||||
param,
|
||||
|
|
@ -2108,6 +2121,7 @@ def memory(
|
|||
nl,
|
||||
arrays,
|
||||
json_cols,
|
||||
ascii_,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -2125,6 +2139,7 @@ def _execute_query(
|
|||
nl,
|
||||
arrays,
|
||||
json_cols,
|
||||
ascii_,
|
||||
):
|
||||
with db.conn:
|
||||
try:
|
||||
|
|
@ -2167,7 +2182,7 @@ def _execute_query(
|
|||
for row in cursor:
|
||||
writer.writerow(row)
|
||||
else:
|
||||
for line in output_rows(cursor, headers, nl, arrays, json_cols):
|
||||
for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_):
|
||||
click.echo(line)
|
||||
|
||||
|
||||
|
|
@ -2211,6 +2226,7 @@ def search(
|
|||
table,
|
||||
fmt,
|
||||
json_cols,
|
||||
ascii_,
|
||||
load_extension,
|
||||
):
|
||||
"""Execute a full-text search against this table
|
||||
|
|
@ -2257,6 +2273,7 @@ def search(
|
|||
table=table,
|
||||
fmt=fmt,
|
||||
json_cols=json_cols,
|
||||
ascii_=ascii_,
|
||||
param=[("query", q)],
|
||||
load_extension=load_extension,
|
||||
)
|
||||
|
|
@ -2317,6 +2334,7 @@ def rows(
|
|||
table,
|
||||
fmt,
|
||||
json_cols,
|
||||
ascii_,
|
||||
load_extension,
|
||||
):
|
||||
"""Output all rows in the specified table
|
||||
|
|
@ -2351,6 +2369,7 @@ def rows(
|
|||
fmt=fmt,
|
||||
param=param,
|
||||
json_cols=json_cols,
|
||||
ascii_=ascii_,
|
||||
load_extension=load_extension,
|
||||
)
|
||||
|
||||
|
|
@ -2377,6 +2396,7 @@ def triggers(
|
|||
table,
|
||||
fmt,
|
||||
json_cols,
|
||||
ascii_,
|
||||
load_extension,
|
||||
):
|
||||
"""Show triggers configured in this database
|
||||
|
|
@ -2406,6 +2426,7 @@ def triggers(
|
|||
table=table,
|
||||
fmt=fmt,
|
||||
json_cols=json_cols,
|
||||
ascii_=ascii_,
|
||||
load_extension=load_extension,
|
||||
)
|
||||
|
||||
|
|
@ -2434,6 +2455,7 @@ def indexes(
|
|||
table,
|
||||
fmt,
|
||||
json_cols,
|
||||
ascii_,
|
||||
load_extension,
|
||||
):
|
||||
"""Show indexes for the whole database or specific tables
|
||||
|
|
@ -2475,6 +2497,7 @@ def indexes(
|
|||
table=table,
|
||||
fmt=fmt,
|
||||
json_cols=json_cols,
|
||||
ascii_=ascii_,
|
||||
load_extension=load_extension,
|
||||
)
|
||||
|
||||
|
|
@ -3112,7 +3135,7 @@ def convert(
|
|||
if multi:
|
||||
|
||||
def preview(v):
|
||||
return json.dumps(fn(v), default=repr) if v else v
|
||||
return json.dumps(fn(v), default=repr, ensure_ascii=False) if v else v
|
||||
|
||||
else:
|
||||
|
||||
|
|
@ -3458,7 +3481,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose):
|
|||
@cli.command(name="plugins")
|
||||
def plugins_list():
|
||||
"List installed plugins"
|
||||
click.echo(json.dumps(get_plugins(), indent=2))
|
||||
click.echo(json.dumps(get_plugins(), indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
ensure_plugins_loaded()
|
||||
|
|
@ -3505,7 +3528,7 @@ FILE_COLUMNS = {
|
|||
}
|
||||
|
||||
|
||||
def output_rows(iterator, headers, nl, arrays, json_cols):
|
||||
def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False):
|
||||
# Duplicate column names would collide as dictionary keys, so rename
|
||||
# later occurrences id, id -> id, id_2 - CSV and table output keep
|
||||
# the original duplicate headers since they never build dictionaries
|
||||
|
|
@ -3526,7 +3549,7 @@ def output_rows(iterator, headers, nl, arrays, json_cols):
|
|||
data = dict(zip(headers, data))
|
||||
line = "{firstchar}{serialized}{maybecomma}{lastchar}".format(
|
||||
firstchar=("[" if first else " ") if not nl else "",
|
||||
serialized=json.dumps(data, default=json_binary),
|
||||
serialized=json.dumps(data, default=json_binary, ensure_ascii=ascii_),
|
||||
maybecomma="," if (not nl and not is_last) else "",
|
||||
lastchar="]" if (is_last and not nl) else "",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1047,6 +1047,34 @@ def test_query_json_with_json_cols(db_path):
|
|||
assert expected == result_rows.output.strip()
|
||||
|
||||
|
||||
def test_query_json_unicode_not_escaped_by_default(db_path):
|
||||
db = Database(db_path)
|
||||
with db.conn:
|
||||
db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id")
|
||||
result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output.strip() == '[{"id": 1, "text": "Japanese 日本語"}]'
|
||||
# Same for --nl
|
||||
result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text", "--nl"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output.strip() == '{"id": 1, "text": "Japanese 日本語"}'
|
||||
|
||||
|
||||
@pytest.mark.parametrize("command", ["query", "rows"])
|
||||
def test_query_json_ascii_option(db_path, command):
|
||||
db = Database(db_path)
|
||||
with db.conn:
|
||||
db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id")
|
||||
if command == "query":
|
||||
args = [db_path, "select id, text from text", "--ascii"]
|
||||
else:
|
||||
args = ["rows", db_path, "text", "--ascii"]
|
||||
result = CliRunner().invoke(cli.cli, args)
|
||||
assert result.exit_code == 0
|
||||
expected = '[{"id": 1, "text": "Japanese ' + "\\u65e5\\u672c\\u8a9e" + '"}]'
|
||||
assert result.output.strip() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content,is_binary",
|
||||
[(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)],
|
||||
|
|
|
|||
|
|
@ -215,6 +215,25 @@ def test_convert_multi_dryrun(test_db_and_path):
|
|||
)
|
||||
|
||||
|
||||
def test_convert_multi_dryrun_unicode_not_escaped(test_db_and_path):
|
||||
db_path = test_db_and_path[1]
|
||||
result = CliRunner().invoke(
|
||||
cli.cli,
|
||||
[
|
||||
"convert",
|
||||
db_path,
|
||||
"example",
|
||||
"dt",
|
||||
"{'text': 'Japanese 日本語'}",
|
||||
"--dry-run",
|
||||
"--multi",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
# Preview should match what jsonify_if_needed() would actually store
|
||||
assert '{"text": "Japanese 日本語"}' in result.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop", (True, False))
|
||||
def test_convert_output_column(test_db_and_path, drop):
|
||||
db, db_path = test_db_and_path
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue