JSON output no longer escapes non-ASCII characters, new --ascii option (#777)

Closes #625

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JaHan1NhaTRAxJ9LQtSLf9

docs/changelog.rst

@@ -4,6 +4,13 @@
  Changelog
 ===========
 
+.. _unreleased:
+
+Unreleased
+----------
+
+- JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`)
+
 .. _v4_0rc3:
 
 4.0rc3 (2026-07-05)

docs/cli-reference.rst

@@ -129,6 +129,8 @@ See :ref:`cli_query`.
                                   simple_outline, textile, tsv, unsafehtml, youtrack
       --json-cols                 Detect JSON cols and output them as JSON, not
                                   escaped strings
+      --ascii                     Escape non-ASCII characters in JSON output as
+                                  \uXXXX
       -r, --raw                   Raw output, first column of first row
       --raw-lines                 Raw output, first column of each row
       -p, --param <TEXT TEXT>...  Named :parameters for SQL query
@@ -198,6 +200,8 @@ See :ref:`cli_memory`.
                                   simple_outline, textile, tsv, unsafehtml, youtrack
       --json-cols                 Detect JSON cols and output them as JSON, not
                                   escaped strings
+      --ascii                     Escape non-ASCII characters in JSON output as
+                                  \uXXXX
       -r, --raw                   Raw output, first column of first row
       --raw-lines                 Raw output, first column of each row
       -p, --param <TEXT TEXT>...  Named :parameters for SQL query
@@ -435,6 +439,7 @@ See :ref:`cli_search`.
                              youtrack
       --json-cols            Detect JSON cols and output them as JSON, not escaped
                              strings
+      --ascii                Escape non-ASCII characters in JSON output as \uXXXX
       --load-extension TEXT  Path to SQLite extension, with optional :entrypoint
       -h, --help             Show this message and exit.
 
@@ -701,6 +706,7 @@ See :ref:`cli_tables`.
                              youtrack
       --json-cols            Detect JSON cols and output them as JSON, not escaped
                              strings
+      --ascii                Escape non-ASCII characters in JSON output as \uXXXX
       --columns              Include list of columns for each table
       --schema               Include schema for each table
       --load-extension TEXT  Path to SQLite extension, with optional :entrypoint
@@ -743,6 +749,7 @@ See :ref:`cli_views`.
                              youtrack
       --json-cols            Detect JSON cols and output them as JSON, not escaped
                              strings
+      --ascii                Escape non-ASCII characters in JSON output as \uXXXX
       --columns              Include list of columns for each view
       --schema               Include schema for each view
       --load-extension TEXT  Path to SQLite extension, with optional :entrypoint
@@ -790,6 +797,8 @@ See :ref:`cli_rows`.
                                   simple_outline, textile, tsv, unsafehtml, youtrack
       --json-cols                 Detect JSON cols and output them as JSON, not
                                   escaped strings
+      --ascii                     Escape non-ASCII characters in JSON output as
+                                  \uXXXX
       --load-extension TEXT       Path to SQLite extension, with optional
                                   :entrypoint
       -h, --help                  Show this message and exit.
@@ -830,6 +839,7 @@ See :ref:`cli_triggers`.
                              youtrack
       --json-cols            Detect JSON cols and output them as JSON, not escaped
                              strings
+      --ascii                Escape non-ASCII characters in JSON output as \uXXXX
       --load-extension TEXT  Path to SQLite extension, with optional :entrypoint
       -h, --help             Show this message and exit.
 
@@ -870,6 +880,7 @@ See :ref:`cli_indexes`.
                              youtrack
       --json-cols            Detect JSON cols and output them as JSON, not escaped
                              strings
+      --ascii                Escape non-ASCII characters in JSON output as \uXXXX
       --load-extension TEXT  Path to SQLite extension, with optional :entrypoint
       -h, --help             Show this message and exit.
 

docs/cli.rst

@@ -111,6 +111,33 @@ If you want to pretty-print the output further, you can pipe it through ``python
         }
     ]
 
+.. _cli_query_json_ascii:
+
+Unicode characters in JSON
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+JSON output includes unicode characters directly, without escaping them:
+
+.. code-block:: bash
+
+    sqlite-utils dogs.db "select '日本語' as text"
+
+.. code-block:: output
+
+    [{"text": "日本語"}]
+
+Use ``--ascii`` to escape non-ASCII characters as ``\uXXXX`` sequences instead:
+
+.. code-block:: bash
+
+    sqlite-utils dogs.db "select '日本語' as text" --ascii
+
+.. code-block:: output
+
+    [{"text": "\u65e5\u672c\u8a9e"}]
+
+The ``--ascii`` option can help on systems that cannot display or process UTF-8, such as Windows consoles using a legacy code page. On Windows, setting the ``PYTHONUTF8=1`` environment variable is an alternative fix for ``UnicodeEncodeError`` crashes when redirecting output to a file.
+
 .. _cli_query_binary_json:
 
 Binary data in JSON

sqlite_utils/cli.py

@@ -131,6 +131,13 @@ def output_options(fn):
                 is_flag=True,
                 default=False,
             ),
+            click.option(
+                "--ascii",
+                "ascii_",
+                help="Escape non-ASCII characters in JSON output as \\uXXXX",
+                is_flag=True,
+                default=False,
+            ),
         )
     ):
         fn = decorator(fn)
@@ -199,6 +206,7 @@ def tables(
     table,
     fmt,
     json_cols,
+    ascii_,
     columns,
     schema,
     load_extension,
@@ -258,7 +266,7 @@ def tables(
         for row in _iter():
             writer.writerow(row)
     else:
-        for line in output_rows(_iter(), headers, nl, arrays, json_cols):
+        for line in output_rows(_iter(), headers, nl, arrays, json_cols, ascii_):
             click.echo(line)
 
 
@@ -296,6 +304,7 @@ def views(
     table,
     fmt,
     json_cols,
+    ascii_,
     columns,
     schema,
     load_extension,
@@ -321,6 +330,7 @@ def views(
         table=table,
         fmt=fmt,
         json_cols=json_cols,
+        ascii_=ascii_,
         columns=columns,
         schema=schema,
         load_extension=load_extension,
@@ -1857,6 +1867,7 @@ def query(
     table,
     fmt,
     json_cols,
+    ascii_,
     raw,
     raw_lines,
     param,
@@ -1895,6 +1906,7 @@ def query(
         nl,
         arrays,
         json_cols,
+        ascii_,
     )
 
 
@@ -1969,6 +1981,7 @@ def memory(
     table,
     fmt,
     json_cols,
+    ascii_,
     raw,
     raw_lines,
     param,
@@ -2108,6 +2121,7 @@ def memory(
         nl,
         arrays,
         json_cols,
+        ascii_,
     )
 
 
@@ -2125,6 +2139,7 @@ def _execute_query(
     nl,
     arrays,
     json_cols,
+    ascii_,
 ):
     with db.conn:
         try:
@@ -2167,7 +2182,7 @@ def _execute_query(
             for row in cursor:
                 writer.writerow(row)
         else:
-            for line in output_rows(cursor, headers, nl, arrays, json_cols):
+            for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_):
                 click.echo(line)
 
 
@@ -2211,6 +2226,7 @@ def search(
     table,
     fmt,
     json_cols,
+    ascii_,
     load_extension,
 ):
     """Execute a full-text search against this table
@@ -2257,6 +2273,7 @@ def search(
             table=table,
             fmt=fmt,
             json_cols=json_cols,
+            ascii_=ascii_,
             param=[("query", q)],
             load_extension=load_extension,
         )
@@ -2317,6 +2334,7 @@ def rows(
     table,
     fmt,
     json_cols,
+    ascii_,
     load_extension,
 ):
     """Output all rows in the specified table
@@ -2351,6 +2369,7 @@ def rows(
         fmt=fmt,
         param=param,
         json_cols=json_cols,
+        ascii_=ascii_,
         load_extension=load_extension,
     )
 
@@ -2377,6 +2396,7 @@ def triggers(
     table,
     fmt,
     json_cols,
+    ascii_,
     load_extension,
 ):
     """Show triggers configured in this database
@@ -2406,6 +2426,7 @@ def triggers(
         table=table,
         fmt=fmt,
         json_cols=json_cols,
+        ascii_=ascii_,
         load_extension=load_extension,
     )
 
@@ -2434,6 +2455,7 @@ def indexes(
     table,
     fmt,
     json_cols,
+    ascii_,
     load_extension,
 ):
     """Show indexes for the whole database or specific tables
@@ -2475,6 +2497,7 @@ def indexes(
         table=table,
         fmt=fmt,
         json_cols=json_cols,
+        ascii_=ascii_,
         load_extension=load_extension,
     )
 
@@ -3112,7 +3135,7 @@ def convert(
         if multi:
 
             def preview(v):
-                return json.dumps(fn(v), default=repr) if v else v
+                return json.dumps(fn(v), default=repr, ensure_ascii=False) if v else v
 
         else:
 
@@ -3458,7 +3481,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose):
 @cli.command(name="plugins")
 def plugins_list():
     "List installed plugins"
-    click.echo(json.dumps(get_plugins(), indent=2))
+    click.echo(json.dumps(get_plugins(), indent=2, ensure_ascii=False))
 
 
 ensure_plugins_loaded()
@@ -3505,7 +3528,7 @@ FILE_COLUMNS = {
 }
 
 
-def output_rows(iterator, headers, nl, arrays, json_cols):
+def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False):
     # Duplicate column names would collide as dictionary keys, so rename
     # later occurrences id, id -> id, id_2 - CSV and table output keep
     # the original duplicate headers since they never build dictionaries
@@ -3526,7 +3549,7 @@ def output_rows(iterator, headers, nl, arrays, json_cols):
             data = dict(zip(headers, data))
         line = "{firstchar}{serialized}{maybecomma}{lastchar}".format(
             firstchar=("[" if first else " ") if not nl else "",
-            serialized=json.dumps(data, default=json_binary),
+            serialized=json.dumps(data, default=json_binary, ensure_ascii=ascii_),
             maybecomma="," if (not nl and not is_last) else "",
             lastchar="]" if (is_last and not nl) else "",
         )

tests/test_cli.py

@@ -1047,6 +1047,34 @@ def test_query_json_with_json_cols(db_path):
     assert expected == result_rows.output.strip()
 
 
+def test_query_json_unicode_not_escaped_by_default(db_path):
+    db = Database(db_path)
+    with db.conn:
+        db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id")
+    result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text"])
+    assert result.exit_code == 0
+    assert result.output.strip() == '[{"id": 1, "text": "Japanese 日本語"}]'
+    # Same for --nl
+    result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text", "--nl"])
+    assert result.exit_code == 0
+    assert result.output.strip() == '{"id": 1, "text": "Japanese 日本語"}'
+
+
+@pytest.mark.parametrize("command", ["query", "rows"])
+def test_query_json_ascii_option(db_path, command):
+    db = Database(db_path)
+    with db.conn:
+        db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id")
+    if command == "query":
+        args = [db_path, "select id, text from text", "--ascii"]
+    else:
+        args = ["rows", db_path, "text", "--ascii"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 0
+    expected = '[{"id": 1, "text": "Japanese ' + "\\u65e5\\u672c\\u8a9e" + '"}]'
+    assert result.output.strip() == expected
+
+
 @pytest.mark.parametrize(
     "content,is_binary",
     [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)],

tests/test_cli_convert.py

@@ -215,6 +215,25 @@ def test_convert_multi_dryrun(test_db_and_path):
     )
 
 
+def test_convert_multi_dryrun_unicode_not_escaped(test_db_and_path):
+    db_path = test_db_and_path[1]
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "{'text': 'Japanese 日本語'}",
+            "--dry-run",
+            "--multi",
+        ],
+    )
+    assert result.exit_code == 0
+    # Preview should match what jsonify_if_needed() would actually store
+    assert '{"text": "Japanese 日本語"}' in result.output
+
+
 @pytest.mark.parametrize("drop", (True, False))
 def test_convert_output_column(test_db_and_path, drop):
     db, db_path = test_db_and_path