From 20e543e9a492f2e764caae73c38e87f18eaec444 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 26 Jul 2020 17:48:36 -0700
Subject: [PATCH] Output binary columns as base64 in JSON, closes #125

---
 docs/cli.rst        | 13 +++++++++++++
 sqlite_utils/cli.py | 10 +++++++++-
 tests/test_cli.py   | 24 ++++++++++++++++++++++++
 3 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/docs/cli.rst b/docs/cli.rst
index cbcf1ee..8baa194 100644
--- a/docs/cli.rst
+++ b/docs/cli.rst
@@ -55,6 +55,19 @@ If you want to pretty-print the output further, you can pipe it through ``python
         }
     ]
 
+Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this::
+
+    $ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool
+    [
+        {
+            "name": "smile.gif",
+            "content": {
+                "$base64": true,
+                "encoded": "eJzt0c1x..."
+            }
+        }
+    ]
+
 If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the comand will return the number of affected rows::
 
     $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'"   
diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py
index 4e70fd4..f387602 100644
--- a/sqlite_utils/cli.py
+++ b/sqlite_utils/cli.py
@@ -1,3 +1,4 @@
+import base64
 import click
 from click_default_group import DefaultGroup
 import sqlite_utils
@@ -748,7 +749,7 @@ def output_rows(iterator, headers, nl, arrays, json_cols):
             data = dict(zip(headers, data))
         line = "{firstchar}{serialized}{maybecomma}{lastchar}".format(
             firstchar=("[" if first else " ") if not nl else "",
-            serialized=json.dumps(data),
+            serialized=json.dumps(data, default=json_binary),
             maybecomma="," if (not nl and not is_last) else "",
             lastchar="]" if (is_last and not nl) else "",
         )
@@ -766,3 +767,10 @@ def maybe_json(value):
         return json.loads(stripped)
     except ValueError:
         return value
+
+
+def json_binary(value):
+    if isinstance(value, bytes):
+        return {"$base64": True, "encoded": base64.b64encode(value).decode("latin-1")}
+    else:
+        raise TypeError
diff --git a/tests/test_cli.py b/tests/test_cli.py
index bf4e8ce..3584840 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -765,6 +765,30 @@ def test_query_json(db_path, sql, args, expected):
     assert expected == result.output.strip()
 
 
+LOREM_IPSUM_COMPRESSED = b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8ef\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3\x85>\x8c\xa4i\x8d\xdaTu\x7f<c\xc9\xf5L\x0f\xd7E\xad/\x9b\x9eI^2\x93\x1a\x9b\xf6F^\n\xd7\xd4\x8f\xca\xfb\x90.\xdd/\xfd\x94\xd4\x11\x87I8\x1a\xaf\xd1S?\x06\x88\xa7\xecBo\xbb$\xbb\t\xe9\xf4\xe8\xe4\x98U\x1bM\x19S\xbe\xa4e\x991x\xfcx\xf6\xe2#\x9e\x93h'&%YK(i)\x7f\t\xc5@N7\xbf+\x1b\xb5\xdd\x10\r\x9e\xb1\xf0y\xa1\xf7W\x92a\xe2;\xc6\xc8\xa0\xa7\xc4\x92\xe2\\\xf2\xa1\x99m\xdf\x88)\xc6\xec\x9a\xa5\xed\x14wR\xf1h\xf22x\xcfM\xfdv\xd3\xa4LY\x96\xcc\xbd[{\xd9m\xf0\x0eH#\x8e\xf5\x9b\xab\xd7\xcb\xe9t\x05\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}"
+
+
+def test_query_json_binary(db_path):
+    db = Database(db_path)
+    with db.conn:
+        db["files"].insert(
+            {"name": "lorem.txt", "sz": 16984, "data": LOREM_IPSUM_COMPRESSED,},
+            pk="name",
+        )
+    result = CliRunner().invoke(cli.cli, [db_path, "select name, sz, data from files"])
+    assert result.exit_code == 0, str(result)
+    assert json.loads(result.output.strip()) == [
+        {
+            "name": "lorem.txt",
+            "sz": 16984,
+            "data": {
+                "$base64": True,
+                "encoded": "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uIjnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3fiCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9",
+            },
+        }
+    ]
+
+
 def test_query_json_with_json_cols(db_path):
     db = Database(db_path)
     with db.conn: