Ability to insert base64 binary data as JSON, closes #126

sqlite_utils/cli.py

@@ -8,7 +8,7 @@ import json
 import sys
 import csv as csv_std
 import tabulate
-from .utils import sqlite3
+from .utils import sqlite3, decode_base64_values
 
 VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB")
 
@@ -451,6 +451,8 @@ def insert_upsert_implementation(
         extra_kwargs["defaults"] = dict(default)
     if upsert:
         extra_kwargs["upsert"] = upsert
+    # Apply {"$base64": true, ...} decoding, if needed
+    docs = (decode_base64_values(doc) for doc in docs)
     db[table].insert_all(
         docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs
     )

sqlite_utils/utils.py

@@ -1,3 +1,5 @@
+import base64
+
 try:
     import pysqlite3 as sqlite3
     import pysqlite3.dbapi2
@@ -58,3 +60,17 @@ def column_affinity(column_type):
         return float
     # Default is 'NUMERIC', which we currently also treat as float
     return float
+
+
+def decode_base64_values(doc):
+    # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them
+    to_fix = [
+        k
+        for k in doc
+        if isinstance(doc[k], dict)
+        and doc[k].get("$base64") is True
+        and "encoded" in doc[k]
+    ]
+    if not to_fix:
+        return doc
+    return dict(doc, **{k: base64.b64decode(doc[k]["encoded"]) for k in to_fix})