From 5309c5c7755818323a0f5353bad0de98ecc866be Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 27 Jan 2019 18:17:38 -0800
Subject: [PATCH] sqlite-utils insert ... --nl option, closes #6

---
 docs/cli.rst        |  5 +++++
 sqlite_utils/cli.py | 13 +++++++++----
 tests/test_cli.py   | 18 +++++++++++++++---
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/docs/cli.rst b/docs/cli.rst
index c98230e..62b117a 100644
--- a/docs/cli.rst
+++ b/docs/cli.rst
@@ -123,6 +123,11 @@ You can import all three records into an automatically created ``dogs`` table an
 
     $ sqlite-utils insert dogs.db dogs dogs.json --pk=id
 
+You can also import newline-delimited JSON using the ``--nl`` option. Since [Datasette](https://datasette.readthedocs.io/) can export newline-delimited JSON, you can combine the two tools like so::
+
+    $ curl -L "https://latest.datasette.io/fixtures/facetable.json?_shape=array&_nl=on" \
+        | sqlite-utils insert nl-demo.db facetable - --pk=id --nl
+
 Upserting data
 ==============
 
diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py
index 6e3d15c..c6117ad 100644
--- a/sqlite_utils/cli.py
+++ b/sqlite_utils/cli.py
@@ -71,12 +71,17 @@ def optimize(path, no_vacuum):
 @click.argument("table")
 @click.argument("json_file", type=click.File(), required=True)
 @click.option("--pk", help="Column to use as the primary key, e.g. id")
-def insert(path, table, json_file, pk):
+@click.option("--nl", is_flag=True, help="Expect newline-delimited JSON")
+def insert(path, table, json_file, pk, nl):
     "Insert records from JSON file into the table, create table if it is missing"
     db = sqlite_utils.Database(path)
-    docs = json_std.load(json_file)
-    if isinstance(docs, dict):
-        docs = [docs]
+    if nl:
+        # TODO: Use a generator once #7 is solved
+        docs = [json_std.loads(line) for line in json_file]
+    else:
+        docs = json_std.load(json_file)
+        if isinstance(docs, dict):
+            docs = [docs]
     db[table].insert_all(docs, pk=pk)
 
 
diff --git a/tests/test_cli.py b/tests/test_cli.py
index d48bc55..3b66871 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -104,13 +104,25 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir):
         cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
     )
     assert 0 == result.exit_code
-    assert dogs == Database(db_path).execute_returning_dicts(
-        "select * from dogs order by id"
-    )
     db = Database(db_path)
+    assert dogs == db.execute_returning_dicts("select * from dogs order by id")
     assert ["id"] == db["dogs"].pks
 
 
+def test_insert_newline_delimited(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl"],
+        input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}',
+    )
+    assert 0 == result.exit_code, result.output
+    db = Database(db_path)
+    assert [
+        {"foo": "bar", "n": 1},
+        {"foo": "baz", "n": 2},
+    ] == db.execute_returning_dicts("select foo, n from from_json_nl")
+
+
 def test_upsert(db_path, tmpdir):
     test_insert_multiple_with_primary_key(db_path, tmpdir)
     json_path = str(tmpdir / "upsert.json")