diff --git a/docs/cli.rst b/docs/cli.rst index abb4267..7636f6c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -11,11 +11,59 @@ Running queries and returning CSV You can execute a SQL query against a database and get the results back as CSV like this:: - $ sqlite-utils csv docs.db "select id, title, author from documents" + $ sqlite-utils csv dogs.db "select * from dogs" + id,age,name + 1,4,Cleo + 2,2,Pancakes This will default to including the column names as a header row. To exclude the headers, use ``--no-headers``:: - $ sqlite-utils csv docs.db "select id, title, author from documents" --no-headers + $ sqlite-utils csv dogs.db "select * from dogs" --no-headers + 1,4,Cleo + 2,2,Pancakes + +Running queries and returning JSON +================================== + +You can execute a SQL query against a database and get the results back as JSON like this:: + + $ sqlite-utils json dogs.db "select * from dogs" + [{"id": 1, "age": 4, "name": "Cleo"}, + {"id": 2, "age": 2, "name": "Pancakes"}] + +Use ``--nl`` to get back newline-delimited JSON objects:: + + $ sqlite-utils json --nl dogs.db "select * from dogs" + {"id": 1, "age": 4, "name": "Cleo"} + {"id": 2, "age": 2, "name": "Pancakes"} + +You can use ``--arrays`` to request ararys instead of objects:: + + $ sqlite-utils json --arrays dogs.db "select * from dogs" + [[1, 4, "Cleo"], + [2, 2, "Pancakes"]] + +You can also combine ``--arrays`` and ``--nl``:: + + $ sqlite-utils json --arrays --nl dogs.db "select * from dogs" + [1, 4, "Cleo"] + [2, 2, "Pancakes"] + +If you want to pretty-print the output further, you can pipe it through ``python -mjson.tool``:: + + $ sqlite-utils json dogs.db "select * from dogs" | python -mjson.tool + [ + { + "id": 1, + "age": 4, + "name": "Cleo" + }, + { + "id": 2, + "age": 2, + "name": "Pancakes" + } + ] Listing tables ============== @@ -71,7 +119,6 @@ You can import all three records into an automatically created ``dogs`` table an $ sqlite-utils insert dogs.db dogs dogs.json --pk=id - Upserting data ============== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index ed4089a..4738215 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,8 +1,10 @@ import click import sqlite_utils -import json +from sqlite_utils.utils import iter_pairs +import json as json_std import sys import csv as csv_std +import sqlite3 @click.group() @@ -72,7 +74,7 @@ def optimize(path, no_vacuum): def insert(path, table, json_file, pk): "Insert records from JSON file into the table, create table if it is missing" db = sqlite_utils.Database(path) - docs = json.load(json_file) + docs = json_std.load(json_file) if isinstance(docs, dict): docs = [docs] db[table].insert_all(docs, pk=pk) @@ -90,7 +92,7 @@ def insert(path, table, json_file, pk): def upsert(path, table, json_file, pk): "Upsert records based on their primary key" db = sqlite_utils.Database(path) - docs = json.load(json_file) + docs = json_std.load(json_file) if isinstance(docs, dict): docs = [docs] db[table].upsert_all(docs, pk=pk) @@ -115,3 +117,42 @@ def csv(path, sql, no_headers): writer.writerow([c[0] for c in cursor.description]) for row in cursor: writer.writerow(row) + + +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("sql") +@click.option("--nl", help="Output newline-delimited JSON", is_flag=True, default=False) +@click.option( + "--arrays", + help="Output rows as arrays instead of objects", + is_flag=True, + default=False, +) +def json(path, sql, nl, arrays): + "Execute SQL query and return the results as JSON" + db = sqlite_utils.Database(path) + cursor = iter(db.conn.execute(sql)) + # We have to iterate two-at-a-time so we can know if we + # should output a trailing comma or if we have reached + # the last row. + row = None + first = True + headers = [c[0] for c in cursor.description] + for row, next_row, is_last in iter_pairs(cursor): + # We now reliably have row and next_row + data = row + if not arrays: + data = dict(zip(headers, row)) + line = "{firstchar}{serialized}{maybecomma}{lastchar}".format( + firstchar=("[" if first else " ") if not nl else "", + serialized=json_std.dumps(data), + maybecomma="," if (not nl and not is_last) else "", + lastchar="]" if (is_last and not nl) else "", + ) + click.echo(line) + first = False diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py new file mode 100644 index 0000000..33e68a1 --- /dev/null +++ b/sqlite_utils/utils.py @@ -0,0 +1,28 @@ +def iter_pairs(iterator): + # Yields (item, next_item, is_last) pairs + # Last row is (item, None, True) + first = True + next_item = None + while True: + next_done = False + if first: + item, done = next_with_done(iterator) + if done: + break + next_item, next_done = next_with_done(iterator) + first = False + else: + item = next_item + next_item, next_done = next_with_done(iterator) + + yield item, next_item, next_done + + if next_done: + break + + +def next_with_done(iterator): + try: + return next(iterator), False + except StopIteration: + return None, True diff --git a/tests/test_cli.py b/tests/test_cli.py index 17d6e89..a8bb7a0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -151,3 +151,39 @@ def test_csv(db_path): cli.cli, ["csv", db_path, "select id, name, age from dogs", "--no-headers"] ) assert "1,Cleo,4\n2,Pancakes,2\n" == result.output + + +@pytest.mark.parametrize( + "args,expected", + [ + ( + [], + '[{"id": 1, "name": "Cleo", "age": 4},\n {"id": 2, "name": "Pancakes", "age": 2}]', + ), + ( + ["--nl"], + '{"id": 1, "name": "Cleo", "age": 4}\n{"id": 2, "name": "Pancakes", "age": 2}', + ), + (["--arrays"], '[[1, "Cleo", 4],\n [2, "Pancakes", 2]]'), + (["--arrays", "--nl"], '[1, "Cleo", 4]\n[2, "Pancakes", 2]'), + ], +) +def test_json(db_path, args, expected): + db = Database(db_path) + with db.conn: + db["dogs"].insert_all( + [ + {"id": 1, "age": 4, "name": "Cleo"}, + {"id": 2, "age": 2, "name": "Pancakes"}, + ] + ) + result = CliRunner().invoke( + cli.cli, ["csv", db_path, "select id, name, age from dogs"] + ) + assert 0 == result.exit_code + assert "id,name,age\n1,Cleo,4\n2,Pancakes,2\n" == result.output + # Test the no-headers option: + result = CliRunner().invoke( + cli.cli, ["json", db_path, "select id, name, age from dogs"] + args + ) + assert expected == result.output.strip()