From 42ec59d8ee3fcfb8ac1affb772aed43b6e2a7381 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 08:00:52 -0700 Subject: [PATCH] sqlite-utils memory command for directly querying CSV/JSON data * Turn SQL errors into click errors * Initial CSV-only prototype of sqlite-utils memory, refs #272 * Implement --save plus tests for --save and --dump, refs #272 * Re-arranged CLI query documentation, refs #272 * Re-organized CLI query docs, refs #272 * Docs for --save and --dump plus made SQL optional for those, refs #273 * Replaced one last :memory: example * Documented --attach option for memory command, refs #272 * Improved arrangement of CLI query documentation --- docs/changelog.rst | 2 +- docs/cli.rst | 225 ++++++++++++++++++++++++++++++--------- sqlite_utils/cli.py | 105 +++++++++++++++++- tests/test_cli_memory.py | 72 +++++++++++++ 4 files changed, 351 insertions(+), 53 deletions(-) create mode 100644 tests/test_cli_memory.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 3339f0d..879d19b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -47,7 +47,7 @@ This release adds the ability to execute queries joining data from more than one database file - similar to the cross database querying feature introduced in `Datasette 0.55 `__. - The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (`#113 `__) -- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) +- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) .. _v3_5: diff --git a/docs/cli.rst b/docs/cli.rst index a45f3b8..b7f0313 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -8,25 +8,31 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite database .. contents:: :local: -.. _cli_query_json: +.. _cli_query: -Running queries and returning JSON -================================== +Running SQL queries +=================== -You can execute a SQL query against a database and get the results back as JSON like this:: +The ``sqlite-utils query`` command lets you run queries directly against a SQLite database file. This is the default subcommand, so the following two examples work the same way:: $ sqlite-utils query dogs.db "select * from dogs" + $ sqlite-utils dogs.db "select * from dogs" + +.. _cli_query_json: + +Returning JSON +-------------- + +The default format returned for queries is JSON:: + + $ sqlite-utils dogs.db "select * from dogs" [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] -This is the default command for ``sqlite-utils``, so you can instead use this:: +.. _cli_query_nl: - $ sqlite-utils dogs.db "select * from dogs" - -You can pass named parameters to the query using ``-p``:: - - $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 - [{":num * :num2": 30}] +Newline-delimited JSON +~~~~~~~~~~~~~~~~~~~~~~ Use ``--nl`` to get back newline-delimited JSON objects:: @@ -34,6 +40,11 @@ Use ``--nl`` to get back newline-delimited JSON objects:: {"id": 1, "age": 4, "name": "Cleo"} {"id": 2, "age": 2, "name": "Pancakes"} +.. _cli_query_arrays: + +JSON arrays +~~~~~~~~~~~ + You can use ``--arrays`` to request arrays instead of objects:: $ sqlite-utils dogs.db "select * from dogs" --arrays @@ -62,6 +73,11 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +.. _cli_query_binary_json: + +Binary data in JSON +~~~~~~~~~~~~~~~~~~~ + Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this:: $ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool @@ -75,27 +91,11 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b } ] -If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: - - $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" - [{"rows_affected": 1}] - -You can run queries against a temporary in-memory database by passing ``:memory:`` as the filename:: - - $ sqlite-utils :memory: "select sqlite_version()" - [{"sqlite_version()": "3.29.0"}] - -You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. - -:: - - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite - [{"spatialite_version()": "4.3.0a"}] .. _cli_json_values: Nested JSON values ------------------- +~~~~~~~~~~~~~~~~~~ If one of your columns contains JSON, by default it will be returned as an escaped string:: @@ -126,24 +126,10 @@ You can use the ``--json-cols`` option to automatically detect these JSON column } ] -.. _cli_attach: - -Attaching additional databases ------------------------------- - -SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. - -You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. - -This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: - - sqlite-utils dogs.db --attach books books.db \ - 'select * from sqlite_master union all select * from books.sqlite_master' - .. _cli_query_csv: -Running queries and returning CSV -================================= +Returning CSV or TSV +-------------------- You can use the ``--csv`` option to return results as CSV:: @@ -167,8 +153,8 @@ Use ``--tsv`` instead of ``--csv`` to get back tab-separated values:: .. _cli_query_table: -Running queries and outputting a table -====================================== +Table-formatted output +---------------------- You can use the ``--table`` option (or ``-t`` shortcut) to output query results as a table:: @@ -192,8 +178,8 @@ For a full list of table format options, run ``sqlite-utils query --help``. .. _cli_query_raw: -Returning raw data from a query, such as binary content -======================================================= +Returning raw data, such as binary content +------------------------------------------ If your table contains binary data in a ``BLOB`` you can use the ``--raw`` option to output specific columns directly to standard out. @@ -201,6 +187,145 @@ For example, to retrieve a binary image from a ``BLOB`` column and store it in a $ sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg + +.. _cli_query_parameters: + +Using named parameters +---------------------- + +You can pass named parameters to the query using ``-p``:: + + $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 + [{":num * :num2": 30}] + +These will be correctly quoted and escaped in the SQL query, providing a safe way to combine other values with SQL. + +.. _cli_query_update_insert_delete: + +UPDATE, INSERT and DELETE +------------------------- + +If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: + + $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" + [{"rows_affected": 1}] + +SQLite extensions +----------------- + +You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. + +:: + + $ sqlite-utils dogs.db "select spatialite_version()" --load-extension=spatialite + [{"spatialite_version()": "4.3.0a"}] + +.. _cli_query_attach: + +Attaching additional databases +------------------------------ + +SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. + +You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. + +This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: + + sqlite-utils dogs.db --attach books books.db \ + 'select * from sqlite_master union all select * from books.sqlite_master' + +.. _cli_query_memory: + +Querying CSV data directly using an in-memory database +====================================================== + +The ``sqlite-utils memory`` command works similar to ``sqlite-utils query``, but allows you to execute queries against an in-memory database. + +You can also pass this command CSV files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite. + +Without any extra arguments, this command executes SQL against the in-memory database directly:: + + $ sqlite-utils memory 'select sqlite_version()' + [{"sqlite_version()": "3.35.5"}] + +It takes all of the same formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``:: + + $ sqlite-utils memory 'select sqlite_version()' --csv + sqlite_version() + 3.35.5 + $ sqlite-utils memory 'select sqlite_version()' --table --fmt grid + +--------------------+ + | sqlite_version() | + +====================+ + | 3.35.5 | + +--------------------+ + +.. _cli_query_memory_csv: + +Running queries directly against CSV +------------------------------------ + +If you have data in CSV format you can load it into an in-memory SQLite database and run queries against it directly in a single command using ``sqlite-utils memory`` like this:: + + $ sqlite-utils memory data.csv "select * from data" + +You can pass multiple files to the command if you want to run joins between different CSV files:: + + $ sqlite-utils memory one.csv two.csv "select * from one join two on one.id = two.other_id" + +The in-memory tables will be named after the CSV files without their ``.csv`` extension. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table:: + + $ sqlite-utils memory example.csv "select * from t" + +To read from standard input, use ``-`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: + + $ cat example.csv | sqlite-utils memory - "select * from stdin" + +.. _cli_query_memory_attach: + +Joining in-memory data against existing databases using \-\-attach +------------------------------------------------------------------ + +The :ref:`attach option ` can be used to attach database files to the in-memory connection, enabling joins between in-memory data loaded from a file and tables in existing SQLite database files. An example:: + + $ echo "id\n1\n3\n5" | sqlite-utils memory - --attach trees trees.db \ + "select * from trees.trees where rowid in (select id from stdin)" + +Here the ``--attach trees trees.db`` option makes the ``trees.db`` database available with an alias of ``trees``. + +``select * from trees.trees where ...`` can then query the ``trees`` table in that database. + +The CSV data that was piped into the script is available in the ``stdin`` table, so ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content. + +.. _cli_query_memory_dump_save: + +\-\-dump and \-\-save +--------------------- + +You can dump out the SQL used for the temporary in-memory database, complete with all imported data, using the ``--dump`` option:: + + % sqlite-utils memory dogs.csv --dump + BEGIN TRANSACTION; + CREATE TABLE [dogs] ( + [rowid] TEXT, + [id] TEXT, + [dog_age] TEXT, + [name] TEXT + ); + INSERT INTO "dogs" VALUES('1','1','4','Cleo'); + INSERT INTO "dogs" VALUES('2','2','2','Pancakes'); + INSERT INTO "dogs" VALUES('3','2','3','Pancakes'); + CREATE VIEW t1 AS select * from [dogs]; + CREATE VIEW t AS select * from [dogs]; + COMMIT; + +Passing ``--save other.db`` will instead use that SQL to populate a new database file:: + + % sqlite-utils memory dogs.csv --save dogs.db + +These features are mainly intented as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. + + .. _cli_rows: Returning all rows in a table @@ -270,7 +395,7 @@ Use ``--schema`` to include the schema of each table:: [age] INTEGER, [name] TEXT) -The ``--nl``, ``--csv``, ``--tsv`` and ``--table`` options are all available. +The ``--nl``, ``--csv``, ``--tsv``, ``--table`` and ``--fmt`` options are also available. .. _cli_views: @@ -1280,5 +1405,5 @@ This option can be applied multiple times to load multiple extensions. Since `SpatiaLite `__ is commonly used with SQLite, the value ``spatialite`` is special: it will search for SpatiaLite in the most common installation locations, saving you from needing to remember exactly where that module is located:: - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite + $ sqlite-utils memory "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 69e04e3..8fe37c8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -716,11 +716,11 @@ def insert_upsert_implementation( raise click.ClickException("Use just one of --nl, --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") + if pk and len(pk) == 1: + pk = pk[0] encoding = encoding or "utf-8-sig" buffered = io.BufferedReader(json_file, buffer_size=4096) decoded = io.TextIOWrapper(buffered, encoding=encoding) - if pk and len(pk) == 1: - pk = pk[0] if csv or tsv: if sniff: # Read first 2048 bytes and use that to detect @@ -1109,6 +1109,107 @@ def query( db.attach(alias, attach_path) _load_extensions(db, load_extension) db.register_fts4_bm25() + + _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + ) + + +@cli.command() +@click.argument( + "paths", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True), + required=False, + nargs=-1, +) +@click.argument("sql") +@click.option( + "--attach", + type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), + multiple=True, + help="Additional databases to attach - specify alias and filepath", +) +@output_options +@click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for SQL query", +) +@click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") +@click.option( + "--save", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + help="Save in-memory database to this file", +) +@load_extension_option +def memory( + paths, + sql, + attach, + nl, + arrays, + csv, + tsv, + no_headers, + table, + fmt, + json_cols, + raw, + param, + dump, + save, + load_extension, +): + "Execute SQL query against an in-memory database, optionally populated by imported data" + db = sqlite_utils.Database(memory=True) + # If --dump or --save used but no paths detected, assume SQL query is a path: + if (dump or save) and not paths: + paths = [sql] + sql = None + for i, path in enumerate(paths): + if path == "-": + csv_fp = sys.stdin + csv_table = "stdin" + else: + csv_path = pathlib.Path(path) + csv_table = csv_path.stem + csv_fp = csv_path.open() + db[csv_table].insert_all(csv_std.DictReader(csv_fp)) + # Add convenient t / t1 / t2 views + view_names = ["t{}".format(i + 1)] + if i == 0: + view_names.append("t") + for view_name in view_names: + if not db[view_name].exists(): + db.create_view(view_name, "select * from [{}]".format(csv_table)) + + if dump: + for line in db.conn.iterdump(): + click.echo(line) + return + + if save: + db2 = sqlite_utils.Database(save) + for line in db.conn.iterdump(): + db2.execute(line) + return + + for alias, attach_path in attach: + db.attach(alias, attach_path) + _load_extensions(db, load_extension) + db.register_fts4_bm25() + + _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + ) + + +def _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols +): with db.conn: try: cursor = db.execute(sql, dict(param)) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py new file mode 100644 index 0000000..f018837 --- /dev/null +++ b/tests/test_cli_memory.py @@ -0,0 +1,72 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner +import pytest + + +def test_memory_basic(): + result = CliRunner().invoke(cli.cli, ["memory", "select 1 + 1"]) + assert result.exit_code == 0 + assert result.output.strip() == '[{"1 + 1": 2}]' + + +@pytest.mark.parametrize("sql_from", ("test", "t", "t1")) +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_csv(tmpdir, sql_from, use_stdin): + content = "id,name\n1,Cleo\n2,Bants" + input = None + if use_stdin: + input = content + csv_path = "-" + if sql_from == "test": + sql_from = "stdin" + else: + csv_path = str(tmpdir / "test.csv") + open(csv_path, "w").write(content) + result = CliRunner().invoke( + cli.cli, + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + input=input, + ) + assert result.exit_code == 0 + assert ( + result.output.strip() + == '{"id": "1", "name": "Cleo"}\n{"id": "2", "name": "Bants"}' + ) + + +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_dump(extra_args): + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--dump"], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + "BEGIN TRANSACTION;\n" + "CREATE TABLE [stdin] (\n" + " [id] TEXT,\n" + " [name] TEXT\n" + ");\n" + "INSERT INTO \"stdin\" VALUES('1','Cleo');\n" + "INSERT INTO \"stdin\" VALUES('2','Bants');\n" + "CREATE VIEW t1 AS select * from [stdin];\n" + "CREATE VIEW t AS select * from [stdin];\n" + "COMMIT;" + ) + + +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_save(tmpdir, extra_args): + save_to = str(tmpdir / "save.db") + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--save", save_to], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + db = Database(save_to) + assert list(db["stdin"].rows) == [ + {"id": "1", "name": "Cleo"}, + {"id": "2", "name": "Bants"}, + ]