diff --git a/docs/cli.rst b/docs/cli.rst index 8b33206..913a9ed 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1078,6 +1078,36 @@ The command will fail if you reference columns that do not exist on the table. T .. note:: ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. + +.. _cli_bulk: + +Executing SQL in bulk +===================== + +If you have a JSON, newline-delimited JSON, CSV or TSV file you can execute a bulk SQL query using each of the records in that file using the ``sqlite-utils bulk`` command. + +The command takes the database file, the SQL to be executed and the file containing records to be used when evaluating the SQL query. + +The SQL query should include ``:named`` parameters that match the keys in the records. + +For example, given a ``chickens.csv`` CSV file containing the following:: + + id,name + 1,Blue + 2,Snowy + 3,Azi + 4,Lila + 5,Suna + 6,Cardi + +You could insert those rows into a pre-created ``chickens`` table like so:: + + $ sqlite-utils bulk chickens.db \ + 'insert into chickens (id, name) values (:id, :name)' \ + chickens.csv --csv + +This command takes the same options as the ``sqlite-utils insert`` command - so it defaults to expecting JSON but can accept other formats using ``--csv`` or ``--tsv`` or ``--nl`` or other options described above. + .. _cli_insert_files: Inserting data from files diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 187795b..0dd7f71 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -660,6 +660,50 @@ def reset_counts(path, load_extension): db.reset_counts() +_import_options = ( + click.option( + "--flatten", + is_flag=True, + help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}', + ), + click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), + click.option("-c", "--csv", is_flag=True, help="Expect CSV input"), + click.option("--tsv", is_flag=True, help="Expect TSV input"), + click.option( + "--lines", + is_flag=True, + help="Treat each line as a single value called 'line'", + ), + click.option( + "--text", + is_flag=True, + help="Treat input as a single value called 'text'", + ), + click.option("--convert", help="Python code to convert each item"), + click.option( + "--import", + "imports", + type=str, + multiple=True, + help="Python modules to import", + ), + click.option("--delimiter", help="Delimiter to use for CSV files"), + click.option("--quotechar", help="Quote character to use for CSV/TSV"), + click.option("--sniff", is_flag=True, help="Detect delimiter and quote character"), + click.option("--no-headers", is_flag=True, help="CSV file has no header row"), + click.option( + "--encoding", + help="Character encoding for input, defaults to utf-8", + ), +) + + +def import_options(fn): + for decorator in reversed(_import_options): + fn = decorator(fn) + return fn + + def insert_upsert_options(fn): for decorator in reversed( ( @@ -673,40 +717,9 @@ def insert_upsert_options(fn): click.option( "--pk", help="Columns to use as the primary key, e.g. id", multiple=True ), - click.option( - "--flatten", - is_flag=True, - help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}', - ), - click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), - click.option("-c", "--csv", is_flag=True, help="Expect CSV input"), - click.option("--tsv", is_flag=True, help="Expect TSV input"), - click.option( - "--lines", - is_flag=True, - help="Treat each line as a single value called 'line'", - ), - click.option( - "--text", - is_flag=True, - help="Treat input as a single value called 'text'", - ), - click.option("--convert", help="Python code to convert each item"), - click.option( - "--import", - "imports", - type=str, - multiple=True, - help="Python modules to import", - ), - click.option("--delimiter", help="Delimiter to use for CSV files"), - click.option("--quotechar", help="Quote character to use for CSV/TSV"), - click.option( - "--sniff", is_flag=True, help="Detect delimiter and quote character" - ), - click.option( - "--no-headers", is_flag=True, help="CSV file has no header row" - ), + ) + + _import_options + + ( click.option( "--batch-size", type=int, default=100, help="Commit every X records" ), @@ -726,10 +739,6 @@ def insert_upsert_options(fn): type=(str, str), help="Default value that should be set for a column", ), - click.option( - "--encoding", - help="Character encoding for input, defaults to utf-8", - ), click.option( "-d", "--detect-types", @@ -767,6 +776,7 @@ def insert_upsert_implementation( quotechar, sniff, no_headers, + encoding, batch_size, alter, upsert, @@ -775,11 +785,11 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, - encoding=None, detect_types=None, analyze=False, load_extension=None, silent=False, + bulk_sql=None, ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) @@ -886,6 +896,12 @@ def insert_upsert_implementation( # Apply {"$base64": true, ...} decoding, if needed docs = (decode_base64_values(doc) for doc in docs) + # For bulk_sql= we use cursor.executemany() instead + if bulk_sql: + with db.conn: + db.conn.cursor().executemany(bulk_sql, docs) + return + try: db[table].insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs @@ -968,9 +984,9 @@ def insert( quotechar, sniff, no_headers, + encoding, batch_size, alter, - encoding, detect_types, analyze, load_extension, @@ -1020,13 +1036,13 @@ def insert( quotechar, sniff, no_headers, + encoding, batch_size, alter=alter, upsert=False, ignore=ignore, replace=replace, truncate=truncate, - encoding=encoding, detect_types=detect_types, analyze=analyze, load_extension=load_extension, @@ -1058,10 +1074,10 @@ def upsert( quotechar, sniff, no_headers, + encoding, alter, not_null, default, - encoding, detect_types, analyze, load_extension, @@ -1090,12 +1106,12 @@ def upsert( quotechar, sniff, no_headers, + encoding, batch_size, alter=alter, upsert=True, not_null=not_null, default=default, - encoding=encoding, detect_types=detect_types, analyze=analyze, load_extension=load_extension, @@ -1105,6 +1121,71 @@ def upsert( raise click.ClickException(UNICODE_ERROR.format(ex)) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("sql") +@click.argument("file", type=click.File("rb"), required=True) +@import_options +@load_extension_option +def bulk( + path, + file, + sql, + flatten, + nl, + csv, + tsv, + lines, + text, + convert, + imports, + delimiter, + quotechar, + sniff, + no_headers, + encoding, + load_extension, +): + """ + Execute parameterized SQL against the provided list of documents. + """ + try: + insert_upsert_implementation( + path=path, + table=None, + file=file, + pk=None, + flatten=flatten, + nl=nl, + csv=csv, + tsv=tsv, + lines=lines, + text=text, + convert=convert, + imports=imports, + delimiter=delimiter, + quotechar=quotechar, + sniff=sniff, + no_headers=no_headers, + encoding=encoding, + batch_size=1, + alter=False, + upsert=False, + not_null=set(), + default={}, + detect_types=False, + load_extension=load_extension, + silent=False, + bulk_sql=sql, + ) + except (sqlite3.OperationalError, sqlite3.IntegrityError) as e: + raise click.ClickException(str(e)) + + @cli.command(name="create-database") @click.argument( "path", diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py new file mode 100644 index 0000000..37cf60b --- /dev/null +++ b/tests/test_cli_bulk.py @@ -0,0 +1,57 @@ +from click.testing import CliRunner +from sqlite_utils import cli, Database +import pathlib +import pytest + + +@pytest.fixture +def test_db_and_path(tmpdir): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = Database(db_path) + db["example"].insert_all( + [ + {"id": 1, "name": "One"}, + {"id": 2, "name": "Two"}, + ], + pk="id", + ) + return db, db_path + + +def test_cli_bulk(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "bulk", + db_path, + "insert into example (id, name) values (:id, :name)", + "-", + "--nl", + ], + input='{"id": 3, "name": "Three"}\n{"id": 4, "name": "Four"}\n', + ) + assert result.exit_code == 0, result.output + assert [ + {"id": 1, "name": "One"}, + {"id": 2, "name": "Two"}, + {"id": 3, "name": "Three"}, + {"id": 4, "name": "Four"}, + ] == list(db["example"].rows) + + +def test_cli_bulk_error(test_db_and_path): + _, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "bulk", + db_path, + "insert into example (id, name) value (:id, :name)", + "-", + "--nl", + ], + input='{"id": 3, "name": "Three"}', + ) + assert result.exit_code == 1 + assert result.output == 'Error: near "value": syntax error\n'