From 7a9a6363ffc1b4f1a9444a22999addabfa756c54 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 21:10:20 -0700 Subject: [PATCH 1/8] sqlite-utils rows --order option, closes #469 --- docs/cli-reference.rst | 1 + docs/cli.rst | 2 ++ sqlite_utils/cli.py | 4 ++++ tests/test_cli.py | 9 +++++++++ 4 files changed, 16 insertions(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 2590595..c0f794c 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -713,6 +713,7 @@ See :ref:`cli_rows`. Options: -c, --column TEXT Columns to return --where TEXT Optional where clause + -o, --order TEXT Order by ('column' or 'column desc') -p, --param ... Named :parameters for where clause --limit INTEGER Number of rows to return - defaults to everything --offset INTEGER SQL offset to use diff --git a/docs/cli.rst b/docs/cli.rst index d2d1812..8cb2c56 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -467,6 +467,8 @@ Or pass named parameters using ``--where`` in combination with ``-p``:: $ sqlite-utils rows dogs.db dogs -c name --where 'name = :name' -p name Cleo [{"name": "Cleo"}] +You can define a sort order using ``--order column`` or ``--order 'column desc'``. + Use ``--limit N`` to only return the first ``N`` rows. Use ``--offset N`` to return rows starting from the specified offset. .. note:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index fe57fbe..43e76fa 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1985,6 +1985,7 @@ def search( @click.argument("dbtable") @click.option("-c", "--column", type=str, multiple=True, help="Columns to return") @click.option("--where", help="Optional where clause") +@click.option("-o", "--order", type=str, help="Order by ('column' or 'column desc')") @click.option( "-p", "--param", @@ -2011,6 +2012,7 @@ def rows( dbtable, column, where, + order, param, limit, offset, @@ -2037,6 +2039,8 @@ def rows( sql = "select {} from [{}]".format(columns, dbtable) if where: sql += " where " + where + if order: + sql += " order by " + order if limit: sql += " limit {}".format(limit) if offset: diff --git a/tests/test_cli.py b/tests/test_cli.py index 061a578..4eadb88 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -881,6 +881,15 @@ def test_query_memory_does_not_create_file(tmpdir): ["-c", "name", "--where", "id = :id", "--param", "id", "1"], '[{"name": "Cleo"}]', ), + # --order + ( + ["-c", "id", "--order", "id desc", "--limit", "1"], + '[{"id": 2}]', + ), + ( + ["-c", "id", "--order", "id", "--limit", "1"], + '[{"id": 1}]', + ), ], ) def test_rows(db_path, args, expected): From 31f062d4a7e6457bfbe94b2e45a7b80028f1e95c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 21:53:55 -0700 Subject: [PATCH 2/8] sqlite-utils query --functions option, refs #471 --- docs/cli-reference.rst | 2 ++ docs/cli.rst | 20 +++++++++++++ sqlite_utils/cli.py | 17 +++++++++++ tests/test_cli.py | 68 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 106 insertions(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c0f794c..af059f1 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -119,6 +119,8 @@ See :ref:`cli_query`. escaped strings -r, --raw Raw output, first column of first row -p, --param ... Named :parameters for SQL query + --functions TEXT Python code defining one or more custom SQL + functions --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index 8cb2c56..1632e2f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -250,6 +250,26 @@ If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will re $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" [{"rows_affected": 1}] +.. _cli_query_functions: + +Defining custom SQL functions +----------------------------- + +You can use the ``--functions`` option to pass a block of Python code that defines additional functions which can then be called by your SQL query. + +This example defines a function which extracts the domain from a URL:: + + $ sqlite-utils query dogs.db "select url, domain(url) from urls" --functions ' + from urllib.parse import urlparse + + def domain(url): + return urlparse(url).netloc + ' + +Every callable object defined in the block will be registered as a SQL function with the same name, with the exception of functions with names that begin with an underscore. + +.. _cli_query_extensions: + SQLite extensions ----------------- diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 43e76fa..1ac8d78 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1633,6 +1633,9 @@ def drop_view(path, view, ignore, load_extension): type=(str, str), help="Named :parameters for SQL query", ) +@click.option( + "--functions", help="Python code defining one or more custom SQL functions" +) @load_extension_option def query( path, @@ -1649,6 +1652,7 @@ def query( raw, param, load_extension, + functions, ): """Execute SQL query and return the results as JSON @@ -1665,6 +1669,19 @@ def query( _load_extensions(db, load_extension) db.register_fts4_bm25() + # Register any Python functions as SQL functions: + if functions: + sqlite3.enable_callback_tracebacks(True) + globals = {} + try: + exec(functions, globals) + except SyntaxError as ex: + raise click.ClickException("Error in functions definition: {}".format(ex)) + # Register all callables in the locals dict: + for name, value in globals.items(): + if callable(value) and not name.startswith("_"): + db.register_function(value, name=name) + _execute_query( db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4eadb88..057f129 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -683,6 +683,11 @@ _one_query = "select id, name, age from dogs where id = 1" (_one_query, ["--nl"], '{"id": 1, "name": "Cleo", "age": 4}'), (_one_query, ["--arrays"], '[[1, "Cleo", 4]]'), (_one_query, ["--arrays", "--nl"], '[1, "Cleo", 4]'), + ( + "select id, dog(age) from dogs", + ["--functions", "def dog(i):\n return i * 7"], + '[{"id": 1, "dog(age)": 28},\n {"id": 2, "dog(age)": 14}]', + ), ], ) def test_query_json(db_path, sql, args, expected): @@ -700,11 +705,72 @@ def test_query_json(db_path, sql, args, expected): def test_query_json_empty(db_path): result = CliRunner().invoke( - cli.cli, [db_path, "select * from sqlite_master where 0"] + cli.cli, + [db_path, "select * from sqlite_master where 0"], ) assert result.output.strip() == "[]" +def test_query_invalid_function(db_path): + result = CliRunner().invoke( + cli.cli, [db_path, "select bad()", "--functions", "def invalid_python"] + ) + assert result.exit_code == 1 + assert ( + result.output.strip() + == "Error: Error in functions definition: invalid syntax (, line 1)" + ) + + +TEST_FUNCTIONS = """ +def zero(): + return 0 + +def one(a): + return a + +def _two(a, b): + return a + b + +def two(a, b): + return _two(a, b) +""" + + +def test_query_complex_function(db_path): + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select zero(), one(1), two(1, 2)", + "--functions", + TEST_FUNCTIONS, + ], + ) + assert result.exit_code == 0 + assert json.loads(result.output.strip()) == [ + {"zero()": 0, "one(1)": 1, "two(1, 2)": 3} + ] + + +def test_hidden_functions_are_hidden(db_path): + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select name from pragma_function_list()", + "--functions", + TEST_FUNCTIONS, + ], + ) + assert result.exit_code == 0 + functions = {r["name"] for r in json.loads(result.output.strip())} + assert "zero" in functions + assert "one" in functions + assert "two" in functions + assert "_two" not in functions + + LOREM_IPSUM_COMPRESSED = ( b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8e" b"f\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J" From 85e7411bbd2884e42c65c3e93330f0ddd986be38 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:01:58 -0700 Subject: [PATCH 3/8] Skip test if pragma_function_list not supported, refs #471 --- tests/test_cli.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 057f129..9b43cd5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,6 +12,15 @@ import textwrap from .utils import collapse_whitespace +def _supports_pragma_function_list(): + db = Database(memory=True) + try: + db.execute("select * from pragma_function_list()") + except Exception: + return False + return True + + @pytest.mark.parametrize( "options", ( @@ -753,6 +762,10 @@ def test_query_complex_function(db_path): ] +@pytest.mark.skipif( + not _supports_pragma_function_list(), + reason="Needs SQLite version that supports pragma_function_list()", +) def test_hidden_functions_are_hidden(db_path): result = CliRunner().invoke( cli.cli, From 59e2cfbdc12082bac03e8ac6f99c8c41a4bc72ba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:03:53 -0700 Subject: [PATCH 4/8] sqlite-utils memory --functions, refs #471 --- docs/cli-reference.rst | 2 ++ sqlite_utils/cli.py | 33 ++++++++++++++++++++++----------- tests/test_cli_memory.py | 9 +++++++++ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index af059f1..9a025ba 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -161,6 +161,8 @@ See :ref:`cli_memory`. sqlite-utils memory animals.csv --schema Options: + --functions TEXT Python code defining one or more custom SQL + functions --attach ... Additional databases to attach - specify alias and filepath --flatten Flatten nested JSON objects, so {"foo": {"bar": diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1ac8d78..24c8c03 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1669,18 +1669,8 @@ def query( _load_extensions(db, load_extension) db.register_fts4_bm25() - # Register any Python functions as SQL functions: if functions: - sqlite3.enable_callback_tracebacks(True) - globals = {} - try: - exec(functions, globals) - except SyntaxError as ex: - raise click.ClickException("Error in functions definition: {}".format(ex)) - # Register all callables in the locals dict: - for name, value in globals.items(): - if callable(value) and not name.startswith("_"): - db.register_function(value, name=name) + _register_functions(db, functions) _execute_query( db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols @@ -1695,6 +1685,9 @@ def query( nargs=-1, ) @click.argument("sql") +@click.option( + "--functions", help="Python code defining one or more custom SQL functions" +) @click.option( "--attach", type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), @@ -1741,6 +1734,7 @@ def query( def memory( paths, sql, + functions, attach, flatten, nl, @@ -1854,6 +1848,9 @@ def memory( _load_extensions(db, load_extension) db.register_fts4_bm25() + if functions: + _register_functions(db, functions) + _execute_query( db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols ) @@ -2993,3 +2990,17 @@ def _load_extensions(db, load_extension): if ext == "spatialite" and not os.path.exists(ext): ext = find_spatialite() db.conn.load_extension(ext) + + +def _register_functions(db, functions): + # Register any Python functions as SQL functions: + sqlite3.enable_callback_tracebacks(True) + globals = {} + try: + exec(functions, globals) + except SyntaxError as ex: + raise click.ClickException("Error in functions definition: {}".format(ex)) + # Register all callables in the locals dict: + for name, value in globals.items(): + if callable(value) and not name.startswith("_"): + db.register_function(value, name=name) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index aee74e7..bb50d23 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -289,3 +289,12 @@ def test_memory_two_files_with_same_stem(tmpdir): ");\n" "CREATE VIEW t2 AS select * from [data_2];\n" ) + + +def test_memory_functions(): + result = CliRunner().invoke( + cli.cli, + ["memory", "select hello()", "--functions", "hello = lambda: 'Hello'"], + ) + assert result.exit_code == 0 + assert result.output.strip() == '[{"hello()": "Hello"}]' From 23ef1d6c20f6a8ef0db508b9711ae0d8ed6a4156 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:10:43 -0700 Subject: [PATCH 5/8] bulk --functions, closes #471 --- docs/cli-reference.rst | 1 + sqlite_utils/cli.py | 8 ++++++++ tests/test_cli_bulk.py | 8 +++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 9a025ba..c194967 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -349,6 +349,7 @@ See :ref:`cli_bulk`. Options: --batch-size INTEGER Commit every X records + --functions TEXT Python code defining one or more custom SQL functions --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 24c8c03..16a99cc 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -926,9 +926,12 @@ def insert_upsert_implementation( load_extension=None, silent=False, bulk_sql=None, + functions=None, ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) + if functions: + _register_functions(db, functions) if (delimiter or quotechar or sniff or no_headers) and not tsv: csv = True if (nl + csv + tsv) >= 2: @@ -1305,6 +1308,9 @@ def upsert( @click.argument("sql") @click.argument("file", type=click.File("rb"), required=True) @click.option("--batch-size", type=int, default=100, help="Commit every X records") +@click.option( + "--functions", help="Python code defining one or more custom SQL functions" +) @import_options @load_extension_option def bulk( @@ -1312,6 +1318,7 @@ def bulk( sql, file, batch_size, + functions, flatten, nl, csv, @@ -1368,6 +1375,7 @@ def bulk( load_extension=load_extension, silent=False, bulk_sql=sql, + functions=functions, ) except (OperationalError, sqlite3.IntegrityError) as e: raise click.ClickException(str(e)) diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 6d01952..909ed09 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -28,9 +28,11 @@ def test_cli_bulk(test_db_and_path): [ "bulk", db_path, - "insert into example (id, name) values (:id, :name)", + "insert into example (id, name) values (:id, myupper(:name))", "-", "--nl", + "--functions", + "myupper = lambda s: s.upper()", ], input='{"id": 3, "name": "Three"}\n{"id": 4, "name": "Four"}\n', ) @@ -38,8 +40,8 @@ def test_cli_bulk(test_db_and_path): assert [ {"id": 1, "name": "One"}, {"id": 2, "name": "Two"}, - {"id": 3, "name": "Three"}, - {"id": 4, "name": "Four"}, + {"id": 3, "name": "THREE"}, + {"id": 4, "name": "FOUR"}, ] == list(db["example"].rows) From a46a5e3a9e03dcdd8c84a92e4a5dbfa02ba461fa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:20:09 -0700 Subject: [PATCH 6/8] Improved code compilation pattern, closes #472 --- docs/cli.rst | 3 --- sqlite_utils/utils.py | 9 ++++----- tests/test_cli_convert.py | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 1632e2f..3c5c595 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1359,12 +1359,9 @@ The following example adds a new ``score`` column, then updates it to list a ran random.seed(10) def convert(value): - global random return random.random() ' -Note the ``global random`` line here. Due to the way the tool compiles Python code, this is necessary to ensure the ``random`` module is available within the ``convert()`` function. If you were to omit this you would see a ``NameError: name 'random' is not defined`` error. - .. _cli_convert_recipes: sqlite-utils convert recipes diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 64bba50..c0b7bf1 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -432,12 +432,11 @@ def progressbar(*args, **kwargs): def _compile_code(code, imports, variable="value"): - locals = {} globals = {"r": recipes, "recipes": recipes} # If user defined a convert() function, return that try: - exec(code, globals, locals) - return locals["convert"] + exec(code, globals) + return globals["convert"] except (AttributeError, SyntaxError, NameError, KeyError, TypeError): pass @@ -464,8 +463,8 @@ def _compile_code(code, imports, variable="value"): for import_ in imports: globals[import_.split(".")[0]] = __import__(import_) - exec(code_o, globals, locals) - return locals["fn"] + exec(code_o, globals) + return globals["fn"] def chunks(sequence: Iterable, size: int) -> Iterable[Iterable]: diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 10b4563..d26ab8c 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -606,3 +606,23 @@ def test_convert_hyphen_workaround(fresh_db_and_path): assert list(db["names"].rows) == [ {"id": 1, "name": "-"}, ] + + +def test_convert_initialization_pattern(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "names", + "name", + "-", + ], + input="import random\nrandom.seed(1)\ndef convert(value): return random.randint(0, 100)", + ) + assert 0 == result.exit_code, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "17"}, + ] From 19dd077944429c1365b513d80cc71c605ae3bed3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:55:47 -0700 Subject: [PATCH 7/8] Support entrypoints for `--load-extension` (#473) * Entrypoint support, closes #470 --- .github/workflows/test.yml | 4 ++ .gitignore | 3 ++ docs/cli-reference.rst | 82 ++++++++++++++++++++------------------ sqlite_utils/cli.py | 8 +++- tests/ext.c | 48 ++++++++++++++++++++++ tests/test_cli.py | 41 +++++++++++++++++++ 6 files changed, 145 insertions(+), 41 deletions(-) create mode 100644 tests/ext.c diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75ef594..9303965 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,10 @@ jobs: - name: Install SpatiaLite if: matrix.os == 'ubuntu-latest' run: sudo apt-get install libsqlite3-mod-spatialite + - name: Build extension for --load-extension test + if: matrix.os == 'ubuntu-latest' + run: |- + (cd tests && gcc ext.c -fPIC -shared -o ext.so && ls -lah) - name: Run tests run: | pytest -v diff --git a/.gitignore b/.gitignore index 32032d5..7947f33 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ venv Pipfile Pipfile.lock pyproject.toml +tests/*.dylib +tests/*.so +tests/*.dll diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c194967..d3951fd 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -121,7 +121,8 @@ See :ref:`cli_query`. -p, --param ... Named :parameters for SQL query --functions TEXT Python code defining one or more custom SQL functions - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint -h, --help Show this message and exit. @@ -189,7 +190,8 @@ See :ref:`cli_memory`. --dump Dump SQL for in-memory database --save FILE Save in-memory database to this file --analyze Analyze resulting tables and output results - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint -h, --help Show this message and exit. @@ -266,7 +268,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr --default ... Default value that should be set for a column -d, --detect-types Detect types for columns in CSV/TSV data --analyze Run ANALYZE at the end of this operation - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint --silent Do not show progress bar --ignore Ignore records if pk already exists --replace Replace records if pk already exists @@ -320,7 +322,7 @@ See :ref:`cli_upsert`. --default ... Default value that should be set for a column -d, --detect-types Detect types for columns in CSV/TSV data --analyze Run ANALYZE at the end of this operation - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint --silent Do not show progress bar -h, --help Show this message and exit. @@ -364,7 +366,7 @@ See :ref:`cli_bulk`. --sniff Detect delimiter and quote character --no-headers CSV file has no header row --encoding TEXT Character encoding for input, defaults to utf-8 - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -404,7 +406,7 @@ See :ref:`cli_search`. textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -440,7 +442,7 @@ See :ref:`cli_transform_table`. --default-none TEXT Remove default from this column --drop-foreign-key TEXT Drop foreign key constraint for this column --sql Output SQL without executing it - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -465,7 +467,7 @@ See :ref:`cli_extract`. --table TEXT Name of the other table to extract columns to --fk-column TEXT Name of the foreign key column to add to the table --rename ... Rename this column in extracted table - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -487,7 +489,7 @@ See :ref:`cli_schema`. sqlite-utils schema trees.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -525,7 +527,7 @@ See :ref:`cli_insert_files`. --text Store file content as TEXT, not BLOB --encoding TEXT Character encoding for input, defaults to utf-8 -s, --silent Don't show a progress bar - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -549,7 +551,7 @@ See :ref:`cli_analyze_tables`. Options: -c, --column TEXT Specific columns to analyze --save Save results to _analyze_tables table - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -656,7 +658,7 @@ See :ref:`cli_tables`. strings --columns Include list of columns for each table --schema Include schema for each table - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -694,7 +696,7 @@ See :ref:`cli_views`. strings --columns Include list of columns for each view --schema Include schema for each view - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -735,7 +737,8 @@ See :ref:`cli_rows`. simple, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint -h, --help Show this message and exit. @@ -770,7 +773,7 @@ See :ref:`cli_triggers`. textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -806,7 +809,7 @@ See :ref:`cli_indexes`. textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -830,7 +833,7 @@ See :ref:`cli_create_database`. Options: --enable-wal Enable WAL mode on the created database --init-spatialite Enable SpatiaLite on the created database - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -862,7 +865,7 @@ See :ref:`cli_create_table`. foreign key --ignore If table already exists, do nothing --replace If table already exists, replace it - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -892,7 +895,7 @@ See :ref:`cli_create_index`. --unique Make this a unique index --if-not-exists, --ignore Ignore if index already exists --analyze Run ANALYZE after creating the index - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -920,7 +923,7 @@ See :ref:`cli_fts`. --create-triggers Create triggers to update the FTS tables when the parent table changes. --replace Replace existing FTS configuration if it exists - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -940,7 +943,7 @@ populate-fts sqlite-utils populate-fts chickens.db chickens name Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -960,7 +963,7 @@ rebuild-fts sqlite-utils rebuild-fts chickens.db chickens Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -980,7 +983,7 @@ disable-fts sqlite-utils disable-fts chickens.db chickens Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1004,7 +1007,7 @@ See :ref:`cli_optimize`. Options: --no-vacuum Don't run VACUUM - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1069,7 +1072,7 @@ See :ref:`cli_dump`. sqlite-utils dump chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1097,7 +1100,7 @@ See :ref:`cli_add_column`. omitted will automatically use the primary key --not-null-default TEXT Add NOT NULL DEFAULT 'TEXT' constraint --ignore If column already exists, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1123,7 +1126,7 @@ See :ref:`cli_add_foreign_key`. Options: --ignore If foreign key already exists, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1147,7 +1150,7 @@ See :ref:`cli_add_foreign_keys`. authors country_id countries id Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1169,7 +1172,7 @@ See :ref:`cli_index_foreign_keys`. sqlite-utils index-foreign-keys chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1191,7 +1194,7 @@ See :ref:`cli_wal`. sqlite-utils enable-wal chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1211,7 +1214,7 @@ disable-wal sqlite-utils disable-wal chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1233,7 +1236,7 @@ See :ref:`cli_enable_counts`. sqlite-utils enable-counts chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1253,7 +1256,7 @@ reset-counts sqlite-utils reset-counts chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1270,7 +1273,7 @@ duplicate Options: --ignore If table does not exist, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1293,7 +1296,7 @@ See :ref:`cli_drop_table`. Options: --ignore If table does not exist, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1318,7 +1321,7 @@ See :ref:`cli_create_view`. Options: --ignore If view already exists, do nothing --replace If view already exists, replace it - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1341,7 +1344,7 @@ See :ref:`cli_drop_view`. Options: --ignore If view does not exist, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1372,7 +1375,8 @@ See :ref:`cli_spatialite`. --dimensions TEXT Coordinate dimensions. Use XYZ for three- dimensional geometries. --not-null Add a NOT NULL constraint. - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint -h, --help Show this message and exit. @@ -1394,7 +1398,7 @@ See :ref:`cli_spatialite_indexes`. paths. To load it from a specific path, use --load-extension. Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 16a99cc..c09273d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -94,7 +94,7 @@ def load_extension_option(fn): return click.option( "--load-extension", multiple=True, - help="SQLite extensions to load", + help="Path to SQLite extension, with optional :entrypoint", )(fn) @@ -2997,7 +2997,11 @@ def _load_extensions(db, load_extension): for ext in load_extension: if ext == "spatialite" and not os.path.exists(ext): ext = find_spatialite() - db.conn.load_extension(ext) + if ":" in ext: + path, _, entrypoint = ext.partition(":") + db.conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + db.conn.load_extension(ext) def _register_functions(db, functions): diff --git a/tests/ext.c b/tests/ext.c new file mode 100644 index 0000000..f5b3276 --- /dev/null +++ b/tests/ext.c @@ -0,0 +1,48 @@ +/* +** This file implements a SQLite extension with multiple entrypoints. +** +** The default entrypoint, sqlite3_ext_init, has a single function "a". +** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b". +** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c". +** +** Compiling instructions: +** https://www.sqlite.org/loadext.html#compiling_a_loadable_extension +** +*/ + +#include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1 + +// SQL function that returns back the value supplied during sqlite3_create_function() +static void func(sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC); +} + + +// The default entrypoint, since it matches the "ext.dylib"/"ext.so" name +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0); +} + +// Alternate entrypoint #1 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0); +} + +// Alternate entrypoint #2 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0); +} diff --git a/tests/test_cli.py b/tests/test_cli.py index 9b43cd5..24f36ed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ from sqlite_utils import cli, Database from sqlite_utils.db import Index, ForeignKey from click.testing import CliRunner +from pathlib import Path import subprocess import sys from unittest import mock @@ -21,6 +22,17 @@ def _supports_pragma_function_list(): return True +def _has_compiled_ext(): + for ext in ["dylib", "so", "dll"]: + path = Path(__file__).parent / f"ext.{ext}" + if path.is_file(): + return True + return False + + +COMPILED_EXTENSION_PATH = str(Path(__file__).parent / "ext") + + @pytest.mark.parametrize( "options", ( @@ -2284,3 +2296,32 @@ def test_duplicate_table(tmpdir): assert result.exit_code == 0 assert db["one"].columns_dict == db["two"].columns_dict assert list(db["one"].rows) == list(db["two"].rows) + + +@pytest.mark.skipif(not _has_compiled_ext(), reason="Requires compiled ext.c") +@pytest.mark.parametrize( + "entrypoint,should_pass,should_fail", + ( + (None, ("a",), ("b", "c")), + ("sqlite3_ext_b_init", ("b"), ("a", "c")), + ("sqlite3_ext_c_init", ("c"), ("a", "b")), + ), +) +def test_load_extension(entrypoint, should_pass, should_fail): + ext = COMPILED_EXTENSION_PATH + if entrypoint: + ext += ":" + entrypoint + for func in should_pass: + result = CliRunner().invoke( + cli.cli, + ["memory", "select {}()".format(func), "--load-extension", ext], + catch_exceptions=False, + ) + assert result.exit_code == 0 + for func in should_fail: + result = CliRunner().invoke( + cli.cli, + ["memory", "select {}()".format(func), "--load-extension", ext], + catch_exceptions=False, + ) + assert result.exit_code == 1 From c5f8a2eb1a81a18b52825cc649112f71fe419b12 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Sat, 27 Aug 2022 10:45:03 -0400 Subject: [PATCH 8/8] in extract code, check equality witH IS instead of = for nulls (#455) sqlite "IS" is equivalent to SQL "IS NOT DISTINCT FROM" close #423 --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 18a442a..25c0a70 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1791,7 +1791,7 @@ class Table(Queryable): magic_lookup_column=magic_lookup_column, lookup_table=table, where=" AND ".join( - "[{table}].[{column}] = [{lookup_table}].[{lookup_column}]".format( + "[{table}].[{column}] IS [{lookup_table}].[{lookup_column}]".format( table=self.name, lookup_table=table, column=column,