remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-09-16 05:24:26 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'main' into fix-transform-keyword-literal-defaults

Browse source
This commit is contained in:
Simon Willison 2026-07-05 22:26:33 -07:00
commit 750686c6d1
44 changed files with 3440 additions and 376 deletions
Download patch file Download diff file Expand all files Collapse all files

39
.github/actions/setup-sqlite-version/action.yml vendored Normal file
View file

@ -0,0 +1,39 @@
name: "Setup SQLite version"
description: "Build and activate a specific SQLite version from its amalgamation archive"
inputs:
version:
description: "The SQLite version to install"
required: true
cflags:
description: "CFLAGS to use when compiling SQLite"
required: false
default: ""
skip-activate:
description: "Set to true to skip modifying the library path"
required: false
default: "false"
fallback-urls:
description: "Whitespace-separated fallback download URLs to try after sqlite.org"
required: false
default: ""
outputs:
sqlite-location:
description: "Directory containing the compiled SQLite library"
value: ${{ steps.build.outputs.sqlite-location }}
runs:
using: "composite"
steps:
- shell: bash
run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads"
- uses: actions/cache@v6
with:
path: ${{ runner.temp }}/sqlite-versions/downloads
key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1
- id: build
shell: bash
run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh"
env:
SQLITE_VERSION: ${{ inputs.version }}
SQLITE_CFLAGS: ${{ inputs.cflags }}
SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }}
SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }}

144
.github/actions/setup-sqlite-version/setup-sqlite-version.sh vendored Normal file
View file

@ -0,0 +1,144 @@
#!/usr/bin/env bash
set -euo pipefail
version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}"
cflags="${SQLITE_CFLAGS:-}"
skip_activate="${SQLITE_SKIP_ACTIVATE:-false}"
extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}"
case "$version_spec" in
3.46 | 3.46.0)
sqlite_version="3.46.0"
sqlite_year="2024"
amalgamation_id="3460000"
builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip"
;;
3.23.1)
sqlite_version="3.23.1"
sqlite_year="2018"
amalgamation_id="3230100"
builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3230100.zip"
;;
*)
echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh."
exit 1
;;
esac
case "$(uname -s)" in
Linux)
library_name="libsqlite3.so.0"
library_path_var="LD_LIBRARY_PATH"
;;
Darwin)
library_name="libsqlite3.dylib"
library_path_var="DYLD_LIBRARY_PATH"
;;
*)
echo "::error::Unsupported platform $(uname -s)"
exit 1
;;
esac
runner_temp="${RUNNER_TEMP:-}"
if [ -z "$runner_temp" ]; then
runner_temp="$(mktemp -d)"
fi
filename="sqlite-amalgamation-${amalgamation_id}"
official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip"
download_dir="${runner_temp}/sqlite-versions/downloads"
source_root="${runner_temp}/sqlite-versions/source"
source_dir="${source_root}/${filename}"
build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}"
archive_path="${download_dir}/${filename}.zip"
mkdir -p "$download_dir" "$source_root" "$build_dir"
download_archive() {
local url
local candidate_path="${archive_path}.tmp"
local urls=("$official_url")
for url in $builtin_fallback_urls $extra_fallback_urls; do
urls+=("$url")
done
rm -f "$candidate_path"
for url in "${urls[@]}"; do
echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}"
if curl \
--fail \
--location \
--show-error \
--retry 5 \
--retry-delay 2 \
--retry-max-time 180 \
--retry-all-errors \
--connect-timeout 20 \
--max-time 240 \
--output "$candidate_path" \
"$url"; then
mv "$candidate_path" "$archive_path"
return 0
fi
echo "::warning::Download failed from ${url}"
rm -f "$candidate_path"
done
echo "::error::Could not download SQLite ${sqlite_version} amalgamation"
return 1
}
if [ ! -f "${source_dir}/sqlite3.c" ]; then
if [ ! -f "$archive_path" ]; then
download_archive
fi
rm -rf "$source_dir"
unzip -q "$archive_path" -d "$source_root"
fi
if [ ! -f "${source_dir}/sqlite3.c" ]; then
echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}"
exit 1
fi
read -r -a cflag_args <<< "$cflags"
echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}"
gcc \
-fPIC \
-shared \
"${cflag_args[@]}" \
"${source_dir}/sqlite3.c" \
"-I${source_dir}" \
-o "${build_dir}/${library_name}"
if [ "$library_name" = "libsqlite3.so.0" ]; then
ln -sf "$library_name" "${build_dir}/libsqlite3.so"
fi
if [ -n "${GITHUB_OUTPUT:-}" ]; then
echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT"
else
echo "sqlite-location=${build_dir}"
fi
case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in
true | 1 | yes)
echo "Skipping ${library_path_var} activation"
;;
*)
existing_value="${!library_path_var:-}"
if [ -n "${GITHUB_ENV:-}" ]; then
if [ -n "$existing_value" ]; then
echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV"
else
echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV"
fi
fi
echo "Added ${build_dir} to ${library_path_var}"
;;
esac

6
.github/workflows/test-sqlite-support.yml vendored
View file

@ -18,16 +18,16 @@ jobs:
"3.23.1", # 2018-04-10, before UPSERT
]
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
allow-prereleases: true
cache: pip
cache-dependency-path: pyproject.toml
- name: Set up SQLite ${{ matrix.sqlite-version }}
uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6
uses: ./.github/actions/setup-sqlite-version
with:
version: ${{ matrix.sqlite-version }}
cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1"

6
.github/workflows/test.yml vendored
View file

@ -31,9 +31,6 @@ jobs:
- name: Install SpatiaLite
if: matrix.os == 'ubuntu-latest'
run: sudo apt-get install libsqlite3-mod-spatialite
- name: On macOS with Python 3.10 test with sqlean.py
if: matrix.os == 'macos-latest' && matrix.python-version == '3.10'
run: pip install sqlean.py sqlite-dump
- name: Build extension for --load-extension test
if: matrix.os == 'ubuntu-latest'
run: |-
@ -41,6 +38,9 @@ jobs:
- name: Run tests
run: |
pytest -v
- name: Run autocommit tests just on 3.14/Ubuntu
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14'
run: pytest --sqlite-autocommit
- name: run mypy
run: mypy sqlite_utils tests
- name: run flake8

3
Justfile
View file

@ -8,11 +8,12 @@
@run *options:
uv run -- {{options}}
# Run linters: black, flake8, mypy, cog
# Run linters: black, flake8, mypy, ty, cog
@lint:
just run black . --check
uv run flake8
uv run mypy sqlite_utils tests
uv run ty check sqlite_utils
uv run cog --check README.md docs/*.rst
uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt

65
docs/changelog.rst
View file

@ -4,6 +4,71 @@
Changelog
===========
.. _v_unreleased:
Unreleased
----------
Breaking changes:
- ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`)
- Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library.
- The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`)
Compound foreign key support:
- Tables can now be created with compound foreign keys, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`.
- ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely.
- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table.
- ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key.
Other foreign key improvements:
- ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema.
- ``table.add_foreign_key()`` accepts new ``on_delete=`` and ``on_update=`` parameters for creating foreign keys with actions, e.g. ``table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")``. (:issue:`530`)
- Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``.
- Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound.
Case-insensitive column matching:
Column names passed to Python API methods are now matched against the table schema case-insensitively, mirroring how SQLite itself treats identifiers. Previously many methods accepted mixed-case identifiers in the SQL they generated but then failed - or silently did nothing - when performing Python-side comparisons against the schema. (:issue:`760`) Fixes include:
- ``table.insert()`` and ``table.upsert()`` now populate ``table.last_pk`` correctly when the ``pk=`` argument uses different casing to the table schema or the record keys - previously this raised a ``KeyError`` after the row had already been written.
- Upserts no longer raise or misbehave when the casing of ``pk=`` differs from the casing of the record keys. The primary key columns are correctly excluded from the generated ``DO UPDATE SET`` clause.
- ``table.transform()`` arguments ``types=``, ``rename=``, ``drop=``, ``pk=``, ``not_null=``, ``defaults=``, ``column_order=`` and ``drop_foreign_keys=`` all resolve column names case-insensitively. Previously options like ``rename={"name": "title"}`` against a column called ``Name`` were silently ignored.
- ``db.create_table(..., transform=True)`` now recognizes existing columns that differ only by case, instead of attempting to add them again and failing with ``duplicate column name``. The casing used in the existing schema is preserved.
- ``table.lookup()`` returns the primary key value even if ``pk=`` casing differs from the schema, and recognizes existing unique indexes case-insensitively instead of creating redundant ones.
- ``table.extract()`` and ``table.convert()`` - including ``multi=True`` and ``output=`` - accept column names in any casing.
- Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive.
- ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``.
.. _v4_0rc2:
4.0rc2 (2026-07-04)
-------------------
Breaking changes:
- Write statements executed with ``db.execute()`` are now committed automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that stayed open until something committed it - writes appeared to work when read on the same connection but were silently rolled back when the connection closed. Code that relied on rolling back uncommitted ``db.execute()`` writes should use the new ``db.begin()`` method to open an explicit transaction first. The transaction model is documented in full at :ref:`python_api_transactions`.
- ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` are executed and committed immediately without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. A statement rejected this way is rolled back before the error is raised, so it has no effect on the database.
- Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead.
- ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place.
- ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions.
- The ``View`` class no longer has an ``enable_fts()`` method. It existed only to raise ``NotImplementedError``, since full-text search is not supported for views - calling it now raises ``AttributeError`` instead, and the method no longer appears in the API reference. The ``sqlite-utils enable-fts`` command shows a clean error when pointed at a view.
- The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection.
- ``Database()`` now raises a ``sqlite_utils.db.TransactionError`` if passed a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options. ``commit()`` and ``rollback()`` behave differently on those connections, which previously caused every write made by the library to be silently discarded when the connection closed.
Everything else:
- Fixed a bug where ``table.delete_where()``, ``table.optimize()`` and ``table.rebuild_fts()`` did not commit their changes, leaving the connection inside an open transaction. Their work - and any subsequent writes - could then be silently rolled back when the connection was closed. All three now use ``db.atomic()``, consistent with the other write methods.
- The ``sqlite-utils drop-table`` command now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. Both now exit with an error suggesting the correct command to use.
- Migrations applied by the new :ref:`migrations system <migrations>` now run inside a transaction, together with the record of the migration having been applied. If a migration raises an exception its changes are rolled back and it stays pending, so it can be safely re-applied after the error is fixed. Migrations that cannot run inside a transaction, such as those executing ``VACUUM``, can opt out using ``@migrations(transactional=False)`` - see :ref:`migrations_transactions`.
- ``table.upsert()`` and ``table.upsert_all()`` now detect the primary key or compound primary key of an existing table, so the ``pk=`` argument is no longer required when upserting into a table that already has a primary key.
- ``db.table(table_name).insert({})`` can now be used to insert a row consisting entirely of default values into an existing table, using ``INSERT INTO ... DEFAULT VALUES``. (:issue:`759`)
- Improvements to the ``sqlite-utils migrate`` command: ``--stop-before`` values that do not match any known migration are now an error instead of being silently ignored, ``--stop-before`` now works correctly with migration files that still use the older ``sqlite_migrate.Migrations`` class, and ``--list`` is now a read-only operation that no longer creates the database file or the migrations tracking table. ``migrations.applied()`` now returns migrations in the order they were applied.
- New ``db.begin()``, ``db.commit()`` and ``db.rollback()`` methods for taking manual control of transactions, as an alternative to the ``db.atomic()`` context manager.
- New documentation: :ref:`python_api_transactions` describes how transactions work and when changes are committed, and a new :ref:`upgrading` page details the changes needed to move between major versions.
.. _v4_0rc1:
4.0rc1 (2026-06-21)

2
docs/cli-reference.rst
View file

@ -286,7 +286,6 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr
--alter Alter existing table to add any missing columns
--not-null TEXT Columns that should be created as NOT NULL
--default <TEXT TEXT>... Default value that should be set for a column
-d, --detect-types Detect types for columns in CSV/TSV data (default)
--no-detect-types Treat all CSV/TSV columns as TEXT
--analyze Run ANALYZE at the end of this operation
--load-extension TEXT Path to SQLite extension, with optional :entrypoint
@ -344,7 +343,6 @@ See :ref:`cli_upsert`.
--alter Alter existing table to add any missing columns
--not-null TEXT Columns that should be created as NOT NULL
--default <TEXT TEXT>... Default value that should be set for a column
-d, --detect-types Detect types for columns in CSV/TSV data (default)
--no-detect-types Treat all CSV/TSV columns as TEXT
--analyze Run ANALYZE at the end of this operation
--load-extension TEXT Path to SQLite extension, with optional :entrypoint

2
docs/cli.rst
View file

@ -45,6 +45,8 @@ The default format returned for queries is JSON:
[{"id": 1, "age": 4, "name": "Cleo"},
{"id": 2, "age": 2, "name": "Pancakes"}]
If the query returns more than one column with the same name, later occurrences are renamed with a numeric suffix - ``select 1 as id, 2 as id`` returns ``[{"id": 1, "id_2": 2}]``. This only applies to JSON output: :ref:`CSV and TSV <cli_query_csv>` and :ref:`table <cli_query_table>` output keep the duplicate column headers unchanged.
.. _cli_query_nl:
Newline-delimited JSON

1
docs/index.rst
View file

@ -40,5 +40,6 @@ Contents
plugins
reference
cli-reference
upgrading
contributing
changelog

10
docs/installation.rst
View file

@ -52,15 +52,15 @@ On some platforms the ability to load additional extensions (via ``conn.load_ext
You may also see the error ``sqlite3.OperationalError: table sqlite_master may not be modified`` when trying to alter an existing table.
You can work around these limitations by installing either the `pysqlite3 <https://pypi.org/project/pysqlite3/>`__ package or the `sqlean.py <https://pypi.org/project/sqlean.py/>`__ package, both of which provide drop-in replacements for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions.
You can work around these limitations by installing the `pysqlite3 <https://pypi.org/project/pysqlite3/>`__ package, which provides a drop-in replacement for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions.
To install ``sqlean.py`` (which has compiled binary wheels available for all major platforms) run the following:
To install ``pysqlite3`` run the following:
.. code-block:: bash
sqlite-utils install sqlean.py
sqlite-utils install pysqlite3
``pysqlite3`` and ``sqlean.py`` do not provide implementations of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package:
``pysqlite3`` does not provide an implementation of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package:
.. code-block:: bash
@ -87,4 +87,4 @@ For ``zsh``:
Add this code to ``~/.zshrc`` or ``~/.bashrc`` to automatically run it when you start a new shell.
See `the Click documentation <https://click.palletsprojects.com/en/8.1.x/shell-completion/>`__ for more details.
See `the Click documentation <https://click.palletsprojects.com/en/8.1.x/shell-completion/>`__ for more details.

23
docs/migrations.rst
View file

@ -70,6 +70,25 @@ When you apply a set of migrations you can stop part way through by specifying a
migrations.apply(db, stop_before="add_weight")
.. _migrations_transactions:
Migrations and transactions
===========================
Each migration runs inside a transaction, together with the ``_sqlite_migrations`` record of it having been applied. If a migration function raises an exception, everything it did is rolled back, no record is written and the migration stays pending - so fixing the error and re-applying will run that migration again from a clean state. Migrations that completed earlier in the same ``apply()`` run stay applied.
Some operations cannot run inside a transaction, for example ``VACUUM`` or changing the journal mode with ``db.enable_wal()``. Register migrations like these with ``transactional=False``:
.. code-block:: python
@migrations(transactional=False)
def compact(db):
db.execute("VACUUM")
A migration registered with ``transactional=False`` runs without a wrapping transaction, so if it fails part way through any changes it already made will not be rolled back, and re-applying will run the whole function again.
Avoid calling ``db.commit()`` or otherwise managing transactions manually inside a transactional migration - register the migration with ``transactional=False`` if it needs to control its own transactions. Using ``with db.atomic():`` blocks inside a migration is fine: they nest as savepoints within the migration's transaction, so the migration as a whole still commits or rolls back as a single unit. See :ref:`python_api_transactions`.
Applying migrations using the CLI
=================================
@ -98,7 +117,7 @@ Running the command repeatedly is safe. Migrations that already have a matching
Listing migrations
==================
Use ``--list`` to show applied and pending migrations without running them:
Use ``--list`` to show applied and pending migrations without running them. This is a read-only operation - it will not create the database file or the ``_sqlite_migrations`` table:
.. code-block:: bash
@ -138,6 +157,8 @@ You can also target a specific migration set using ``migration_set:migration_nam
The ``--stop-before`` option can be passed more than once.
If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything.
Verbose output
==============

7
docs/plugins.rst
View file

@ -140,9 +140,7 @@ As a special niche feature, if your plugin needs to import some files and then a
prepare_connection(conn)
~~~~~~~~~~~~~~~~~~~~~~~~
This hook is called when a new SQLite database connection is created. You can
use it to `register custom SQL functions <https://docs.python.org/2/library/sqlite3.html#sqlite3.Connection.create_function>`_,
aggregates and collations. For example:
This hook is called when a new SQLite database connection is created. You can use it to `register custom SQL functions <https://docs.python.org/2/library/sqlite3.html#sqlite3.Connection.create_function>`_, aggregates and collations. For example:
.. code-block:: python
@ -154,8 +152,7 @@ aggregates and collations. For example:
"hello", 1, lambda name: f"Hello, {name}!"
)
This registers a SQL function called ``hello`` which takes a single
argument and can be called like this:
This registers a SQL function called ``hello`` which takes a single argument and can be called like this:
.. code-block:: sql

295
docs/python-api.rst
View file

@ -33,6 +33,8 @@ Here's how to create a new SQLite database file containing a new ``chickens`` ta
"color": "black",
}])
The inserted rows are saved to the database file straight away - methods like ``insert_all()`` commit their own changes, so no ``commit()`` call is needed. See :ref:`python_api_transactions` for how this works.
You can loop through those rows like this:
.. code-block:: python
@ -93,6 +95,8 @@ Instead of a file path you can pass in an existing SQLite connection:
db = Database(sqlite3.connect("my_database.db"))
The connection must use Python's default transaction handling. Connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options are rejected with a ``sqlite_utils.db.TransactionError`` - see :ref:`python_api_transactions_modes`.
If you want to create an in-memory database, you can do so like this:
.. code-block:: python
@ -144,6 +148,12 @@ The ``Database`` object also works as a context manager, which will automaticall
db["my_table"].insert({"name": "Example"})
# Connection is automatically closed here
Exiting the block is equivalent to calling ``db.close()``: the connection is closed and any transaction still open at that point is rolled back. This matches SQLite's own behavior when a connection closes.
This rarely matters in practice. Everything that writes to the database - including raw ``db.execute()`` statements - commits automatically, so a transaction can only be open here if you explicitly started one with ``db.begin()`` and have not yet committed it. In that case the decision to commit stays with you: committing automatically on exit could silently persist half-finished work, for example if your code returned early from the block. Call ``db.commit()`` when the work is complete.
Note this differs from the ``sqlite3.Connection`` context manager in the standard library, which commits on success but does not close the connection. See :ref:`python_api_transactions` for the full transaction model.
.. _python_api_attach:
Attaching additional databases
@ -219,6 +229,21 @@ The ``db.query(sql)`` function executes a SQL query and returns an iterator over
# {'name': 'Cleo'}
# {'name': 'Pancakes'}
The SQL query is executed as soon as ``db.query()`` is called. The resulting rows are fetched lazily as you iterate, so large result sets are not loaded into memory all at once. Because execution is immediate, an error in your SQL will raise an exception straight away, and a statement such as ``INSERT ... RETURNING`` will take effect - and be committed, unless a transaction is open - even if you do not iterate over its results.
``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. The rejected statement is rolled back, so it has no effect on the database. Use :ref:`db.execute() <python_api_execute>` for those statements instead.
If a query returns more than one column with the same name - a join between two tables that share column names, for example - later occurrences are renamed with a numeric suffix, so every value is included in the dictionary:
.. code-block:: python
row = next(db.query("select 1 as id, 2 as id, 3 as id"))
print(row)
# Outputs:
# {'id': 1, 'id_2': 2, 'id_3': 3}
A suffix that would collide with another column in the query is skipped - ``select 1 as id, 2 as id, 3 as id_2`` returns ``{'id': 1, 'id_3': 2, 'id_2': 3}``. The same renaming is applied by ``table.rows_where()`` and ``table.search()``.
.. _python_api_execute:
db.execute(sql, params)
@ -239,34 +264,8 @@ The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around
Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor>`__.
.. _python_api_atomic:
Transactions with db.atomic()
-----------------------------
Use ``db.atomic()`` to group multiple operations in a transaction:
.. code-block:: python
with db.atomic():
db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id")
db.table("dogs").insert({"id": 2, "name": "Pancakes"})
If an exception is raised, changes made inside the block will be rolled back.
``db.atomic()`` can be nested. Nested blocks use SQLite savepoints, so an exception in an inner block can roll back to that savepoint without rolling back the entire outer transaction:
.. code-block:: python
with db.atomic():
db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id")
try:
with db.atomic():
db.table("dogs").insert({"id": 2, "name": "Pancakes"})
raise ValueError("skip this one")
except ValueError:
pass
db.table("dogs").insert({"id": 3, "name": "Marnie"})
.. note::
Write statements executed this way are committed automatically, unless a transaction is already open in which case they become part of it - see :ref:`python_api_transactions_execute`.
.. _python_api_parameters:
@ -296,6 +295,114 @@ Named parameters using ``:name`` can be filled using a dictionary:
In this example ``next()`` is used to retrieve the first result in the iterator returned by the ``db.query()`` method.
.. _python_api_transactions:
Transactions and saving your changes
====================================
Every method in this library that writes to the database - ``insert()``, ``upsert()``, ``update()``, ``delete()``, ``delete_where()``, ``transform()``, ``create_table()``, ``create_index()``, ``enable_fts()`` and the rest - runs inside its own transaction and commits it before returning. Your changes are saved to disk as soon as the method call finishes:
.. code-block:: python
db = Database("data.db")
db.table("news").insert({"headline": "Dog wins award"})
# The new row is already saved - no commit() required
The same applies to raw SQL executed with :ref:`db.execute() <python_api_transactions_execute>` - a write statement is committed as soon as it has run.
You never need to call ``commit()``, and you do not need to close the database to persist your changes. There are exactly two situations where you need to think about transactions:
1. You want to group several write operations together, so they either all succeed or all fail - use :ref:`db.atomic() <python_api_atomic>`.
2. You are :ref:`managing a transaction yourself <python_api_transactions_manual>` with ``db.begin()``, in which case nothing is committed until you commit - the library will never commit a transaction you opened.
.. _python_api_atomic:
Grouping changes with db.atomic()
---------------------------------
Use ``db.atomic()`` to group multiple operations in a single transaction:
.. code-block:: python
with db.atomic():
db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id")
db.table("dogs").insert({"id": 2, "name": "Pancakes"})
The transaction commits when the block exits. If an exception is raised, changes made inside the block will be rolled back.
``db.atomic()`` can be nested. Nested blocks use SQLite savepoints, so an exception in an inner block can roll back to that savepoint without rolling back the entire outer transaction:
.. code-block:: python
with db.atomic():
db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id")
try:
with db.atomic():
db.table("dogs").insert({"id": 2, "name": "Pancakes"})
raise ValueError("skip this one")
except ValueError:
pass
db.table("dogs").insert({"id": 3, "name": "Marnie"})
The transaction is opened with a deferred ``BEGIN`` - SQLite takes the necessary locks when the first statement inside the block runs.
.. _python_api_transactions_execute:
Raw SQL writes with db.execute()
--------------------------------
Write statements executed with :ref:`db.execute() <python_api_execute>` follow the same rule as everything else: they are committed automatically as soon as they have run.
.. code-block:: python
db.execute("insert into news (headline) values (?)", ["Dog wins award"])
# Already committed
If a transaction is open - because the call happens inside a ``db.atomic()`` block, or after ``db.begin()`` - the statement becomes part of that transaction instead, and commits when the transaction commits:
.. code-block:: python
with db.atomic():
db.execute("insert into news (headline) values (?)", ["Dog wins award"])
db.execute("insert into news (headline) values (?)", ["Cat unimpressed"])
# Both rows committed together
One corner case: a row-returning write such as ``INSERT ... RETURNING`` executed through ``db.execute()`` cannot be auto-committed, because its rows have not been read yet - call ``db.commit()`` after fetching them, or use :ref:`db.query() <python_api_query>` for those statements, which executes the write and commits it immediately.
.. _python_api_transactions_manual:
Managing transactions yourself
------------------------------
You can take full manual control using the ``db.begin()``, ``db.commit()`` and ``db.rollback()`` methods:
.. code-block:: python
db.begin()
db.table("news").insert({"headline": "Dog wins award"})
if all_looks_good:
db.commit()
else:
db.rollback()
``db.begin()`` raises ``sqlite3.OperationalError`` if a transaction is already open. ``db.commit()`` and ``db.rollback()`` do nothing if there is no open transaction.
The library will never commit a transaction you opened. If you call write methods such as ``insert()`` - or use ``db.atomic()`` - while your transaction is open, they participate in it using SQLite savepoints instead of committing: exiting an ``atomic()`` block releases its savepoint, but nothing is saved to disk until you commit the outer transaction yourself. If you roll back, their changes are rolled back too.
Two related safeguards to be aware of:
- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect.
- Closing the database - explicitly with ``db.close()``, or by exiting a ``with Database(...) as db:`` block - rolls back any transaction that is still open, see :ref:`python_api_close`.
.. _python_api_transactions_modes:
Supported connection modes
--------------------------
``db.atomic()`` and the automatic per-method transactions require a connection in Python's default transaction handling mode. Passing a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options to ``Database()`` raises a ``sqlite_utils.db.TransactionError``.
This is because ``commit()`` and ``rollback()`` behave differently on those connections - under ``autocommit=True`` they are documented no-ops - which would cause every write made by this library to be silently discarded when the connection closed, rather than failing loudly.
.. _python_api_table:
Accessing tables
@ -721,6 +828,61 @@ You can leave off the third item in the tuple to have the referenced column auto
("author_id", "authors")
])
.. _python_api_compound_foreign_keys:
Compound foreign keys
~~~~~~~~~~~~~~~~~~~~~
To create a compound (multi-column) foreign key, use tuples of column names in place of the single column names:
.. code-block:: python
db.table("courses").create({
"course_code": str,
"campus_name": str,
"dept_code": str,
}, pk="course_code", foreign_keys=[
(("campus_name", "dept_code"), "departments", ("campus_name", "dept_code"))
])
This creates a table-level constraint:
.. code-block:: sql
CREATE TABLE "courses" (
"course_code" TEXT PRIMARY KEY,
"campus_name" TEXT,
"dept_code" TEXT,
FOREIGN KEY ("campus_name", "dept_code") REFERENCES "departments"("campus_name", "dept_code")
)
As with single columns, you can leave off the tuple of other columns to reference the compound primary key of the other table:
.. code-block:: python
foreign_keys=[
(("campus_name", "dept_code"), "departments")
]
To specify ``ON DELETE`` or ``ON UPDATE`` actions, pass ``ForeignKey`` objects instead:
.. code-block:: python
from sqlite_utils.db import ForeignKey
db.table("books").create({
"id": int,
"author_id": int,
}, pk="id", foreign_keys=[
ForeignKey(
table="books", column="author_id",
other_table="authors", other_column="id",
on_delete="CASCADE",
)
])
Foreign key actions are preserved by :ref:`table.transform() <python_api_transform>` - prior to sqlite-utils 4.0 they were silently dropped when a table was transformed.
.. _python_api_table_configuration:
Table configuration options
@ -984,8 +1146,7 @@ You can delete all records in a table that match a specific WHERE statement usin
>>> db = sqlite_utils.Database("dogs.db")
>>> # Delete every dog with age less than 3
>>> with db.atomic():
>>> db.table("dogs").delete_where("age < ?", [3])
>>> db.table("dogs").delete_where("age < ?", [3])
Calling ``table.delete_where()`` with no other arguments will delete every row in the table.
@ -1018,6 +1179,8 @@ Note that the ``pk`` and ``column_order`` parameters here are optional if you ar
An ``upsert_all()`` method is also available, which behaves like ``insert_all()`` but performs upserts instead.
Every record passed to ``upsert()`` or ``upsert_all()`` must include a value for each primary key column - a record without one could never match an existing row, so a ``sqlite_utils.db.PrimaryKeyRequired`` exception is raised instead of quietly inserting a new row.
.. note::
``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 <https://github.com/simonw/sqlite-utils/issues/66>`__ for details of this change.
@ -1429,6 +1592,26 @@ To ignore the case where the key already exists, use ``ignore=True``:
db.table("books").add_foreign_key("author_id", "authors", "id", ignore=True)
To add a compound foreign key, pass tuples of columns:
.. code-block:: python
db.table("courses").add_foreign_key(
("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")
)
As with single columns, omitting the other columns will use the compound primary key of the other table. ``other_table`` must always be specified for a compound foreign key.
Use ``on_delete=`` and ``on_update=`` to specify ``ON DELETE`` and ``ON UPDATE`` actions for the foreign key:
.. code-block:: python
db.table("books").add_foreign_key(
"author_id", "authors", "id", on_delete="CASCADE"
)
This creates a foreign key with an ``ON DELETE CASCADE`` clause, so deleting an author will also delete their books (provided foreign key enforcement is enabled with ``PRAGMA foreign_keys = ON``). Valid actions are ``"SET NULL"``, ``"SET DEFAULT"``, ``"CASCADE"``, ``"RESTRICT"`` and the default ``"NO ACTION"``.
.. _python_api_add_foreign_keys:
Adding multiple foreign key constraints at once
@ -1458,6 +1641,8 @@ If you want to ensure that every foreign key column in your database has a corre
db.index_foreign_keys()
Compound foreign keys get a single composite index across their columns.
.. _python_api_drop:
Dropping a table or view
@ -1660,6 +1845,16 @@ This example drops two foreign keys - the one from ``places.country`` to ``count
drop_foreign_keys=("country", "continent")
)
A bare column name drops any foreign key that column participates in, including compound foreign keys. To target a compound foreign key precisely, pass a tuple of its columns:
.. code-block:: python
db.table("courses").transform(
drop_foreign_keys=[("campus_name", "dept_code")]
)
Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint.
.. _python_api_transform_sql:
Custom transformations with .transform_sql()
@ -2000,7 +2195,7 @@ The ``db.iterdump()`` method returns a sequence of SQL strings representing a co
This uses the `sqlite3.Connection.iterdump() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.iterdump>`__ method.
If you are using ``pysqlite3`` or ``sqlean.py`` the underlying method may be missing. If you install the `sqlite-dump <https://pypi.org/project/sqlite-dump/>`__ package then the ``db.iterdump()`` method will use that implementation instead:
If you are using ``pysqlite3`` the underlying method may be missing. If you install the `sqlite-dump <https://pypi.org/project/sqlite-dump/>`__ package then the ``db.iterdump()`` method will use that implementation instead:
.. code-block:: bash
@ -2107,17 +2302,39 @@ Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly
.foreign_keys
-------------
The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views.
The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey`` objects. It is not available on views.
Each ``ForeignKey`` has the following attributes:
``table``
The table the foreign key is defined on.
``column``
The column on this table, or ``None`` for a compound foreign key.
``other_table``
The table being referenced.
``other_column``
The referenced column, or ``None`` for a compound foreign key.
``columns``
A tuple of the columns on this table, always populated (a one-item tuple for single-column foreign keys).
``other_columns``
A tuple of the referenced columns.
``is_compound``
``True`` if this is a compound (multi-column) foreign key.
``on_delete``
The ``ON DELETE`` action, e.g. ``"CASCADE"`` - ``"NO ACTION"`` if not set.
``on_update``
The ``ON UPDATE`` action - ``"NO ACTION"`` if not set.
``ForeignKey`` was a ``namedtuple`` prior to sqlite-utils 4.0. It is now a dataclass and can no longer be unpacked or indexed as a tuple - access its fields by name instead. See :ref:`upgrading_3_to_4` for details.
::
>>> db.table("Street_Tree_List").foreign_keys
[ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id'),
ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id'),
ForeignKey(table='Street_Tree_List', column='qSiteInfo', other_table='qSiteInfo', other_column='id'),
ForeignKey(table='Street_Tree_List', column='qSpecies', other_table='qSpecies', other_column='id'),
ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'),
ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')]
[ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=('qLegalStatus',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'),
ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=('qCareAssistant',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'),
...]
Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` tuples.
.. _python_api_introspection_schema:
@ -2697,6 +2914,8 @@ You can disable WAL mode using ``.disable_wal()``:
Database("my_database.db").disable_wal()
The journal mode can only be changed outside of a transaction. Calling either method while a transaction is open - inside a ``db.atomic()`` block, for example - raises a ``sqlite_utils.db.TransactionError``, unless the database is already in the requested mode in which case the call is a no-op.
You can check the current journal mode for a database using the ``journal_mode`` property:
.. code-block:: python

7
docs/reference.rst
View file

@ -70,6 +70,13 @@ sqlite_utils.db.ColumnDetails
.. autoclass:: sqlite_utils.db.ColumnDetails
.. _reference_db_other_foreign_key:
sqlite_utils.db.ForeignKey
--------------------------
.. autoclass:: sqlite_utils.db.ForeignKey
sqlite_utils.utils
==================

144
docs/upgrading.rst Normal file
View file

@ -0,0 +1,144 @@
.. _upgrading:
===========
Upgrading
===========
This page describes the changes you may need to make to your own code or scripts when upgrading between major versions of ``sqlite-utils``.
For the full list of changes in every release see the :ref:`changelog`.
.. _upgrading_3_to_4:
Upgrading from 3.x to 4.0
=========================
Requirements
------------
- Python 3.10 or higher is required.
- The ``click`` dependency must be version 8.3.1 or later.
Command-line changes
--------------------
**Type detection is now the default for CSV and TSV imports.** ``sqlite-utils insert`` and ``sqlite-utils upsert`` now detect column types when importing CSV or TSV data - previously every column was created as ``TEXT`` unless you passed ``--detect-types``. To restore the old behavior pass the new ``--no-detect-types`` flag:
.. code-block:: bash
sqlite-utils insert data.db rows data.csv --csv --no-detect-types
Two related things have been removed:
- The ``SQLITE_UTILS_DETECT_TYPES`` environment variable.
- The old ``-d/--detect-types`` flag itself. Since detection is now the default the flag did nothing - remove it from any scripts that used it.
**The convert command no longer skips falsey values.** ``sqlite-utils convert`` previously skipped values that evaluated to ``False`` (empty strings, ``0``) unless you passed ``--no-skip-false``. All values are now converted and the ``--no-skip-false`` flag has been removed.
**drop-table and drop-view check the object type.** ``sqlite-utils drop-table`` now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. If you relied on that (unlikely), use the matching command instead.
**sqlite-utils tui has moved to a plugin.** The optional terminal interface is now provided by the `sqlite-utils-tui <https://github.com/simonw/sqlite-utils-tui>`__ plugin:
.. code-block:: bash
sqlite-utils install sqlite-utils-tui
Python API changes
------------------
**db.query() now rejects SQL that does not return rows.** This is likely the most common change you will need to make to existing code. ``db.query()`` used to accept any SQL statement - passing one that returns no rows, such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause or a ``CREATE TABLE``, did nothing at all, silently. Those statements now raise a ``ValueError``, and are rolled back so they have no effect on the database. Transaction control statements (``BEGIN``, ``COMMIT``, ``END``, ``ROLLBACK``, ``SAVEPOINT``, ``RELEASE``) plus ``VACUUM``, ``ATTACH`` and ``DETACH`` are also rejected with a ``ValueError``, without being executed at all. Use ``db.execute()`` for statements that do not return rows:
.. code-block:: python
# 3.x accepted this but silently did nothing:
db.query("update dogs set name = 'Cleopaws'")
# In 4.0 use execute() for SQL that does not return rows:
db.execute("update dogs set name = 'Cleopaws'")
**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily, but errors in your SQL raise at the ``db.query()`` call site rather than on first iteration, and a write with a ``RETURNING`` clause takes effect even if you never iterate over its results.
**db.table() no longer returns views.** ``db.table(name)`` now raises a ``sqlite_utils.db.NoTable`` exception if ``name`` is a SQL view. Use the new ``db.view(name)`` method for views:
.. code-block:: python
table = db.table("my_table")
view = db.view("my_view")
``db["name"]`` still returns either a ``Table`` or a ``View`` depending on what exists in the database.
**Upserts use INSERT ... ON CONFLICT.** Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax rather than the previous ``INSERT OR IGNORE`` followed by ``UPDATE``. If your code depends on the old behavior, pass ``use_old_upsert=True`` to the ``Database()`` constructor - see :ref:`python_api_old_upsert`.
**Upsert records must include their primary keys.** ``table.upsert()`` and ``table.upsert_all()`` now raise ``sqlite_utils.db.PrimaryKeyRequired`` if a record is missing a value for any primary key column (or has ``None`` for one). Previously such records were quietly inserted as new rows. Relatedly, ``pk=`` is now optional when the table already exists with a primary key - it is detected automatically.
**Floating point columns are now REAL.** Auto-detected floating point columns are created with the correct SQLite type ``REAL`` instead of ``FLOAT``. Code that inspects column types should expect ``REAL``.
**Generated schemas use double quotes.** Tables created by this library now wrap table and column names in ``"double-quotes"`` where they previously used ``[square-braces]``. If you compare ``table.schema`` strings against expected values you will need to update them.
**table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values.
**ensure_autocommit_off() is now ensure_autocommit_on().** The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``. The old name described the opposite of what the method did: it temporarily puts the connection into driver-level autocommit mode (by setting ``isolation_level = None``), so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. The behavior is unchanged - update any calls to use the new name.
**View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method.
**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained new fields - ``columns``, ``other_columns``, ``is_compound``, ``on_delete`` and ``on_update`` - so that compound (multi-column) foreign keys and foreign key actions can be represented. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead:
.. code-block:: python
# 3.x - tuple unpacking, no longer works:
for table, column, other_table, other_column in db["courses"].foreign_keys:
...
# 4.0 - access fields by name:
for fk in db["courses"].foreign_keys:
fk.table, fk.column, fk.other_table, fk.other_column
Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave.
Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` tuples instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are one-item tuples.
Two related behavior changes to ``table.transform()``: compound foreign keys now survive a transform (previously they were split into separate single-column keys), and ``ON DELETE``/``ON UPDATE`` actions such as ``ON DELETE CASCADE`` are now preserved (previously they were silently stripped from the schema).
**Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``.
**Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() <python_api_atomic>` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice:
- Write statements executed with raw ``db.execute()`` calls now commit automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that nothing committed - if your code used ``db.execute()`` for writes and relied on ``db.conn.rollback()`` to undo them, open an explicit transaction with the new ``db.begin()`` method first.
- Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead.
- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, instead of silently committing it.
- Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - a transaction you explicitly opened with ``db.begin()`` and did not commit is rolled back.
- ``Database()`` rejects connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options, raising ``sqlite_utils.db.TransactionError``. On those connections every write the library made was silently discarded when the connection closed.
Packaging changes
-----------------
- ``sqlite-utils`` now uses ``pyproject.toml`` in place of ``setup.py``.
- ``pip`` is now a runtime dependency, used by the ``sqlite-utils install`` and ``uninstall`` commands.
New features to be aware of
---------------------------
Not breaking changes, but new in 4.0 and worth knowing about when you upgrade:
- A :ref:`database migrations system <migrations>`, incorporating the functionality of the ``sqlite-migrate`` plugin. If you used that plugin, the built-in system reads the same ``_sqlite_migrations`` table - your applied migrations will not run again. Update your migration files to use ``from sqlite_utils import Migrations``.
- :ref:`db.atomic() <python_api_atomic>` for nested transaction support.
- ``table.insert_all()`` and ``table.upsert_all()`` accept an iterator of lists or tuples as an alternative to dictionaries - see :ref:`python_api_insert_lists`.
.. _upgrading_2_to_3:
Upgrading from 2.x to 3.0
=========================
The 3.0 release redesigned search. The breaking changes were minor:
- ``table.search()`` returns a generator of dictionaries, sorted by relevance. It previously returned a list of tuples sorted by ``rowid``.
- The ``-c`` shortcut for ``--csv`` and the ``-f`` shortcut for ``--fmt`` were removed from the CLI - use the full option names.
.. _upgrading_1_to_2:
Upgrading from 1.x to 2.0
=========================
The 2.0 release changed the meaning of *upsert*. In 1.x, ``table.upsert()`` and ``table.upsert_all()`` actually performed ``INSERT OR REPLACE`` operations - entirely replacing the existing row. Since 2.0 an upsert updates only the columns you provide, leaving other columns untouched.
If you want the 1.x behavior, use ``table.insert(..., replace=True)`` or ``table.insert_all(..., replace=True)`` instead.

3
mypy.ini
View file

@ -16,9 +16,6 @@ ignore_errors = True
[mypy-pysqlite3.*]
ignore_missing_imports = True
[mypy-sqlean.*]
ignore_missing_imports = True
[mypy-sqlite_dump.*]
ignore_missing_imports = True

5
pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "sqlite-utils"
version = "4.0rc1"
version = "4.0rc2"
description = "CLI tool and Python library for manipulating SQLite databases"
readme = { file = "README.md", content-type = "text/markdown" }
authors = [
@ -72,7 +72,8 @@ CI = "https://github.com/simonw/sqlite-utils/actions"
sqlite-utils = "sqlite_utils.cli:cli"
[build-system]
requires = ["setuptools"]
# setuptools 77+ is needed for the PEP 639 license = "Apache-2.0" expression
requires = ["setuptools>=77"]
build-backend = "setuptools.build_meta"
[tool.flake8]

93
sqlite_utils/cli.py
View file

@ -14,6 +14,7 @@ from sqlite_utils.db import (
DEFAULT,
DescIndex,
NoTable,
NoView,
quote_identifier,
)
from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins
@ -33,6 +34,7 @@ from .utils import (
OperationalError,
_compile_code,
chunks,
dedupe_keys,
file_progress,
find_spatialite,
flatten as _flatten,
@ -701,14 +703,14 @@ def enable_fts(
_register_db_for_cleanup(db)
_load_extensions(db, load_extension)
try:
db[table].enable_fts(
db.table(table).enable_fts(
column,
fts_version=fts_version,
tokenize=tokenize,
create_triggers=create_triggers,
replace=replace,
)
except OperationalError as ex:
except (NoTable, OperationalError) as ex:
raise click.ClickException(str(ex))
@ -939,12 +941,6 @@ def insert_upsert_options(*, require_pk=False):
type=(str, str),
help="Default value that should be set for a column",
),
click.option(
"-d",
"--detect-types",
is_flag=True,
help="Detect types for columns in CSV/TSV data (default)",
),
click.option(
"--no-detect-types",
is_flag=True,
@ -999,7 +995,6 @@ def insert_upsert_implementation(
truncate=False,
not_null=None,
default=None,
detect_types=None,
no_detect_types=False,
analyze=False,
load_extension=None,
@ -1066,7 +1061,7 @@ def insert_upsert_implementation(
)
else:
docs = (dict(zip(headers, row)) for row in reader)
# detect_types is now the default, unless --no-detect-types is passed
# Type detection is the default, unless --no-detect-types is passed
if not no_detect_types:
tracker = TypeTracker()
docs = tracker.wrap(docs)
@ -1146,7 +1141,7 @@ def insert_upsert_implementation(
else:
doc_chunks = [docs]
for doc_chunk in doc_chunks:
with db.conn:
with db.atomic():
db.conn.cursor().executemany(bulk_sql, doc_chunk)
return
@ -1154,6 +1149,8 @@ def insert_upsert_implementation(
db.table(table).insert_all(
docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs
)
except NoTable as e:
raise click.ClickException(str(e))
except Exception as e:
if (
isinstance(e, OperationalError)
@ -1238,7 +1235,6 @@ def insert(
batch_size,
stop_after,
alter,
detect_types,
no_detect_types,
analyze,
load_extension,
@ -1321,7 +1317,6 @@ def insert(
ignore=ignore,
replace=replace,
truncate=truncate,
detect_types=detect_types,
no_detect_types=no_detect_types,
analyze=analyze,
load_extension=load_extension,
@ -1360,7 +1355,6 @@ def upsert(
alter,
not_null,
default,
detect_types,
no_detect_types,
analyze,
load_extension,
@ -1406,7 +1400,6 @@ def upsert(
upsert=True,
not_null=not_null,
default=default,
detect_types=detect_types,
no_detect_types=no_detect_types,
analyze=analyze,
load_extension=load_extension,
@ -1494,7 +1487,6 @@ def bulk(
upsert=False,
not_null=set(),
default={},
detect_types=False,
no_detect_types=True,
load_extension=load_extension,
silent=False,
@ -1725,7 +1717,13 @@ def drop_table(path, table, ignore, load_extension):
_register_db_for_cleanup(db)
_load_extensions(db, load_extension)
try:
db[table].drop(ignore=ignore)
db.table(table).drop(ignore=ignore)
except NoTable:
# A view exists with this name
if not ignore:
raise click.ClickException(
'"{}" is a view, not a table - use drop-view to drop it'.format(table)
)
except OperationalError:
raise click.ClickException('Table "{}" does not exist'.format(table))
@ -1797,8 +1795,14 @@ def drop_view(path, view, ignore, load_extension):
_register_db_for_cleanup(db)
_load_extensions(db, load_extension)
try:
db[view].drop(ignore=ignore)
except OperationalError:
db.view(view).drop(ignore=ignore)
except NoView:
if ignore:
return
if view in db.table_names():
raise click.ClickException(
'"{}" is a table, not a view - use drop-table to drop it'.format(view)
)
raise click.ClickException('View "{}" does not exist'.format(view))
@ -3374,24 +3378,57 @@ def migrate(db_path, migrations, stop_before, list_, verbose):
if not migration_sets:
raise click.ClickException("No migrations.py files found")
db = sqlite_utils.Database(db_path)
_register_db_for_cleanup(db)
if list_:
if pathlib.Path(db_path).exists():
db = sqlite_utils.Database(db_path)
else:
# Listing is read-only - don't create the database file
db = sqlite_utils.Database(memory=True)
_register_db_for_cleanup(db)
_display_migration_list(db, migration_sets)
return
db = sqlite_utils.Database(db_path)
_register_db_for_cleanup(db)
prev_schema = db.schema
if verbose:
click.echo("Migrating {}".format(db_path))
click.echo("\nSchema before:\n")
click.echo(textwrap.indent(prev_schema, " ") or " (empty)")
click.echo()
if stop_before:
# Every --stop-before value must match at least one known migration
known_names = set()
for migration_set in migration_sets:
names = {m.name for m in migration_set.pending(db)}
names.update(m.name for m in migration_set.applied(db))
known_names.update(names)
known_names.update(
"{}:{}".format(migration_set.name, name) for name in names
)
unknown = [value for value in stop_before if value not in known_names]
if unknown:
raise click.ClickException(
"--stop-before did not match any migrations: {}".format(
", ".join(unknown)
)
)
for migration_set in migration_sets:
migration_set.apply(
db,
stop_before=_stop_before_for_migration_set(stop_before, migration_set.name),
)
matches = _stop_before_for_migration_set(stop_before, migration_set.name)
if isinstance(migration_set, sqlite_utils.Migrations):
migration_set.apply(db, stop_before=matches)
else:
# Legacy sqlite-migrate Migrations objects take a single string
# for stop_before, not a list
distinct = list(dict.fromkeys(matches))
if len(distinct) > 1:
raise click.ClickException(
"Migration set '{}' uses the older sqlite-migrate class, "
"which only supports a single --stop-before value - "
"got: {}".format(migration_set.name, ", ".join(distinct))
)
migration_set.apply(db, stop_before=distinct[0] if distinct else None)
if verbose:
click.echo("Schema after:\n")
post_schema = db.schema
@ -3457,6 +3494,10 @@ FILE_COLUMNS = {
def output_rows(iterator, headers, nl, arrays, json_cols):
# Duplicate column names would collide as dictionary keys, so rename
# later occurrences id, id -> id, id_2 - CSV and table output keep
# the original duplicate headers since they never build dictionaries
headers = dedupe_keys(headers)
# We have to iterate two-at-a-time so we can know if we
# should output a trailing comma or if we have reached
# the last row.

1113
sqlite_utils/db.py
View file

File diff suppressed because it is too large Load diff

75
sqlite_utils/migrations.py
View file

@ -14,11 +14,14 @@ class Migrations:
class _Migration:
name: str
fn: Callable
transactional: bool = True
@dataclass
class _AppliedMigration:
name: str
applied_at: datetime.datetime
# A string timestamp such as "2026-07-04 12:00:00.000000+00:00" -
# stored as TEXT in the _sqlite_migrations table
applied_at: str
def __init__(self, name: str):
"""
@ -27,15 +30,29 @@ class Migrations:
self.name = name
self._migrations: list[Migrations._Migration] = []
def __call__(self, *, name: str | None = None) -> Callable:
def __call__(
self, *, name: str | None = None, transactional: bool = True
) -> Callable:
"""
:param name: The name to use for this migration - if not provided,
the name of the function will be used.
:param transactional: If ``True`` (the default) the migration and the
record of it having been applied are wrapped in a transaction, which
will be rolled back if the migration raises an exception. Pass
``False`` for migrations that cannot run inside a transaction, for
example those that execute ``VACUUM``.
"""
def inner(func: Callable) -> Callable:
migration_name = name or getattr(func, "__name__")
if any(m.name == migration_name for m in self._migrations):
raise ValueError(
"Migration '{}' is already registered in set '{}'".format(
migration_name, self.name
)
)
self._migrations.append(
self._Migration(name or getattr(func, "__name__"), func)
self._Migration(migration_name, func, transactional)
)
return func
@ -44,14 +61,10 @@ class Migrations:
def pending(self, db: "Database") -> list["Migrations._Migration"]:
"""
Return a list of pending migrations.
This is a read-only operation - it does not write to the database.
"""
self.ensure_migrations_table(db)
already_applied = {
r["name"]
for r in db[self.migrations_table].rows_where(
"migration_set = ?", [self.name]
)
}
already_applied = {migration.name for migration in self.applied(db)}
return [
migration
for migration in self._migrations
@ -60,19 +73,29 @@ class Migrations:
def applied(self, db: "Database") -> list["Migrations._AppliedMigration"]:
"""
Return a list of applied migrations.
Return a list of applied migrations, in the order they were applied.
This is a read-only operation - it does not write to the database.
"""
self.ensure_migrations_table(db)
table = _table(db, self.migrations_table)
if not table.exists():
return []
return [
self._AppliedMigration(name=row["name"], applied_at=row["applied_at"])
for row in db[self.migrations_table].rows_where(
"migration_set = ?", [self.name]
for row in table.rows_where(
"migration_set = ?", [self.name], order_by="rowid"
)
]
def apply(self, db: "Database", *, stop_before: str | Iterable[str] | None = None):
"""
Apply any pending migrations to the database.
Each migration runs inside a transaction, together with the record of
it having been applied - if the migration raises an exception its
changes are rolled back, no record is written and the migration stays
pending. Migrations registered with ``transactional=False`` run
outside of a transaction.
"""
self.ensure_migrations_table(db)
if stop_before is None:
@ -85,14 +108,22 @@ class Migrations:
name = migration.name
if name in stop_before_names:
return
migration.fn(db)
_table(db, self.migrations_table).insert(
{
"migration_set": self.name,
"name": name,
"applied_at": str(datetime.datetime.now(datetime.timezone.utc)),
}
)
if migration.transactional:
with db.atomic():
migration.fn(db)
self._record_applied(db, name)
else:
migration.fn(db)
self._record_applied(db, name)
def _record_applied(self, db: "Database", name: str):
_table(db, self.migrations_table).insert(
{
"migration_set": self.name,
"name": name,
"applied_at": str(datetime.datetime.now(datetime.timezone.utc)),
}
)
def ensure_migrations_table(self, db: "Database"):
"""

42
sqlite_utils/utils.py
View file

@ -43,15 +43,10 @@ else:
dbapi2 = importlib.import_module("pysqlite3.dbapi2")
OperationalError = dbapi2.OperationalError
except ImportError:
try:
sqlite3 = importlib.import_module("sqlean")
dbapi2 = importlib.import_module("sqlean.dbapi2")
OperationalError = dbapi2.OperationalError
except ImportError:
import sqlite3 # noqa: F401
from sqlite3 import dbapi2 # noqa: F401
import sqlite3 # noqa: F401
from sqlite3 import dbapi2 # noqa: F401
OperationalError = dbapi2.OperationalError
OperationalError = dbapi2.OperationalError
SPATIALITE_PATHS = (
@ -618,6 +613,37 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) ->
).hexdigest()
def dedupe_keys(keys: Iterable[str]) -> List[str]:
"""
Rename duplicates in a list of column names so every name is unique,
by appending ``_2``, ``_3``... to later occurrences - skipping any
suffix that would collide with another column in the list.
Used when converting SQL query rows to dictionaries, where duplicate
column names would otherwise silently overwrite each other.
:param keys: List of column names, possibly containing duplicates
"""
keys = list(keys)
taken = set(keys)
if len(taken) == len(keys):
# No duplicates - the common case
return keys
seen: set = set()
result = []
for key in keys:
if key in seen:
new_key = key
suffix = 2
while new_key in seen or new_key in taken:
new_key = "{}_{}".format(key, suffix)
suffix += 1
key = new_key
seen.add(key)
result.append(key)
return result
def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]:
for key, value in d.items():
if isinstance(value, dict):

25
tests/conftest.py
View file

@ -8,11 +8,36 @@ create table Gosh2 (c1 text, c2 text, c3 text);
"""
def pytest_addoption(parser):
parser.addoption(
"--sqlite-autocommit",
action="store_true",
default=False,
help=(
"Run every test against connections created with the Python 3.12+ "
"sqlite3.connect(autocommit=True) mode"
),
)
def pytest_configure(config):
import sys
sys._called_from_test = True # type: ignore[attr-defined]
if config.getoption("--sqlite-autocommit"):
if sys.version_info < (3, 12):
raise pytest.UsageError(
"--sqlite-autocommit requires Python 3.12 or higher"
)
real_connect = sqlite3.connect
def autocommit_connect(*args, **kwargs):
kwargs.setdefault("autocommit", True)
return real_connect(*args, **kwargs)
sqlite3.connect = autocommit_connect
@pytest.fixture(autouse=True)
def close_all_databases():

103
tests/test_atomic.py
View file

@ -1,6 +1,6 @@
import pytest
from sqlite_utils.db import _iter_complete_sql_statements
from sqlite_utils.db import Database, _iter_complete_sql_statements
from sqlite_utils.utils import sqlite3
@ -172,3 +172,104 @@ def test_transform_detects_foreign_key_check_violations(fresh_db):
assert fresh_db["books"].foreign_keys == []
assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db):
fresh_db["t"].insert({"id": 1}, pk="id")
fresh_db.execute("begin")
with fresh_db.atomic():
fresh_db["t"].insert({"id": 2}, pk="id")
# Nothing is committed until the user's own transaction commits
assert fresh_db.conn.in_transaction
fresh_db.rollback()
assert [r["id"] for r in fresh_db["t"].rows] == [1]
# And with a commit instead, the atomic block's writes persist
fresh_db.execute("begin")
with fresh_db.atomic():
fresh_db["t"].insert({"id": 3}, pk="id")
fresh_db.commit()
assert [r["id"] for r in fresh_db["t"].rows] == [1, 3]
def test_begin_commit_rollback(tmpdir):
path = str(tmpdir / "test.db")
db = Database(path)
db["t"].insert({"id": 1}, pk="id")
db.begin()
db["t"].insert({"id": 2}, pk="id")
assert db.conn.in_transaction
db.rollback()
assert not db.conn.in_transaction
assert [r["id"] for r in db["t"].rows] == [1]
db.begin()
db["t"].insert({"id": 3}, pk="id")
db.commit()
db.close()
db2 = Database(path)
assert [r["id"] for r in db2["t"].rows] == [1, 3]
db2.close()
def test_begin_inside_transaction_errors(fresh_db):
fresh_db.begin()
with pytest.raises(sqlite3.OperationalError):
fresh_db.begin()
fresh_db.rollback()
def test_commit_and_rollback_without_transaction_are_noops(fresh_db):
fresh_db.commit()
fresh_db.rollback()
assert not fresh_db.conn.in_transaction
def test_execute_write_commits_immediately(tmpdir):
path = str(tmpdir / "test.db")
db = Database(path)
db["t"].insert({"id": 1}, pk="id")
db.execute("insert into t (id) values (2)")
# No implicit transaction is left open
assert not db.conn.in_transaction
# A completely separate connection sees the row straight away
other = sqlite3.connect(path)
assert other.execute("select count(*) from t").fetchone()[0] == 2
other.close()
db.close()
def test_execute_write_respects_explicit_transaction(fresh_db):
fresh_db["t"].insert({"id": 1}, pk="id")
fresh_db.begin()
fresh_db.execute("insert into t (id) values (2)")
# Still inside the explicit transaction - not committed
assert fresh_db.conn.in_transaction
fresh_db.rollback()
assert [r["id"] for r in fresh_db["t"].rows] == [1]
def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db):
# A BEGIN hidden behind a leading comment must not be auto-committed
# out from under the caller
fresh_db["t"].insert({"id": 1}, pk="id")
fresh_db.execute("-- start a transaction\nbegin")
assert fresh_db.conn.in_transaction
fresh_db.execute("insert into t (id) values (2)")
fresh_db.rollback()
assert [r["id"] for r in fresh_db["t"].rows] == [1]
def test_query_returning_commits_after_iteration(tmpdir):
if sqlite3.sqlite_version_info < (3, 35, 0):
import pytest as _pytest
_pytest.skip("RETURNING requires SQLite 3.35.0 or higher")
path = str(tmpdir / "test.db")
db = Database(path)
db["t"].insert({"id": 1}, pk="id")
rows = list(db.query("insert into t (id) values (2) returning id"))
assert rows == [{"id": 2}]
assert not db.conn.in_transaction
other = sqlite3.connect(path)
assert other.execute("select count(*) from t").fetchone()[0] == 2
other.close()
db.close()

91
tests/test_cli.py
View file

@ -746,6 +746,26 @@ def test_query_json_empty(db_path):
assert result.output.strip() == "[]"
def test_query_json_duplicate_columns_are_deduped(db_path):
# https://github.com/simonw/sqlite-utils/issues/624
result = CliRunner().invoke(
cli.cli,
[db_path, "select 1 as id, 2 as id, 'x' as value, 'y' as value"],
)
assert result.output.strip() == (
'[{"id": 1, "id_2": 2, "value": "x", "value_2": "y"}]'
)
def test_query_csv_duplicate_columns_are_preserved(db_path):
# CSV output should keep the duplicate headers, not rename them
result = CliRunner().invoke(
cli.cli,
[db_path, "select 1 as id, 2 as id", "--csv"],
)
assert result.output.replace("\r", "").strip() == "id,id\n1,2"
def test_query_invalid_function(db_path):
result = CliRunner().invoke(
cli.cli, [db_path, "select bad()", "--functions", "def invalid_python"]
@ -1467,6 +1487,24 @@ def test_drop_table_error():
assert result.exit_code == 0
def test_drop_table_on_view_errors():
runner = CliRunner()
with runner.isolated_filesystem():
db = Database("test.db")
db["t"].insert({"id": 1})
db.create_view("v", "select * from t")
result = runner.invoke(cli.cli, ["drop-table", "test.db", "v"])
assert result.exit_code == 1
assert 'Error: "v" is a view, not a table - use drop-view to drop it' == (
result.output.strip()
)
assert "v" in db.view_names()
# --ignore exits cleanly but must still not drop the view
result = runner.invoke(cli.cli, ["drop-table", "test.db", "v", "--ignore"])
assert result.exit_code == 0
assert "v" in db.view_names()
def test_drop_view():
runner = CliRunner()
with runner.isolated_filesystem():
@ -1485,6 +1523,23 @@ def test_drop_view():
assert "hello" not in db.view_names()
def test_drop_view_on_table_errors():
runner = CliRunner()
with runner.isolated_filesystem():
db = Database("test.db")
db["t"].insert({"id": 1})
result = runner.invoke(cli.cli, ["drop-view", "test.db", "t"])
assert result.exit_code == 1
assert 'Error: "t" is a table, not a view - use drop-table to drop it' == (
result.output.strip()
)
assert "t" in db.table_names()
# --ignore exits cleanly but must still not drop the table
result = runner.invoke(cli.cli, ["drop-view", "test.db", "t", "--ignore"])
assert result.exit_code == 0
assert "t" in db.table_names()
def test_drop_view_error():
runner = CliRunner()
with runner.isolated_filesystem():
@ -2296,18 +2351,14 @@ def test_csv_insert_bom(tmpdir):
]
@pytest.mark.parametrize("option", (None, "-d", "--detect-types"))
def test_insert_detect_types(tmpdir, option):
"""Test that type detection is now the default behavior"""
def test_insert_detect_types(tmpdir):
"""Test that type detection is the default behavior"""
db_path = str(tmpdir / "test.db")
data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5"
extra = []
if option:
extra = [option]
result = CliRunner().invoke(
cli.cli,
["insert", db_path, "creatures", "-", "--csv"] + extra,
["insert", db_path, "creatures", "-", "--csv"],
catch_exceptions=False,
input=data,
)
@ -2319,17 +2370,27 @@ def test_insert_detect_types(tmpdir, option):
]
@pytest.mark.parametrize("option", (None, "-d", "--detect-types"))
def test_upsert_detect_types(tmpdir, option):
"""Test that type detection is now the default behavior for upsert"""
@pytest.mark.parametrize("command", ("insert", "upsert"))
@pytest.mark.parametrize("option", ("-d", "--detect-types"))
def test_detect_types_flag_removed(tmpdir, command, option):
# The old no-op flag was removed in 4.0 - it should now error
db_path = str(tmpdir / "test.db")
data = "id,name,age,weight\n1,Cleo,6,45.5\n2,Dori,1,3.5"
extra = []
if option:
extra = [option]
result = CliRunner().invoke(
cli.cli,
["upsert", db_path, "creatures", "-", "--csv", "--pk", "id"] + extra,
[command, db_path, "creatures", "-", "--csv", "--pk", "id", option],
input="id,name\n1,Cleo",
)
assert result.exit_code == 2
assert "No such option" in result.output
def test_upsert_detect_types(tmpdir):
"""Test that type detection is the default behavior for upsert"""
db_path = str(tmpdir / "test.db")
data = "id,name,age,weight\n1,Cleo,6,45.5\n2,Dori,1,3.5"
result = CliRunner().invoke(
cli.cli,
["upsert", db_path, "creatures", "-", "--csv", "--pk", "id"],
catch_exceptions=False,
input=data,
)

13
tests/test_cli_insert.py
View file

@ -615,3 +615,16 @@ def test_insert_csv_headers_only(tmpdir):
# Table should not exist since there were no data rows
db = Database(db_path)
assert not db["data"].exists()
def test_insert_into_view_errors(tmpdir):
db_path = str(tmpdir / "test.db")
db = Database(db_path)
db["t"].insert({"id": 1})
db.create_view("v", "select * from t")
db.close()
result = CliRunner().invoke(
cli.cli, ["insert", db_path, "v", "-"], input='{"id": 2}'
)
assert result.exit_code == 1
assert result.output.strip() == "Error: Table v is actually a view"

162
tests/test_cli_migrate.py
View file

@ -301,3 +301,165 @@ def test_stop_before_multiple_qualified(two_sets_same_migration_name):
assert not db["creature_weights"].exists()
assert db["sales"].exists()
assert not db["sales_weights"].exists()
LEGACY_MIGRATIONS = """
import datetime
class _Migration:
def __init__(self, name, fn):
self.name = name
self.fn = fn
class _Applied:
def __init__(self, name, applied_at):
self.name = name
self.applied_at = applied_at
class LegacyMigrations:
# Mimics the sqlite-migrate 0.x Migrations class, in particular
# apply(db, stop_before=None) taking a single string
migrations_table = "_sqlite_migrations"
def __init__(self, name):
self.name = name
self._migrations = []
def __call__(self, fn):
self._migrations.append(_Migration(fn.__name__, fn))
return fn
def ensure_migrations_table(self, db):
db[self.migrations_table].create(
{"migration_set": str, "name": str, "applied_at": str},
pk=("migration_set", "name"),
if_not_exists=True,
)
def applied(self, db):
self.ensure_migrations_table(db)
return [
_Applied(row["name"], row["applied_at"])
for row in db[self.migrations_table].rows_where(
"migration_set = ?", [self.name]
)
]
def pending(self, db):
applied = {m.name for m in self.applied(db)}
return [m for m in self._migrations if m.name not in applied]
def apply(self, db, stop_before=None):
for migration in self.pending(db):
if migration.name == stop_before:
return
migration.fn(db)
db[self.migrations_table].insert(
{
"migration_set": self.name,
"name": migration.name,
"applied_at": str(
datetime.datetime.now(datetime.timezone.utc)
),
}
)
legacy = LegacyMigrations("legacy_set")
@legacy
def first(db):
db["first"].insert({"hello": "world"})
@legacy
def second(db):
db["second"].insert({"hello": "world"})
"""
def test_stop_before_unknown_name_errors(two_migrations):
path, _ = two_migrations
db_path = str(path / "test.db")
runner = CliRunner()
result = runner.invoke(
sqlite_utils.cli.cli,
["migrate", db_path, str(path), "--stop-before", "fooo"],
)
assert result.exit_code == 1
assert "--stop-before did not match any migrations: fooo" in result.output
# Nothing should have been applied
db = sqlite_utils.Database(db_path)
assert "foo" not in db.table_names()
assert "bar" not in db.table_names()
def test_stop_before_with_legacy_migrations_class(tmpdir):
path = pathlib.Path(tmpdir)
(path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8")
db_path = str(path / "test.db")
runner = CliRunner()
result = runner.invoke(
sqlite_utils.cli.cli,
["migrate", db_path, str(path), "--stop-before", "second"],
)
assert result.exit_code == 0, result.output
db = sqlite_utils.Database(db_path)
assert "first" in db.table_names()
assert "second" not in db.table_names()
def test_stop_before_multiple_values_for_legacy_set_errors(tmpdir):
path = pathlib.Path(tmpdir)
(path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8")
db_path = str(path / "test.db")
runner = CliRunner()
result = runner.invoke(
sqlite_utils.cli.cli,
[
"migrate",
db_path,
str(path),
"--stop-before",
"legacy_set:first",
"--stop-before",
"legacy_set:second",
],
)
assert result.exit_code == 1
assert "single --stop-before" in result.output
def test_list_does_not_create_database_file(two_migrations):
path, _ = two_migrations
db_path = path / "test.db"
runner = CliRunner()
result = runner.invoke(
sqlite_utils.cli.cli, ["migrate", str(db_path), str(path), "--list"]
)
assert result.exit_code == 0, result.output
assert "Pending:\n foo\n bar" in result.output
# Listing migrations must not create the database file
assert not db_path.exists()
def test_list_does_not_upgrade_legacy_migrations_table(two_migrations):
path, _ = two_migrations
db_path = str(path / "test.db")
db = sqlite_utils.Database(db_path)
db["_sqlite_migrations"].create(
{"migration_set": str, "name": str, "applied_at": str},
pk=("migration_set", "name"),
)
db["_sqlite_migrations"].insert(
{"migration_set": "hello", "name": "foo", "applied_at": "x"}
)
db.close()
runner = CliRunner()
result = runner.invoke(
sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"]
)
assert result.exit_code == 0, result.output
assert "foo - x" in result.output
# --list must not perform the one-way legacy schema upgrade
db2 = sqlite_utils.Database(db_path)
assert db2["_sqlite_migrations"].pks == ["migration_set", "name"]
db2.close()

233
tests/test_column_casing.py Normal file
View file

@ -0,0 +1,233 @@
"""
SQLite treats column names as case-insensitive. These tests exercise the
places where sqlite-utils performs Python-side lookups of column names
provided by the caller, which should match the schema case-insensitively.
https://github.com/simonw/sqlite-utils/issues/760
"""
import pytest
from sqlite_utils import Database
from sqlite_utils.db import ForeignKey
def test_insert_populates_last_pk_case_insensitively(fresh_db):
books = fresh_db["books"]
books.create({"Id": int, "Title": str}, pk="Id")
books.insert({"Id": 1, "Title": "One"}, pk="id")
assert books.last_pk == 1
def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db):
books = fresh_db["books"]
books.create({"Author": str, "Position": int, "Title": str})
books.insert(
{"Author": "Sue", "Position": 1, "Title": "One"}, pk=("author", "position")
)
assert books.last_pk == ("Sue", 1)
@pytest.mark.parametrize("use_old_upsert", (False, True))
def test_upsert_pk_case_differs_from_schema(use_old_upsert):
db = Database(memory=True, use_old_upsert=use_old_upsert)
books = db["books"]
books.create({"Id": int, "Title": str}, pk="Id")
books.insert({"Id": 1, "Title": "One"})
books.upsert({"id": 1, "title": "Won"}, pk="id")
assert list(books.rows) == [{"Id": 1, "Title": "Won"}]
assert books.last_pk == 1
@pytest.mark.parametrize("use_old_upsert", (False, True))
def test_upsert_record_key_case_differs_from_pk(use_old_upsert):
# all_columns comes from the record keys, pk= from the caller
db = Database(memory=True, use_old_upsert=use_old_upsert)
books = db["books"]
books.create({"Id": int, "Title": str}, pk="Id")
books.upsert({"ID": 1, "Title": "One"}, pk="id")
assert list(books.rows) == [{"Id": 1, "Title": "One"}]
assert books.last_pk == 1
def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db):
# pk is inferred from the existing schema as "Id", records use "id"
books = fresh_db["books"]
books.create({"Id": int, "Title": str}, pk="Id")
books.upsert({"id": 1, "title": "One"})
assert list(books.rows) == [{"Id": 1, "Title": "One"}]
assert books.last_pk == 1
def test_upsert_list_mode_pk_case_insensitive(fresh_db):
books = fresh_db["books"]
books.create({"Id": int, "Title": str}, pk="Id")
books.upsert_all([["id", "title"], [1, "One"]], pk="Id")
assert list(books.rows) == [{"Id": 1, "Title": "One"}]
assert books.last_pk == 1
def test_lookup_pk_case_insensitive(fresh_db):
fresh_db["species"].create({"ID": int, "Name": str}, pk="ID")
fresh_db["species"].insert({"ID": 5, "Name": "Palm"})
fresh_db["species"].create_index(["Name"], unique=True)
assert fresh_db["species"].lookup({"Name": "Palm"}, pk="id") == 5
def test_lookup_does_not_create_redundant_index(fresh_db):
fresh_db["species"].create({"id": int, "Name": str}, pk="id")
fresh_db["species"].create_index(["Name"], unique=True)
fresh_db["species"].lookup({"name": "Palm"})
assert len(fresh_db["species"].indexes) == 1
def test_create_table_transform_same_columns_different_case(fresh_db):
fresh_db["t"].create({"Name": str, "Age": int})
fresh_db["t"].insert({"Name": "Cleo", "Age": 5})
fresh_db.create_table("t", {"name": str, "age": int}, transform=True)
# Schema casing is preserved - SQLite considers these the same columns
assert fresh_db["t"].columns_dict == {"Name": str, "Age": int}
assert list(fresh_db["t"].rows) == [{"Name": "Cleo", "Age": 5}]
def test_create_table_transform_case_insensitive_with_changes(fresh_db):
fresh_db["t"].create({"Name": str, "Age": int})
fresh_db.create_table("t", {"name": str, "age": str, "size": int}, transform=True)
# age changed type, size added, Name untouched
assert fresh_db["t"].columns_dict == {"Name": str, "Age": str, "size": int}
def test_transform_types_case_insensitive(fresh_db):
fresh_db["t"].create({"Name": str, "Age": str})
fresh_db["t"].transform(types={"age": int})
assert fresh_db["t"].columns_dict == {"Name": str, "Age": int}
def test_transform_rename_case_insensitive(fresh_db):
fresh_db["t"].create({"Name": str})
fresh_db["t"].transform(rename={"name": "title"})
assert fresh_db["t"].columns_dict == {"title": str}
def test_transform_drop_case_insensitive(fresh_db):
fresh_db["t"].create({"Name": str, "Age": int})
fresh_db["t"].transform(drop=["name"])
assert fresh_db["t"].columns_dict == {"Age": int}
def test_transform_not_null_and_defaults_case_insensitive(fresh_db):
fresh_db["t"].create({"Name": str, "Age": int})
fresh_db["t"].transform(not_null={"name"}, defaults={"age": 3})
columns = {c.name: c for c in fresh_db["t"].columns}
assert columns["Name"].notnull
assert fresh_db["t"].default_values == {"Age": 3}
def test_transform_pk_case_insensitive(fresh_db):
fresh_db["t"].create({"Id": int, "Name": str})
fresh_db["t"].transform(pk="id")
assert fresh_db["t"].pks == ["Id"]
assert fresh_db["t"].columns_dict == {"Id": int, "Name": str}
def test_transform_drop_foreign_keys_case_insensitive(fresh_db):
fresh_db["parent"].create({"Id": int}, pk="Id")
fresh_db["child"].create(
{"id": int, "Parent_ID": int},
pk="id",
foreign_keys=[("Parent_ID", "parent", "Id")],
)
fresh_db["child"].transform(drop_foreign_keys=["parent_id"])
assert fresh_db["child"].foreign_keys == []
def test_add_foreign_key_case_insensitive(fresh_db):
fresh_db["parent"].create({"Id": int}, pk="Id")
fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id")
fresh_db["child"].add_foreign_key("parent_id", "parent", "id")
fks = fresh_db["child"].foreign_keys
assert len(fks) == 1
# The foreign key should use the schema casing of the columns
assert fks[0].column == "Parent_ID"
assert fks[0].other_column == "Id"
def test_add_foreign_keys_case_insensitive(fresh_db):
fresh_db["parent"].create({"Id": int}, pk="Id")
fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id")
fresh_db.add_foreign_keys([("child", "parent_id", "parent", "id")])
fks = fresh_db["child"].foreign_keys
assert len(fks) == 1
assert fks[0].column == "Parent_ID"
assert fks[0].other_column == "Id"
def test_add_foreign_key_detects_existing_case_insensitively(fresh_db):
fresh_db["parent"].create({"Id": int}, pk="Id")
fresh_db["child"].create(
{"id": int, "Parent_ID": int},
pk="id",
foreign_keys=[("Parent_ID", "parent", "Id")],
)
# ignore=True should treat this as already existing, not add a duplicate
fresh_db["child"].add_foreign_key("parent_id", "parent", "id", ignore=True)
assert len(fresh_db["child"].foreign_keys) == 1
def test_add_column_fk_col_case_insensitive(fresh_db):
fresh_db["parent"].create({"Id": int}, pk="Id")
fresh_db["child"].create({"id": int}, pk="id")
fresh_db["child"].add_column("parent_id", int, fk="parent", fk_col="id")
fks = fresh_db["child"].foreign_keys
assert len(fks) == 1
assert fks[0].other_column == "Id"
def test_extract_case_insensitive(fresh_db):
fresh_db["trees"].insert({"id": 1, "Species": "Palm"}, pk="id")
fresh_db["trees"].extract("species")
assert fresh_db["trees"].columns_dict == {"id": int, "Species_id": int}
assert list(fresh_db["Species"].rows) == [{"id": 1, "Species": "Palm"}]
def test_convert_multi_case_insensitive(fresh_db):
fresh_db["t"].insert({"id": 1, "Name": "Cleo"}, pk="id")
fresh_db["t"].convert("name", lambda v: {"upper": v.upper()}, multi=True)
assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "upper": "CLEO"}]
def test_convert_output_case_insensitive(fresh_db):
fresh_db["t"].insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id")
fresh_db["t"].convert("name", lambda v: v.upper(), output="upper")
assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "Upper": "CLEO"}]
def test_create_table_sql_pk_case_insensitive(fresh_db):
fresh_db["t"].create({"Id": int, "Name": str}, pk="id")
# Should not have created an extra lowercase "id" column
assert fresh_db["t"].columns_dict == {"Id": int, "Name": str}
assert fresh_db["t"].pks == ["Id"]
def test_create_table_not_null_and_defaults_case_insensitive(fresh_db):
fresh_db["t"].create(
{"Name": str, "Age": int}, not_null={"name"}, defaults={"age": 1}
)
columns = {c.name: c for c in fresh_db["t"].columns}
assert columns["Name"].notnull
assert fresh_db["t"].default_values == {"Age": 1}
def test_create_table_foreign_keys_case_insensitive(fresh_db):
fresh_db["parent"].create({"Id": int}, pk="Id")
fresh_db["child"].create(
{"id": int, "Parent_ID": int},
pk="id",
foreign_keys=[("parent_id", "parent", "id")],
)
fks = fresh_db["child"].foreign_keys
assert fks == [
ForeignKey(
table="child", column="Parent_ID", other_table="parent", other_column="Id"
)
]

48
tests/test_constructor.py
View file

@ -1,6 +1,8 @@
from sqlite_utils import Database
from sqlite_utils.db import TransactionError
from sqlite_utils.utils import sqlite3
import pytest
import sys
def test_recursive_triggers():
@ -29,6 +31,24 @@ def test_sqlite_version():
assert actual == as_string
def test_database_context_manager(tmpdir):
path = str(tmpdir / "test.db")
with Database(path) as db:
db["t"].insert({"id": 1})
# Raw writes commit automatically too
db.execute("insert into t (id) values (2)")
# An explicitly opened transaction left uncommitted on purpose:
db.begin()
db.execute("insert into t (id) values (3)")
# The connection is closed...
with pytest.raises(sqlite3.ProgrammingError):
db.execute("select 1")
# ... and the open explicit transaction was rolled back, not committed
db2 = Database(path)
assert [r["id"] for r in db2["t"].rows] == [1, 2]
db2.close()
@pytest.mark.parametrize("memory", [True, False])
def test_database_close(tmpdir, memory):
if memory:
@ -39,3 +59,31 @@ def test_database_close(tmpdir, memory):
db.close()
with pytest.raises(sqlite3.ProgrammingError):
db.execute("select 1 + 1")
@pytest.mark.skipif(
sys.version_info < (3, 12),
reason="sqlite3.connect(autocommit=) requires Python 3.12",
)
@pytest.mark.parametrize("autocommit", [True, False])
def test_autocommit_connections_are_rejected(tmpdir, autocommit):
# These connection modes break commit()/rollback() in ways that
# silently lose data, so the constructor refuses them
conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=autocommit)
with pytest.raises(TransactionError):
Database(conn)
conn.close()
@pytest.mark.skipif(
sys.version_info < (3, 12),
reason="sqlite3.LEGACY_TRANSACTION_CONTROL requires Python 3.12",
)
def test_legacy_transaction_control_connection_is_accepted(tmpdir):
conn = sqlite3.connect(
str(tmpdir / "test.db"), autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL
)
db = Database(conn)
db["t"].insert({"id": 1}, pk="id")
assert [r["id"] for r in db["t"].rows] == [1]
db.close()

2
tests/test_convert.py
View file

@ -77,7 +77,7 @@ def test_convert_output(fresh_db, drop, expected):
def test_convert_output_multiple_column_error(fresh_db):
table = fresh_db["table"]
with pytest.raises(AssertionError) as excinfo:
with pytest.raises(ValueError) as excinfo:
table.convert(["title", "other"], lambda v: v, output="out")
assert "output= can only be used with a single column" in str(excinfo.value)

17
tests/test_create.py
View file

@ -108,7 +108,7 @@ def test_create_table_with_defaults(fresh_db):
def test_create_table_with_bad_not_null(fresh_db):
with pytest.raises(AssertionError):
with pytest.raises(ValueError):
fresh_db.create_table(
"players", {"name": str, "score": int}, not_null={"mouse"}
)
@ -243,11 +243,11 @@ def test_create_table_column_order(fresh_db, use_table_factory):
# If you specify a column that doesn't point to a table, you get an error:
(("one_id", "two_id", "three_id"), NoObviousTable),
# Tuples of the wrong length get an error:
((("one_id", "one", "id", "five"), ("two_id", "two", "id")), AssertionError),
((("one_id", "one", "id", "five"), ("two_id", "two", "id")), ValueError),
# Likewise a bad column:
((("one_id", "one", "id2"),), AlterError),
# Or a list of dicts
(({"one_id": "one"},), AssertionError),
(({"one_id": "one"},), ValueError),
),
)
@pytest.mark.parametrize("use_table_factory", [True, False])
@ -700,7 +700,7 @@ def test_bulk_insert_more_than_999_values(fresh_db):
def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error):
record = dict([("c{}".format(i), i) for i in range(num_columns)])
if should_error:
with pytest.raises(AssertionError):
with pytest.raises(ValueError):
fresh_db["big"].insert(record)
else:
fresh_db["big"].insert(record)
@ -1061,7 +1061,7 @@ def test_create_table_numpy(fresh_db):
def test_cannot_provide_both_filename_and_memory():
with pytest.raises(
AssertionError, match="Either specify a filename_or_conn or pass memory=True"
ValueError, match="Either specify a filename_or_conn or pass memory=True"
):
Database("/tmp/foo.db", memory=True)
@ -1214,7 +1214,7 @@ def test_create_if_not_exists(fresh_db):
def test_create_if_no_columns(fresh_db):
with pytest.raises(AssertionError) as error:
with pytest.raises(ValueError) as error:
fresh_db["t"].create({})
assert error.value.args[0] == "Tables must have at least one column"
@ -1382,7 +1382,10 @@ def test_bad_table_and_view_exceptions(fresh_db):
assert ex.value.args[0] == "Table v is actually a view"
with pytest.raises(NoView) as ex2:
fresh_db.view("t")
assert ex2.value.args[0] == "View t does not exist"
assert ex2.value.args[0] == "View t does not exist - t is a table"
with pytest.raises(NoView) as ex3:
fresh_db.view("missing")
assert ex3.value.args[0] == "View missing does not exist"
# Tests for issue #655: Table configuration should be stored in _defaults

2
tests/test_create_view.py
View file

@ -15,7 +15,7 @@ def test_create_view_error(fresh_db):
def test_create_view_only_arrow_one_param(fresh_db):
with pytest.raises(AssertionError):
with pytest.raises(ValueError):
fresh_db.create_view("bar", "select 1 + 2", ignore=True, replace=True)

18
tests/test_delete.py
View file

@ -1,3 +1,6 @@
import sqlite_utils
def test_delete_rowid_table(fresh_db):
table = fresh_db["table"]
table.insert({"foo": 1}).last_pk
@ -32,6 +35,21 @@ def test_delete_where_all(fresh_db):
assert table.count == 0
def test_delete_where_commits(tmpdir):
path = str(tmpdir / "test.db")
db = sqlite_utils.Database(path)
db["table"].insert_all([{"id": i} for i in range(5)], pk="id")
db["table"].delete_where("id > ?", [2])
# The connection must not be left inside an open transaction,
# otherwise subsequent atomic() blocks never commit either
assert not db.conn.in_transaction
db["table"].insert({"id": 100})
db.close()
db2 = sqlite_utils.Database(path)
assert [r["id"] for r in db2["table"].rows] == [0, 1, 2, 100]
db2.close()
def test_delete_where_analyze(fresh_db):
table = fresh_db["table"]
table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id")

526
tests/test_foreign_keys.py Normal file
View file

@ -0,0 +1,526 @@
"""Tests for compound (multi-column) foreign keys - issue #594."""
import pytest
from sqlite_utils import Database
from sqlite_utils.db import AlterError, ForeignKey
from sqlite_utils.utils import sqlite3
COMPOUND_SCHEMA = """
CREATE TABLE departments (
campus_name TEXT NOT NULL,
dept_code TEXT NOT NULL,
dept_name TEXT,
PRIMARY KEY (campus_name, dept_code)
);
CREATE TABLE courses (
course_code TEXT PRIMARY KEY,
course_name TEXT,
campus_name TEXT NOT NULL,
dept_code TEXT NOT NULL,
FOREIGN KEY (campus_name, dept_code)
REFERENCES departments(campus_name, dept_code)
);
"""
@pytest.fixture
def compound_db():
db = Database(memory=True)
db.executescript(COMPOUND_SCHEMA)
return db
def test_compound_foreign_key(compound_db):
fks = compound_db["courses"].foreign_keys
assert len(fks) == 1
fk = fks[0]
assert fk.is_compound is True
assert fk.table == "courses"
assert fk.other_table == "departments"
assert fk.columns == ("campus_name", "dept_code")
assert fk.other_columns == ("campus_name", "dept_code")
# Scalar column/other_column can't sensibly hold a compound key
assert fk.column is None
assert fk.other_column is None
def test_single_foreign_key_gets_columns_fields(fresh_db):
fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1})
fresh_db["books"].add_foreign_key("author_id", "authors", "id")
fk = fresh_db["books"].foreign_keys[0]
assert fk.is_compound is False
assert fk.column == "author_id"
assert fk.other_column == "id"
assert fk.columns == ("author_id",)
assert fk.other_columns == ("id",)
def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db):
# Clean break in 4.0: ForeignKey is a dataclass, not a namedtuple, so the
# old tuple unpacking and indexing patterns now fail hard.
fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1})
fresh_db["books"].add_foreign_key("author_id", "authors", "id")
fk = fresh_db["books"].foreign_keys[0]
with pytest.raises(TypeError):
table, column, other_table, other_column = fk
with pytest.raises(TypeError):
fk[0]
def test_foreign_keys_are_sortable(fresh_db):
fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
fresh_db["categories"].insert({"id": 1, "name": "Wildlife"}, pk="id")
fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1, "category_id": 1})
fresh_db.add_foreign_keys(
[
("books", "author_id", "authors", "id"),
("books", "category_id", "categories", "id"),
]
)
fks = sorted(fresh_db["books"].foreign_keys)
assert fks[0].column == "author_id"
assert fks[1].column == "category_id"
def test_mixed_compound_and_single_foreign_keys_are_sortable():
# compound FKs have column=None, which must not break sorting
# against single-column FKs (None < str raises TypeError)
db = Database(memory=True)
db.executescript("""
CREATE TABLE departments (
campus_name TEXT NOT NULL,
dept_code TEXT NOT NULL,
PRIMARY KEY (campus_name, dept_code)
);
CREATE TABLE accreditations (id INTEGER PRIMARY KEY);
CREATE TABLE courses (
course_code TEXT PRIMARY KEY,
campus_name TEXT NOT NULL,
dept_code TEXT NOT NULL,
accreditation_id INTEGER REFERENCES accreditations(id),
FOREIGN KEY (campus_name, dept_code)
REFERENCES departments(campus_name, dept_code)
);
""")
fks = db["courses"].foreign_keys
assert len(fks) == 2
assert {fk.is_compound for fk in fks} == {True, False}
fks_sorted = sorted(fks)
assert fks_sorted[0].other_table == "accreditations"
assert fks_sorted[1].other_table == "departments"
@pytest.fixture
def departments_db():
db = Database(memory=True)
db.create_table(
"departments",
{"campus_name": str, "dept_code": str, "dept_name": str},
pk=("campus_name", "dept_code"),
)
return db
EXPECTED_COURSES_SCHEMA = (
'CREATE TABLE "courses" (\n'
' "course_code" TEXT PRIMARY KEY,\n'
' "campus_name" TEXT,\n'
' "dept_code" TEXT,\n'
' FOREIGN KEY ("campus_name", "dept_code") '
'REFERENCES "departments"("campus_name", "dept_code")\n'
")"
)
@pytest.mark.parametrize(
"foreign_keys",
(
[
ForeignKey(
table="courses",
column=None,
other_table="departments",
other_column=None,
columns=("campus_name", "dept_code"),
other_columns=("campus_name", "dept_code"),
is_compound=True,
)
],
[(("campus_name", "dept_code"), "departments", ("campus_name", "dept_code"))],
# Two-item form guesses the other table's primary key:
[(("campus_name", "dept_code"), "departments")],
# Lists work too, though tuples are the documented form:
[(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])],
),
)
def test_create_table_with_compound_foreign_key(departments_db, foreign_keys):
departments_db.create_table(
"courses",
{"course_code": str, "campus_name": str, "dept_code": str},
pk="course_code",
foreign_keys=foreign_keys,
)
assert departments_db["courses"].schema == EXPECTED_COURSES_SCHEMA
fks = departments_db["courses"].foreign_keys
assert len(fks) == 1
fk = fks[0]
assert fk.is_compound is True
assert fk.columns == ("campus_name", "dept_code")
assert fk.other_table == "departments"
assert fk.other_columns == ("campus_name", "dept_code")
def test_create_table_compound_foreign_key_enforced(departments_db):
departments_db.execute("PRAGMA foreign_keys = ON")
departments_db.create_table(
"courses",
{"course_code": str, "campus_name": str, "dept_code": str},
pk="course_code",
foreign_keys=[(("campus_name", "dept_code"), "departments")],
)
departments_db["departments"].insert(
{"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"}
)
departments_db["courses"].insert(
{"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"}
)
with pytest.raises(sqlite3.IntegrityError):
departments_db.execute(
"insert into courses (course_code, campus_name, dept_code) "
"values ('X1', 'Nowhere', 'NOPE')"
)
def test_create_table_compound_foreign_key_missing_other_column(departments_db):
with pytest.raises(AlterError):
departments_db.create_table(
"courses",
{"course_code": str, "campus_name": str, "dept_code": str},
pk="course_code",
foreign_keys=[
(("campus_name", "dept_code"), "departments", ("campus_name", "nope"))
],
)
def test_transform_preserves_compound_foreign_key(compound_db):
compound_db["courses"].transform(rename={"course_name": "title"})
fks = compound_db["courses"].foreign_keys
assert len(fks) == 1
fk = fks[0]
assert fk.is_compound is True
assert fk.columns == ("campus_name", "dept_code")
assert fk.other_table == "departments"
assert fk.other_columns == ("campus_name", "dept_code")
def test_transform_rename_member_column_updates_compound_foreign_key(compound_db):
compound_db["courses"].transform(rename={"campus_name": "campus"})
fks = compound_db["courses"].foreign_keys
assert len(fks) == 1
fk = fks[0]
assert fk.is_compound is True
assert fk.columns == ("campus", "dept_code")
# Referenced columns in the other table are unchanged
assert fk.other_columns == ("campus_name", "dept_code")
def test_transform_drop_member_column_drops_compound_foreign_key(compound_db):
# Matches single-column behavior: dropping the column silently
# drops the foreign key that used it
compound_db["courses"].transform(drop={"dept_code"})
assert compound_db["courses"].foreign_keys == []
assert "FOREIGN KEY" not in compound_db["courses"].schema
@pytest.mark.parametrize(
"drop_foreign_keys",
(
# A bare column name matches any foreign key it participates in:
["campus_name"],
# A tuple must match the full compound key:
[("campus_name", "dept_code")],
),
)
def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys):
compound_db["courses"].transform(drop_foreign_keys=drop_foreign_keys)
assert compound_db["courses"].foreign_keys == []
# The columns themselves survive
assert {"campus_name", "dept_code"} <= set(
compound_db["courses"].columns_dict.keys()
)
@pytest.fixture
def courses_db(departments_db):
departments_db.create_table(
"courses",
{"course_code": str, "campus_name": str, "dept_code": str},
pk="course_code",
)
return departments_db
def test_add_compound_foreign_key(courses_db):
t = courses_db["courses"].add_foreign_key(
("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")
)
# Returns self
assert t.name == "courses"
fks = courses_db["courses"].foreign_keys
assert len(fks) == 1
fk = fks[0]
assert fk.is_compound is True
assert fk.columns == ("campus_name", "dept_code")
assert fk.other_table == "departments"
assert fk.other_columns == ("campus_name", "dept_code")
def test_add_compound_foreign_key_guesses_other_columns(courses_db):
# Lists work here too, though tuples are the documented form
courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments")
fk = courses_db["courses"].foreign_keys[0]
assert fk.other_columns == ("campus_name", "dept_code")
def test_add_compound_foreign_key_error_if_already_exists(courses_db):
courses_db["courses"].add_foreign_key(("campus_name", "dept_code"), "departments")
with pytest.raises(AlterError) as ex:
courses_db["courses"].add_foreign_key(
("campus_name", "dept_code"), "departments"
)
assert "already exists" in ex.value.args[0]
# ignore=True should not raise
courses_db["courses"].add_foreign_key(
("campus_name", "dept_code"), "departments", ignore=True
)
def test_add_compound_foreign_key_error_if_column_missing(courses_db):
with pytest.raises(AlterError):
courses_db["courses"].add_foreign_key(("campus_name", "nope"), "departments")
def test_db_add_foreign_keys_compound(courses_db):
courses_db.add_foreign_keys(
[
(
"courses",
("campus_name", "dept_code"),
"departments",
("campus_name", "dept_code"),
)
]
)
fk = courses_db["courses"].foreign_keys[0]
assert fk.is_compound is True
assert fk.columns == ("campus_name", "dept_code")
def test_index_foreign_keys_compound_creates_composite_index(compound_db):
compound_db.index_foreign_keys()
index_columns = [i.columns for i in compound_db["courses"].indexes]
assert ["campus_name", "dept_code"] in index_columns
# No separate single-column indexes for the members
assert ["campus_name"] not in index_columns
assert ["dept_code"] not in index_columns
def test_foreign_key_captures_on_delete_and_on_update():
db = Database(memory=True)
db.executescript("""
CREATE TABLE authors (id INTEGER PRIMARY KEY);
CREATE TABLE books (
id INTEGER PRIMARY KEY,
author_id INTEGER REFERENCES authors(id)
ON DELETE CASCADE ON UPDATE RESTRICT
);
""")
fk = db["books"].foreign_keys[0]
assert fk.on_delete == "CASCADE"
assert fk.on_update == "RESTRICT"
def test_foreign_key_on_delete_defaults_to_no_action(fresh_db):
fresh_db["authors"].insert({"id": 1}, pk="id")
fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
fresh_db["books"].add_foreign_key("author_id", "authors", "id")
fk = fresh_db["books"].foreign_keys[0]
assert fk.on_delete == "NO ACTION"
assert fk.on_update == "NO ACTION"
def test_create_table_foreign_key_with_on_delete(fresh_db):
fresh_db["authors"].insert({"id": 1}, pk="id")
fresh_db.create_table(
"books",
{"id": int, "author_id": int},
pk="id",
foreign_keys=[
ForeignKey(
table="books",
column="author_id",
other_table="authors",
other_column="id",
on_delete="CASCADE",
)
],
)
assert "ON DELETE CASCADE" in fresh_db["books"].schema
assert fresh_db["books"].foreign_keys[0].on_delete == "CASCADE"
def test_transform_preserves_on_delete_cascade():
db = Database(memory=True)
db.executescript("""
CREATE TABLE authors (id INTEGER PRIMARY KEY);
CREATE TABLE books (
id INTEGER PRIMARY KEY,
title TEXT,
author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE
);
""")
db["books"].transform(rename={"title": "book_title"})
fk = db["books"].foreign_keys[0]
assert fk.on_delete == "CASCADE"
assert fk.on_update == "NO ACTION"
assert "ON DELETE CASCADE" in db["books"].schema
def test_transform_preserves_compound_foreign_key_on_delete():
db = Database(memory=True)
db.executescript("""
CREATE TABLE departments (
campus_name TEXT NOT NULL,
dept_code TEXT NOT NULL,
PRIMARY KEY (campus_name, dept_code)
);
CREATE TABLE courses (
course_code TEXT PRIMARY KEY,
campus_name TEXT NOT NULL,
dept_code TEXT NOT NULL,
FOREIGN KEY (campus_name, dept_code)
REFERENCES departments(campus_name, dept_code) ON DELETE CASCADE
);
""")
db["courses"].transform(rename={"course_code": "code"})
fk = db["courses"].foreign_keys[0]
assert fk.is_compound is True
assert fk.on_delete == "CASCADE"
assert "ON DELETE CASCADE" in db["courses"].schema
def test_implicit_primary_key_reference_is_resolved():
# REFERENCES authors (no column) has "to" of None in the pragma -
# it should be resolved to the primary key of the other table
db = Database(memory=True)
db.executescript("""
CREATE TABLE authors (author_id INTEGER PRIMARY KEY);
CREATE TABLE books (
id INTEGER PRIMARY KEY,
author_id INTEGER REFERENCES authors
);
""")
fk = db["books"].foreign_keys[0]
assert fk.is_compound is False
assert fk.other_column == "author_id"
assert fk.other_columns == ("author_id",)
def test_implicit_compound_primary_key_reference_is_resolved():
db = Database(memory=True)
db.executescript("""
CREATE TABLE departments (
campus_name TEXT NOT NULL,
dept_code TEXT NOT NULL,
PRIMARY KEY (campus_name, dept_code)
);
CREATE TABLE courses (
course_code TEXT PRIMARY KEY,
campus_name TEXT NOT NULL,
dept_code TEXT NOT NULL,
FOREIGN KEY (campus_name, dept_code) REFERENCES departments
);
""")
fk = db["courses"].foreign_keys[0]
assert fk.is_compound is True
assert fk.other_columns == ("campus_name", "dept_code")
def test_foreign_key_normalizes_list_columns_to_tuples():
# Compound columns passed as lists are normalized to tuples, so they
# compare equal to introspected ForeignKeys
fk = ForeignKey(
table="courses",
column=None,
other_table="departments",
other_column=None,
columns=["campus_name", "dept_code"],
other_columns=["campus_name", "dept_code"],
is_compound=True,
)
assert fk.columns == ("campus_name", "dept_code")
assert fk.other_columns == ("campus_name", "dept_code")
def test_add_foreign_keys_preserves_actions(fresh_db):
# https://github.com/simonw/sqlite-utils/issues/594 review finding:
# ForeignKey objects passed to db.add_foreign_keys() were flattened
# to plain tuples, losing on_delete/on_update
fresh_db["authors"].insert({"id": 1}, pk="id")
fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
fresh_db.add_foreign_keys(
[ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")]
)
fk = fresh_db["books"].foreign_keys[0]
assert fk.on_delete == "CASCADE"
assert "ON DELETE CASCADE" in fresh_db["books"].schema
def test_add_foreign_keys_preserves_actions_compound(courses_db):
courses_db.add_foreign_keys(
[
ForeignKey(
table="courses",
column=None,
other_table="departments",
other_column=None,
columns=("campus_name", "dept_code"),
other_columns=("campus_name", "dept_code"),
is_compound=True,
on_delete="CASCADE",
)
]
)
fk = courses_db["courses"].foreign_keys[0]
assert fk.is_compound is True
assert fk.on_delete == "CASCADE"
assert "ON DELETE CASCADE" in courses_db["courses"].schema
def test_add_foreign_key_on_delete_on_update(fresh_db):
fresh_db["authors"].insert({"id": 1}, pk="id")
fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
fresh_db["books"].add_foreign_key(
"author_id", "authors", "id", on_delete="CASCADE", on_update="RESTRICT"
)
fk = fresh_db["books"].foreign_keys[0]
assert fk.on_delete == "CASCADE"
assert fk.on_update == "RESTRICT"
assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db["books"].schema
# The cascade should actually fire
fresh_db.execute("PRAGMA foreign_keys = ON")
fresh_db.execute("delete from authors where id = 1")
assert fresh_db["books"].count == 0
def test_add_compound_foreign_key_on_delete(courses_db):
courses_db["courses"].add_foreign_key(
("campus_name", "dept_code"), "departments", on_delete="SET NULL"
)
fk = courses_db["courses"].foreign_keys[0]
assert fk.is_compound is True
assert fk.on_delete == "SET NULL"
assert "ON DELETE SET NULL" in courses_db["courses"].schema

54
tests/test_fts.py
View file

@ -83,6 +83,20 @@ def test_enable_fts_escape_table_names(fresh_db):
assert [] == list(table.search("bar"))
def test_search_duplicate_columns_are_deduped(fresh_db):
# https://github.com/simonw/sqlite-utils/issues/624
table = fresh_db["t"]
table.insert_all(search_records)
table.enable_fts(["text", "country"], fts_version="FTS4")
rows = list(table.search("tanuki", columns=["text", "text"]))
assert rows == [
{
"text": "tanuki are running tricksters",
"text_2": "tanuki are running tricksters",
}
]
def test_search_limit_offset(fresh_db):
table = fresh_db["t"]
table.insert_all(search_records)
@ -336,6 +350,24 @@ def test_rebuild_fts(fresh_db):
assert len(rows2) == 2
@pytest.mark.parametrize("method", ["optimize", "rebuild_fts"])
def test_optimize_and_rebuild_fts_commit(tmpdir, method):
path = str(tmpdir / "test.db")
db = Database(path)
table = db["searchable"]
table.insert(search_records[0])
table.enable_fts(["text", "country"])
getattr(table, method)()
# The connection must not be left inside an open transaction,
# otherwise this and all subsequent writes are lost on close
assert not db.conn.in_transaction
table.insert(search_records[1])
db.close()
db2 = Database(path)
assert db2["searchable"].count == 2
db2.close()
@pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"])
def test_rebuild_fts_invalid(fresh_db, invalid_table):
fresh_db["not_searchable"].insert({"foo": "bar"})
@ -443,12 +475,12 @@ def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table():
assert 'content="books"' in db["books_fts"].schema
def test_enable_fts_error_message_on_views():
def test_view_has_no_enable_fts():
db = Database(memory=True)
db.create_view("hello", "select 1 + 1")
with pytest.raises(NotImplementedError) as e:
db["hello"].enable_fts() # type: ignore[call-arg]
assert e.value.args[0] == "enable_fts() is supported on tables but not on views"
# Views deliberately do not have an enable_fts() method
with pytest.raises(AttributeError):
db["hello"].enable_fts() # type: ignore[union-attr]
@pytest.mark.parametrize(
@ -700,3 +732,17 @@ def test_search_quote(fresh_db):
list(table.search(query))
# No exception with quote=True
list(table.search(query, quote=True))
def test_enable_fts_cli_on_view_errors(tmpdir):
db_path = str(tmpdir / "test.db")
db = Database(db_path)
db["t"].insert({"text": "hello"})
db.create_view("v", "select * from t")
db.close()
from click.testing import CliRunner
from sqlite_utils import cli as cli_module
result = CliRunner().invoke(cli_module.cli, ["enable-fts", db_path, "v", "text"])
assert result.exit_code == 1
assert result.output.strip() == "Error: Table v is actually a view"

9
tests/test_gis.py
View file

@ -6,12 +6,6 @@ from sqlite_utils.cli import cli
from sqlite_utils.db import Database
from sqlite_utils.utils import find_spatialite, sqlite3
try:
import sqlean # type: ignore[import-not-found]
except ImportError:
sqlean = None
pytestmark = [
pytest.mark.skipif(
not find_spatialite(), reason="Could not find SpatiaLite extension"
@ -20,9 +14,6 @@ pytestmark = [
not hasattr(sqlite3.Connection, "enable_load_extension"),
reason="sqlite3.Connection missing enable_load_extension",
),
pytest.mark.skipif(
sqlean is not None, reason="sqlean.py is not compatible with SpatiaLite"
),
]

4
tests/test_m2m.py
View file

@ -139,9 +139,9 @@ def test_m2m_lookup(fresh_db):
def test_m2m_requires_either_records_or_lookup(fresh_db):
people = fresh_db.table("people", pk="id").insert({"name": "Wahyu"})
with pytest.raises(AssertionError):
with pytest.raises(ValueError):
people.m2m("tags")
with pytest.raises(AssertionError):
with pytest.raises(ValueError):
people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"})

110
tests/test_migrations.py
View file

@ -14,7 +14,7 @@ def migrations():
@migrations()
def m002(db):
db["cats"].create({"name": str})
db.query("insert into dogs (name) values ('Pancakes')")
db.execute("insert into dogs (name) values ('Pancakes')")
return migrations
@ -32,7 +32,7 @@ def migrations_not_ordered_alphabetically():
@migrations()
def m001(db):
db["cats"].create({"name": str})
db.query("insert into dogs (name) values ('Pancakes')")
db.execute("insert into dogs (name) values ('Pancakes')")
return migrations
@ -80,6 +80,88 @@ def test_order_does_not_matter(migrations, migrations_not_ordered_alphabetically
assert db1.schema == db2.schema
def test_applied_at_is_a_string(migrations):
db = sqlite_utils.Database(memory=True)
migrations.apply(db)
applied = migrations.applied(db)
assert len(applied) == 2
for migration in applied:
# applied_at is the TEXT timestamp straight from the
# _sqlite_migrations table, e.g. "2026-07-04 12:00:00.000000+00:00"
assert isinstance(migration.applied_at, str)
assert migration.applied_at.endswith("+00:00")
def test_failing_migration_rolls_back(migrations):
@migrations()
def m003(db):
db["birds"].create({"name": str})
db.execute("insert into dogs (name) values ('Dozer')")
raise ValueError("boom")
db = sqlite_utils.Database(memory=True)
with pytest.raises(ValueError):
migrations.apply(db)
# m001 and m002 committed before the failure and stay applied
assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"}
assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"]
assert [m.name for m in migrations.applied(db)] == ["m001", "m002"]
# Everything m003 did was rolled back and it is still pending
assert [m.name for m in migrations.pending(db)] == ["m003"]
def test_rerun_after_failure_applies_each_migration_once():
state = {"fail": True}
migrations = Migrations("test")
@migrations()
def m001(db):
db["dogs"].insert({"name": "Cleo"})
@migrations()
def m002(db):
db["dogs"].insert({"name": "Pancakes"})
if state["fail"]:
raise ValueError("boom")
db = sqlite_utils.Database(memory=True)
with pytest.raises(ValueError):
migrations.apply(db)
state["fail"] = False
migrations.apply(db)
# m001 must not have been re-applied, m002 applied exactly once
assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"]
assert [m.name for m in migrations.applied(db)] == ["m001", "m002"]
def test_non_transactional_migration_allows_vacuum(tmpdir):
path = str(tmpdir / "test.db")
db = sqlite_utils.Database(path)
migrations = Migrations("test")
@migrations()
def m001(db):
db["dogs"].insert({"name": "Cleo"})
@migrations(transactional=False)
def m002(db):
db.execute("VACUUM")
migrations.apply(db)
assert [m.name for m in migrations.applied(db)] == ["m001", "m002"]
db.close()
def test_apply_composes_inside_outer_transaction(migrations):
db = sqlite_utils.Database(memory=True)
with pytest.raises(ZeroDivisionError):
with db.atomic():
migrations.apply(db)
raise ZeroDivisionError
# The outer transaction rolled back, taking the migrations with it
assert db.table_names() == []
@pytest.mark.parametrize(
"create_table,pk",
(
@ -108,3 +190,27 @@ def test_upgrades_sqlite_migrations(migrations, create_table, pk):
assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk))
migrations.apply(db)
assert db["_sqlite_migrations"].pks == ["id"]
def test_pending_and_applied_are_read_only(migrations):
db = sqlite_utils.Database(memory=True)
assert [m.name for m in migrations.pending(db)] == ["m001", "m002"]
assert migrations.applied(db) == []
# Neither call should have created the tracking table
assert db.table_names() == []
def test_duplicate_migration_name_errors():
migrations = Migrations("test")
@migrations()
def m001(db):
pass
with pytest.raises(ValueError) as ex:
@migrations(name="m001")
def m001_again(db):
pass
assert "m001" in str(ex.value)

226
tests/test_query.py
View file

@ -1,5 +1,8 @@
import pytest
import types
from sqlite_utils.utils import sqlite3
def test_query(fresh_db):
fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}])
@ -8,6 +11,229 @@ def test_query(fresh_db):
assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}]
def test_query_executes_eagerly(fresh_db):
# The SQL runs when query() is called, not when the result is iterated,
# so errors are raised at the call site
with pytest.raises(sqlite3.OperationalError):
fresh_db.query("select * from missing_table")
def test_query_rejects_statements_that_return_no_rows(fresh_db):
fresh_db["dogs"].insert({"name": "Cleo"})
with pytest.raises(ValueError) as ex:
fresh_db.query("update dogs set name = 'Cleopaws'")
assert "execute()" in str(ex.value)
# The rejected update was rolled back, and no transaction is left open
assert not fresh_db.conn.in_transaction
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
def test_query_rejected_ddl_is_rolled_back(fresh_db):
with pytest.raises(ValueError):
fresh_db.query("create table dogs (id integer primary key)")
assert not fresh_db.conn.in_transaction
assert fresh_db.table_names() == []
def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db):
fresh_db["dogs"].insert({"name": "Cleo"})
fresh_db.begin()
fresh_db.execute("insert into dogs (name) values ('Pancakes')")
with pytest.raises(ValueError):
fresh_db.query("update dogs set name = 'Cleopaws'")
# The transaction is still open and the earlier insert is intact
assert fresh_db.conn.in_transaction
fresh_db.commit()
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo", "Pancakes"]
@pytest.mark.parametrize(
"sql",
[
"begin",
"commit",
"rollback",
"vacuum",
"detach database foo",
"/* comment */ commit",
"-- comment\nbegin",
"/* multi\nline */ -- and another\n vacuum",
"\t /* a */ /* b */ savepoint s1",
],
)
def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql):
with pytest.raises(ValueError) as ex:
fresh_db.query(sql)
assert "execute()" in str(ex.value)
assert not fresh_db.conn.in_transaction
def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db):
# A COMMIT hidden behind a leading comment must not slip past the
# keyword check - previously it committed the caller's open
# transaction before the ValueError was raised
fresh_db["dogs"].insert({"name": "Cleo"})
fresh_db.begin()
fresh_db.execute("insert into dogs (name) values ('Pancakes')")
with pytest.raises(ValueError):
fresh_db.query("/* comment */ COMMIT")
# The explicit transaction is still open and can still be rolled back
assert fresh_db.conn.in_transaction
fresh_db.rollback()
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
def test_query_error_leaves_no_transaction_open(fresh_db):
with pytest.raises(sqlite3.OperationalError):
fresh_db.query("select * from missing_table")
assert not fresh_db.conn.in_transaction
def test_query_pragma(tmpdir):
from sqlite_utils import Database
db = Database(str(tmpdir / "test.db"))
# A row-returning PRAGMA works, including one that cannot run in a transaction
assert list(db.query("pragma journal_mode = wal")) == [{"journal_mode": "wal"}]
# A PRAGMA that returns no rows raises ValueError
with pytest.raises(ValueError):
db.query("pragma user_version = 5")
db.close()
def test_query_comment_prefixed_pragma(tmpdir):
from sqlite_utils import Database
db = Database(str(tmpdir / "test.db"))
# A leading comment must not stop a PRAGMA being recognized as one -
# previously it was executed inside the savepoint guard, where
# journal mode changes are refused
assert list(db.query("-- set WAL mode\npragma journal_mode = wal")) == [
{"journal_mode": "wal"}
]
db.close()
def test_query_comment_prefixed_pragma_inside_transaction(fresh_db):
fresh_db.begin()
assert list(fresh_db.query("-- check version\npragma user_version")) == [
{"user_version": 0}
]
assert fresh_db.conn.in_transaction
fresh_db.rollback()
@pytest.mark.parametrize(
"sql,expected",
[
("select 1", "SELECT"),
(" \t\n select 1", "SELECT"),
("-- comment\nbegin", "BEGIN"),
("/* one */ /* two */ pragma user_version", "PRAGMA"),
("/* multi\nline */vacuum", "VACUUM"),
("insert into t values (1)", "INSERT"),
("-- only a comment", ""),
("/* unterminated", ""),
("", ""),
(" ", ""),
("123", ""),
],
)
def test_first_keyword(sql, expected):
from sqlite_utils.db import _first_keyword
assert _first_keyword(sql) == expected
@pytest.mark.skipif(
sqlite3.sqlite_version_info < (3, 35, 0),
reason="RETURNING requires SQLite 3.35.0 or higher",
)
def test_query_insert_returning(fresh_db):
fresh_db["dogs"].insert({"name": "Cleo"})
rows = list(
fresh_db.query("insert into dogs (name) values ('Pancakes') returning name")
)
assert rows == [{"name": "Pancakes"}]
assert fresh_db["dogs"].count == 2
@pytest.mark.skipif(
sqlite3.sqlite_version_info < (3, 35, 0),
reason="RETURNING requires SQLite 3.35.0 or higher",
)
def test_query_insert_returning_commits_without_iteration(tmpdir):
from sqlite_utils import Database
path = str(tmpdir / "test.db")
db = Database(path)
db["dogs"].insert({"name": "Cleo"})
# Never iterate over the results
db.query("insert into dogs (name) values ('Pancakes') returning name")
assert not db.conn.in_transaction
# A completely separate connection sees the new row straight away
other = sqlite3.connect(path)
assert other.execute("select count(*) from dogs").fetchone()[0] == 2
other.close()
db.close()
@pytest.mark.skipif(
sqlite3.sqlite_version_info < (3, 35, 0),
reason="RETURNING requires SQLite 3.35.0 or higher",
)
def test_query_insert_returning_partial_iteration_still_commits(tmpdir):
from sqlite_utils import Database
path = str(tmpdir / "test.db")
db = Database(path)
db["dogs"].insert({"name": "Cleo"})
row = next(
db.query(
"insert into dogs (name) values ('Pancakes'), ('Marnie') returning name"
)
)
assert row == {"name": "Pancakes"}
assert not db.conn.in_transaction
other = sqlite3.connect(path)
assert other.execute("select count(*) from dogs").fetchone()[0] == 3
other.close()
db.close()
@pytest.mark.skipif(
sqlite3.sqlite_version_info < (3, 35, 0),
reason="RETURNING requires SQLite 3.35.0 or higher",
)
def test_query_insert_returning_respects_explicit_transaction(fresh_db):
fresh_db["dogs"].insert({"name": "Cleo"})
fresh_db.begin()
rows = list(
fresh_db.query("insert into dogs (name) values ('Pancakes') returning name")
)
assert rows == [{"name": "Pancakes"}]
# Still inside the explicit transaction - not committed
assert fresh_db.conn.in_transaction
fresh_db.rollback()
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
def test_query_duplicate_column_names_are_deduped(fresh_db):
# https://github.com/simonw/sqlite-utils/issues/624
fresh_db["one"].insert({"id": 1, "value": "left"})
fresh_db["two"].insert({"id": 2, "value": "right"})
rows = list(
fresh_db.query("select one.id, two.id, one.value, two.value from one, two")
)
assert rows == [{"id": 1, "id_2": 2, "value": "left", "value_2": "right"}]
def test_query_deduped_column_avoids_existing_names(fresh_db):
# The renamed duplicate must not overwrite a real column called id_2
rows = list(fresh_db.query("select 1 as id, 2 as id, 3 as id_2"))
assert rows == [{"id": 1, "id_3": 2, "id_2": 3}]
def test_execute_returning_dicts(fresh_db):
# Like db.query() but returns a list, included for backwards compatibility
# see https://github.com/simonw/sqlite-utils/issues/290

2
tests/test_recreate.py
View file

@ -15,7 +15,7 @@ def test_recreate_ignored_for_in_memory():
def test_recreate_not_allowed_for_connection():
conn = sqlite3.connect(":memory:")
try:
with pytest.raises(AssertionError):
with pytest.raises(ValueError):
Database(conn, recreate=True)
finally:
conn.close()

7
tests/test_rows.py
View file

@ -104,3 +104,10 @@ def test_pks_and_rows_where_compound_pk(fresh_db):
(("number", 1), {"type": "number", "number": 1, "plusone": 2}),
(("number", 2), {"type": "number", "number": 2, "plusone": 3}),
]
def test_rows_where_duplicate_select_columns_are_deduped(fresh_db):
# https://github.com/simonw/sqlite-utils/issues/624
fresh_db["t"].insert({"id": 1, "name": "Cleo"})
rows = list(fresh_db["t"].rows_where(select="id, id, name"))
assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}]

2
tests/test_sniff.py
View file

@ -6,7 +6,7 @@ import pytest
sniff_dir = pathlib.Path(__file__).parent / "sniff"
@pytest.mark.parametrize("filepath", sniff_dir.glob("example*"))
@pytest.mark.parametrize("filepath", sorted(sniff_dir.glob("example*")))
def test_sniff(tmpdir, filepath):
db_path = str(tmpdir / "test.db")
runner = CliRunner()

36
tests/test_upsert.py
View file

@ -49,6 +49,42 @@ def test_upsert_error_if_no_pk(fresh_db):
table.upsert({"id": 1, "name": "Cleo"})
@pytest.mark.parametrize("use_old_upsert", (False, True))
def test_upsert_empty_record_errors(use_old_upsert):
db = Database(memory=True, use_old_upsert=use_old_upsert)
table = db["table"]
table.insert({"id": 1, "name": "Cleo"}, pk="id")
with pytest.raises(PrimaryKeyRequired):
table.upsert({}, pk="id")
with pytest.raises(PrimaryKeyRequired):
table.upsert_all([{}, {}], pk="id")
# No rows can have been inserted
assert table.count == 1
@pytest.mark.parametrize("use_old_upsert", (False, True))
def test_upsert_missing_pk_value_errors(use_old_upsert):
db = Database(memory=True, use_old_upsert=use_old_upsert)
table = db["table"]
table.insert({"id": 1, "name": "Cleo"}, pk="id")
# Records that omit the pk column entirely
with pytest.raises(PrimaryKeyRequired):
table.upsert_all([{"name": "Pancakes"}, {"name": "Marnie"}], pk="id")
# A record with an explicit None pk value can never conflict
with pytest.raises(PrimaryKeyRequired):
table.upsert({"id": None, "name": "Pancakes"}, pk="id")
assert list(table.rows) == [{"id": 1, "name": "Cleo"}]
def test_upsert_missing_compound_pk_value_errors(fresh_db):
table = fresh_db["table"]
table.insert({"a": "x", "b": "y", "v": 1}, pk=("a", "b"))
# Missing one component of the detected compound primary key
with pytest.raises(PrimaryKeyRequired):
table.upsert({"a": "x", "v": 2})
assert list(table.rows) == [{"a": "x", "b": "y", "v": 1}]
def test_upsert_error_if_existing_table_has_no_pk(fresh_db):
table = fresh_db.create_table("table", {"id": int, "name": str})
with pytest.raises(PrimaryKeyRequired):

17
tests/test_utils.py
View file

@ -83,3 +83,20 @@ def test_maximize_csv_field_size_limit():
)
def test_flatten(input, expected):
assert utils.flatten(input) == expected
@pytest.mark.parametrize(
"input,expected",
(
([], []),
(["id", "name"], ["id", "name"]),
(["id", "id"], ["id", "id_2"]),
(["id", "id", "id"], ["id", "id_2", "id_3"]),
# A renamed duplicate must not clobber a real column called id_2
(["id", "id", "id_2"], ["id", "id_3", "id_2"]),
(["id_2", "id", "id"], ["id_2", "id", "id_3"]),
(["id", "id", "id_2", "id_2"], ["id", "id_3", "id_2", "id_2_2"]),
),
)
def test_dedupe_keys(input, expected):
assert utils.dedupe_keys(input) == expected

48
tests/test_wal.py
View file

@ -1,5 +1,6 @@
import pytest
from sqlite_utils import Database
from sqlite_utils.db import TransactionError
@pytest.fixture
@ -21,3 +22,50 @@ def test_enable_disable_wal(db_path_tmpdir):
db.disable_wal()
assert "delete" == db.journal_mode
assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()]
def test_enable_wal_inside_transaction_raises(db_path_tmpdir):
db, path, tmpdir = db_path_tmpdir
db["test"].insert({"id": 1}, pk="id")
with pytest.raises(TransactionError):
with db.atomic():
db["test"].insert({"id": 2}, pk="id")
db.enable_wal()
# The atomic() block must have rolled back cleanly and the
# journal mode must be unchanged
assert db.journal_mode == "delete"
assert [r["id"] for r in db["test"].rows] == [1]
def test_disable_wal_inside_transaction_raises(db_path_tmpdir):
db, path, tmpdir = db_path_tmpdir
db.enable_wal()
db["test"].insert({"id": 1}, pk="id")
with pytest.raises(TransactionError):
with db.atomic():
db["test"].insert({"id": 2}, pk="id")
db.disable_wal()
assert db.journal_mode == "wal"
assert [r["id"] for r in db["test"].rows] == [1]
def test_ensure_autocommit_on(db_path_tmpdir):
db, path, tmpdir = db_path_tmpdir
previous_isolation_level = db.conn.isolation_level
assert previous_isolation_level is not None
with db.ensure_autocommit_on():
# isolation_level of None means driver-level autocommit mode
assert db.conn.isolation_level is None
# Restored afterwards
assert db.conn.isolation_level == previous_isolation_level
def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir):
# Calling enable_wal() when WAL is already enabled is a no-op,
# so it is fine inside a transaction
db, path, tmpdir = db_path_tmpdir
db.enable_wal()
with db.atomic():
db["test"].insert({"id": 1}, pk="id")
db.enable_wal()
assert [r["id"] for r in db["test"].rows] == [1]