diff --git a/.github/actions/setup-sqlite-version/action.yml b/.github/actions/setup-sqlite-version/action.yml deleted file mode 100644 index fdbc71c..0000000 --- a/.github/actions/setup-sqlite-version/action.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: "Setup SQLite version" -description: "Build and activate a specific SQLite version from its amalgamation archive" -inputs: - version: - description: "The SQLite version to install" - required: true - cflags: - description: "CFLAGS to use when compiling SQLite" - required: false - default: "" - skip-activate: - description: "Set to true to skip modifying the library path" - required: false - default: "false" - fallback-urls: - description: "Whitespace-separated fallback download URLs to try after sqlite.org" - required: false - default: "" -outputs: - sqlite-location: - description: "Directory containing the compiled SQLite library" - value: ${{ steps.build.outputs.sqlite-location }} -runs: - using: "composite" - steps: - - shell: bash - run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads" - - uses: actions/cache@v6 - with: - path: ${{ runner.temp }}/sqlite-versions/downloads - key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1 - - id: build - shell: bash - run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh" - env: - SQLITE_VERSION: ${{ inputs.version }} - SQLITE_CFLAGS: ${{ inputs.cflags }} - SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }} - SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }} diff --git a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh deleted file mode 100644 index 0df7290..0000000 --- a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}" -cflags="${SQLITE_CFLAGS:-}" -skip_activate="${SQLITE_SKIP_ACTIVATE:-false}" -extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}" - -case "$version_spec" in - 3.46 | 3.46.0) - sqlite_version="3.46.0" - sqlite_year="2024" - amalgamation_id="3460000" - builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip" - ;; - 3.23.1) - sqlite_version="3.23.1" - sqlite_year="2018" - amalgamation_id="3230100" - builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3230100.zip" - ;; - *) - echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh." - exit 1 - ;; -esac - -case "$(uname -s)" in - Linux) - library_name="libsqlite3.so.0" - library_path_var="LD_LIBRARY_PATH" - ;; - Darwin) - library_name="libsqlite3.dylib" - library_path_var="DYLD_LIBRARY_PATH" - ;; - *) - echo "::error::Unsupported platform $(uname -s)" - exit 1 - ;; -esac - -runner_temp="${RUNNER_TEMP:-}" -if [ -z "$runner_temp" ]; then - runner_temp="$(mktemp -d)" -fi - -filename="sqlite-amalgamation-${amalgamation_id}" -official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip" -download_dir="${runner_temp}/sqlite-versions/downloads" -source_root="${runner_temp}/sqlite-versions/source" -source_dir="${source_root}/${filename}" -build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}" -archive_path="${download_dir}/${filename}.zip" - -mkdir -p "$download_dir" "$source_root" "$build_dir" - -download_archive() { - local url - local candidate_path="${archive_path}.tmp" - local urls=("$official_url") - - for url in $builtin_fallback_urls $extra_fallback_urls; do - urls+=("$url") - done - - rm -f "$candidate_path" - for url in "${urls[@]}"; do - echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}" - if curl \ - --fail \ - --location \ - --show-error \ - --retry 5 \ - --retry-delay 2 \ - --retry-max-time 180 \ - --retry-all-errors \ - --connect-timeout 20 \ - --max-time 240 \ - --output "$candidate_path" \ - "$url"; then - mv "$candidate_path" "$archive_path" - return 0 - fi - - echo "::warning::Download failed from ${url}" - rm -f "$candidate_path" - done - - echo "::error::Could not download SQLite ${sqlite_version} amalgamation" - return 1 -} - -if [ ! -f "${source_dir}/sqlite3.c" ]; then - if [ ! -f "$archive_path" ]; then - download_archive - fi - - rm -rf "$source_dir" - unzip -q "$archive_path" -d "$source_root" -fi - -if [ ! -f "${source_dir}/sqlite3.c" ]; then - echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}" - exit 1 -fi - -read -r -a cflag_args <<< "$cflags" - -echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}" -gcc \ - -fPIC \ - -shared \ - "${cflag_args[@]}" \ - "${source_dir}/sqlite3.c" \ - "-I${source_dir}" \ - -o "${build_dir}/${library_name}" - -if [ "$library_name" = "libsqlite3.so.0" ]; then - ln -sf "$library_name" "${build_dir}/libsqlite3.so" -fi - -if [ -n "${GITHUB_OUTPUT:-}" ]; then - echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT" -else - echo "sqlite-location=${build_dir}" -fi - -case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in - true | 1 | yes) - echo "Skipping ${library_path_var} activation" - ;; - *) - existing_value="${!library_path_var:-}" - if [ -n "${GITHUB_ENV:-}" ]; then - if [ -n "$existing_value" ]; then - echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV" - else - echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV" - fi - fi - echo "Added ${build_dir} to ${library_path_var}" - ;; -esac diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 23b23bd..396e2b8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,9 +12,9 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] os: [ubuntu-latest, windows-latest, macos-latest] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip @@ -29,9 +29,9 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: python-version: '3.14' cache: pip diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 7668f1b..1f55f4e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -12,9 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v7 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: python-version: "3.11" cache: pip diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index f195cba..aa2ab9b 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -18,16 +18,16 @@ jobs: "3.23.1", # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: ./.github/actions/setup-sqlite-version + uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 with: version: ${{ matrix.sqlite-version }} cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 923de2e..7e1e953 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,9 @@ jobs: numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest, macos-14] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true @@ -31,6 +31,9 @@ jobs: - name: Install SpatiaLite if: matrix.os == 'ubuntu-latest' run: sudo apt-get install libsqlite3-mod-spatialite + - name: On macOS with Python 3.10 test with sqlean.py + if: matrix.os == 'macos-latest' && matrix.python-version == '3.10' + run: pip install sqlean.py sqlite-dump - name: Build extension for --load-extension test if: matrix.os == 'ubuntu-latest' run: |- diff --git a/Justfile b/Justfile index 5caa120..f4f0e7f 100644 --- a/Justfile +++ b/Justfile @@ -8,12 +8,11 @@ @run *options: uv run -- {{options}} -# Run linters: black, flake8, mypy, ty, cog +# Run linters: black, flake8, mypy, cog @lint: just run black . --check uv run flake8 uv run mypy sqlite_utils tests - uv run ty check sqlite_utils uv run cog --check README.md docs/*.rst uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt diff --git a/README.md b/README.md index c444c64..e425461 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,9 @@ Python CLI utility and library for manipulating SQLite databases. - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance - Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as changing the type of a column - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data -- [Manage database migrations](https://sqlite-utils.datasette.io/en/stable/migrations.html) using Python migration files and the `sqlite-utils migrate` command - [Install plugins](https://sqlite-utils.datasette.io/en/stable/plugins.html) to add custom SQL functions and additional features -Upgrading from sqlite-utils 3.x? See the [4.0 upgrade guide](https://sqlite-utils.datasette.io/en/stable/upgrading.html#upgrading-from-3-x-to-4-0). - -Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqlite-utils](https://simonwillison.net/tags/sqlite-utils/). +Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). ## Installation diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c868f4..f110e2e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,137 +4,10 @@ Changelog =========== -.. _v4_1_1: +.. _unreleased: -4.1.1 (2026-07-12) ------------------- - -- ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) -- The :ref:`CLI ` and :ref:`Python API ` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) -.. _v4_1: - -4.1 (2026-07-11) ----------------- - -- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) -- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) -- New ``table.drop_index(name)`` method and ``sqlite-utils drop-index`` command for dropping an index by name. Both accept ``ignore=True``/``--ignore`` to ignore a missing index. (:issue:`626`) -- ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) -- ``sqlite-utils upsert`` can now infer the primary key of an existing table, so ``--pk`` can be omitted when upserting into a table that already has a primary key. -- ``table.transform()`` and ``table.transform_sql()`` now accept ``strict=True`` or ``strict=False`` to change a table's `SQLite strict mode `__. Omitting the option preserves the existing mode. (:issue:`787`) -- The ``sqlite-utils transform`` command now accepts ``--strict`` and ``--no-strict`` to change a table's strict mode. (:issue:`787`) - -.. _v4_0: - -4.0 (2026-07-07) ----------------- - -The 4.0 release includes some minor backwards-incompatible fixes (hence the major version number bump) and introduces three major new features: - -- :ref:`Database migrations `, providing a structured mechanism for evolving a project's schema over time. (:issue:`752`) -- :ref:`Nested transaction support ` via ``db.atomic()``, plus numerous improvements to how transactions work across the library. (:issue:`755`) -- Support for :ref:`compound foreign keys `, including creation, transformation and introspection through :ref:`table.foreign_keys `. (:issue:`594`) - -Other notable changes include: - -- Upserts now use SQLite's ``INSERT ... ON CONFLICT ... DO UPDATE SET`` syntax, detect existing table primary keys automatically and reject records that are missing required primary key values. (:issue:`652`) -- ``db.query()`` now executes immediately and rejects statements that do not return rows; use ``db.execute()`` for writes and DDL. -- CSV and TSV imports now detect column types by default, while inserts into existing tables preserve those tables' column types. (:issue:`679`) -- Foreign key handling now preserves ``ON DELETE``/``ON UPDATE`` actions during transforms and resolves referenced primary keys more accurately. (:issue:`530`) -- Column names passed to Python API methods are now matched case-insensitively, mirroring SQLite's own identifier behavior. (:issue:`760`) -- The command-line tool now emits UTF-8 JSON output by default, with ``--ascii`` available to restore escaped output. (:issue:`625`) -- ``table.extract()`` and ``extracts=`` no longer create lookup table records for all-``null`` values. (:issue:`186`) - -See :ref:`upgrading_3_to_4` for details on backwards-incompatible changes. - -The detailed release notes for the features and fixes shipped during the 4.0 pre-release cycle are available in :ref:`4.0a0 `, :ref:`4.0a1 `, :ref:`4.0rc1 `, :ref:`4.0rc2 `, :ref:`4.0rc3 ` and :ref:`4.0rc4 `. - -Bug fixes since 4.0rc4 -~~~~~~~~~~~~~~~~~~~~~~ - -- Fixed 4.0 regressions in ``insert``/``upsert`` against tables that use SQLite's implicit ``rowid`` primary key. Passing ``pk="rowid"``, ``pk="_rowid_"`` or ``pk="oid"`` now works again for rowid tables, and ``last_pk`` is set correctly. (:issue:`781`) -- Fixed ``insert(..., ignore=True)`` and ``insert_all(..., ignore=True)`` so an ignored insert that conflicts with an existing primary key row now reports that existing row in ``last_rowid`` and ``last_pk`` where possible. This also works for compound primary keys and list-mode inserts. (:issue:`783`) - -.. _v4_0rc4: - -4.0rc4 (2026-07-06) -------------------- - -- **Breaking change**: ``table.extract()`` - and the ``sqlite-utils extract`` command - no longer extract rows where every extracted column is ``null``. Those rows now keep a ``null`` value in the new foreign key column instead of pointing at an all-``null`` record in the lookup table. When extracting multiple columns, rows are still extracted if at least one of the columns has a value. (:issue:`186`) -- The ``extracts=`` option to ``table.insert()`` and friends no longer creates a lookup table record for ``None`` values - the column value stays ``null``. Previously every batch of inserted rows containing a ``None`` value would add a duplicate ``null`` record to the lookup table. -- Fixed a bug where ``table.lookup()`` inserted a duplicate row on every call if any of the lookup values were ``None``. Lookup values are now compared using ``IS`` so that ``None`` values match existing rows correctly. -- JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) -- ``--no-headers`` now omits the header row from ``--fmt`` and ``--table`` output, not just CSV and TSV output. (:issue:`566`) -- ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) -- Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) -- Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. -- Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. -- Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. -- Fixed exception masking when a statement destroys the enclosing transaction. An error such as a ``RAISE(ROLLBACK)`` trigger or ``INSERT OR ROLLBACK`` conflict rolls back the whole transaction, destroying every savepoint - the cleanup in ``db.atomic()`` and ``db.query()`` then failed with ``OperationalError: no such savepoint`` (or ``cannot rollback - no transaction is active``), hiding the original ``IntegrityError`` from code that tried to catch it. Cleanup now checks whether a transaction is still open first, so the original exception propagates. -- ``sqlite-utils migrate --list`` is now read-only even when the migrations file uses the legacy ``sqlite_migrate.Migrations`` class, whose listing methods create the ``_sqlite_migrations`` table as a side effect. The listing now runs inside a transaction that is rolled back. -- ``sqlite-utils insert ... --pk `` and ``sqlite-utils extract `` now show a clean ``Error:`` message instead of a raw Python traceback. The ``extract`` command also shows a clean error when pointed at a view. -- Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. -- ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. -- ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. -- ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. -- Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. -- Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. -- Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. -- The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. -- ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. -- Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. - -.. _v4_0rc3: - -4.0rc3 (2026-07-05) -------------------- - -Breaking changes -~~~~~~~~~~~~~~~~ - -- :ref:`table.foreign_keys ` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) -- Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. -- The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`) - -Compound foreign key support -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- Tables can now be created with :ref:`compound foreign keys `, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. -- ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. -- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table. -- ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. - -Other foreign key improvements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema. -- ``table.add_foreign_key()`` accepts new ``on_delete=`` and ``on_update=`` parameters for creating foreign keys with actions, e.g. ``table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")``. (:issue:`530`) -- Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. -- Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. - -Case-insensitive column matching -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Column names passed to Python API methods are now matched against the table schema case-insensitively, mirroring how SQLite itself treats identifiers. Previously many methods accepted mixed-case identifiers in the SQL they generated but then failed - or silently did nothing - when performing Python-side comparisons against the schema. (:issue:`760`) Fixes include: - -- ``table.insert()`` and ``table.upsert()`` now populate ``table.last_pk`` correctly when the ``pk=`` argument uses different casing to the table schema or the record keys - previously this raised a ``KeyError`` after the row had already been written. -- Upserts no longer raise or misbehave when the casing of ``pk=`` differs from the casing of the record keys. The primary key columns are correctly excluded from the generated ``DO UPDATE SET`` clause. -- ``table.transform()`` arguments ``types=``, ``rename=``, ``drop=``, ``pk=``, ``not_null=``, ``defaults=``, ``column_order=`` and ``drop_foreign_keys=`` all resolve column names case-insensitively. Previously options like ``rename={"name": "title"}`` against a column called ``Name`` were silently ignored. -- ``db.create_table(..., transform=True)`` now recognizes existing columns that differ only by case, instead of attempting to add them again and failing with ``duplicate column name``. The casing used in the existing schema is preserved. -- ``table.lookup()`` returns the primary key value even if ``pk=`` casing differs from the schema, and recognizes existing unique indexes case-insensitively instead of creating redundant ones. -- ``table.extract()`` and ``table.convert()`` - including ``multi=True`` and ``output=`` - accept column names in any casing. -- Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive. -- ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``. - -Everything else -~~~~~~~~~~~~~~~ - -- Fixed a bug where ``table.transform()`` could convert ``DEFAULT TRUE``, ``DEFAULT FALSE`` and ``DEFAULT NULL`` column defaults into quoted string defaults when rebuilding a table. Thanks, `Vincent Gao `__. (`#764 `__) - -.. _v4_0rc2: - -4.0rc2 (2026-07-04) -------------------- +Unreleased +---------- Breaking changes: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 9fafe28..cbac48b 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -19,7 +19,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. go_first = [ "query", "memory", "insert", "upsert", "bulk", "search", "transform", "extract", "schema", "insert-files", "analyze-tables", "convert", "tables", "views", "rows", - "triggers", "indexes", "create-database", "create-table", "create-index", "drop-index", + "triggers", "indexes", "create-database", "create-table", "create-index", "migrate", "enable-fts", "populate-fts", "rebuild-fts", "disable-fts" ] refs = { @@ -46,7 +46,6 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "add-foreign-keys": "cli_add_foreign_keys", "index-foreign-keys": "cli_index_foreign_keys", "create-index": "cli_create_index", - "drop-index": "cli_drop_index", "enable-wal": "cli_wal", "enable-counts": "cli_enable_counts", "bulk": "cli_bulk", @@ -110,10 +109,6 @@ See :ref:`cli_query`. "select * from chickens where age > :age" \ -p age 1 - Pass "-" as the SQL to read the query from standard input: - - echo "select * from chickens" | sqlite-utils data.db - - Options: --attach ... Additional databases to attach - specify alias and filepath @@ -121,7 +116,7 @@ See :ref:`cli_query`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit headers from CSV/TSV and table/--fmt output + --no-headers Omit CSV headers -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, @@ -134,13 +129,11 @@ See :ref:`cli_query`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --ascii Escape non-ASCII characters in JSON output as - \uXXXX -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query - --functions TEXT Python code or a file path defining custom SQL - functions; can be used multiple times + --functions TEXT Python code or file path defining custom SQL + functions --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -182,8 +175,8 @@ See :ref:`cli_memory`. sqlite-utils memory animals.csv --schema Options: - --functions TEXT Python code or a file path defining custom SQL - functions; can be used multiple times + --functions TEXT Python code or file path defining custom SQL + functions --attach ... Additional databases to attach - specify alias and filepath --flatten Flatten nested JSON objects, so {"foo": {"bar": @@ -192,7 +185,7 @@ See :ref:`cli_memory`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit headers from CSV/TSV and table/--fmt output + --no-headers Omit CSV headers -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, @@ -205,8 +198,6 @@ See :ref:`cli_memory`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --ascii Escape non-ASCII characters in JSON output as - \uXXXX -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query @@ -231,7 +222,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr :: - Usage: sqlite-utils insert [OPTIONS] PATH TABLE [FILE] + Usage: sqlite-utils insert [OPTIONS] PATH TABLE FILE Insert records from FILE into a table, creating the table if it does not already exist. @@ -247,9 +238,6 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr - Use --lines to write each incoming line to a column called "line" - Use --text to write the entire input to a column called "text" - Use --type column-name type to override the type automatically chosen when the - table is created. - You can also use --convert to pass a fragment of Python code that will be used to convert each input. @@ -276,20 +264,8 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr echo 'A bunch of words' | sqlite-utils insert words.db words - \ --text --convert '({"word": w} for w in text.split())' - Instead of a FILE you can use --code to provide a block of Python code that - defines the rows to insert, as either a rows() function that yields - dictionaries or a "rows" iterable. --code can also be a path to a .py file: - - sqlite-utils insert data.db creatures --code ' - def rows(): - yield {"id": 1, "name": "Cleo"} - yield {"id": 2, "name": "Suna"} - ' --pk id - Options: --pk TEXT Columns to use as the primary key, e.g. id - --code TEXT Python code defining a rows() function or iterable - of rows to insert --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON @@ -310,7 +286,6 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column - --type ... Column types to use when creating the table --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -332,17 +307,12 @@ See :ref:`cli_upsert`. :: - Usage: sqlite-utils upsert [OPTIONS] PATH TABLE [FILE] + Usage: sqlite-utils upsert [OPTIONS] PATH TABLE FILE Upsert records based on their primary key. Works like 'insert' but if an incoming record has a primary key that matches an existing record the existing record will be updated. - If the table already exists and has a primary key, --pk can be omitted. - - Use --type column-name type to override the type automatically chosen when the - table is created. - Example: echo '[ @@ -352,8 +322,7 @@ See :ref:`cli_upsert`. Options: --pk TEXT Columns to use as the primary key, e.g. id - --code TEXT Python code defining a rows() function or iterable - of rows to insert + [required] --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON @@ -374,7 +343,6 @@ See :ref:`cli_upsert`. --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column - --type ... Column types to use when creating the table --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -407,8 +375,7 @@ See :ref:`cli_bulk`. Options: --batch-size INTEGER Commit every X records - --functions TEXT Python code or a file path defining custom SQL - functions; can be used multiple times + --functions TEXT Python code or file path defining custom SQL functions --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON @@ -455,7 +422,7 @@ See :ref:`cli_search`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit headers from CSV/TSV and table/--fmt output + --no-headers Omit CSV headers -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -468,7 +435,6 @@ See :ref:`cli_search`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -508,8 +474,6 @@ See :ref:`cli_transform_table`. Add a foreign key constraint from a column to another table with another column --drop-foreign-key TEXT Drop foreign key constraint for this column - --strict / --no-strict Enable or disable STRICT mode (default: - preserve current mode) --sql Output SQL without executing it --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -647,11 +611,6 @@ See :ref:`cli_convert`. "value" is a variable with the column value to be converted. - CODE can also be a reference to a callable that takes the value, for example: - - sqlite-utils convert my.db mytable date r.parsedate - sqlite-utils convert my.db mytable data json.loads --import json - Use "-" for CODE to read Python code from standard input. The following common operations are available as recipe functions: @@ -665,6 +624,7 @@ See :ref:`cli_convert`. errors: 'Optional[object]' = None) -> 'Optional[str]' Parse a date and convert it to ISO date format: yyyy-mm-dd + - dayfirst=True: treat xx as the day in xx/yy/zz - yearfirst=True: treat xx as the year in xx/yy/zz - errors=r.IGNORE to ignore values that cannot be parsed @@ -674,6 +634,7 @@ See :ref:`cli_convert`. False, errors: 'Optional[object]' = None) -> 'Optional[str]' Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS + - dayfirst=True: treat xx as the day in xx/yy/zz - yearfirst=True: treat xx as the year in xx/yy/zz - errors=r.IGNORE to ignore values that cannot be parsed @@ -727,7 +688,7 @@ See :ref:`cli_tables`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit headers from CSV/TSV and table/--fmt output + --no-headers Omit CSV headers -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -740,7 +701,6 @@ See :ref:`cli_tables`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --ascii Escape non-ASCII characters in JSON output as \uXXXX --columns Include list of columns for each table --schema Include schema for each table --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -770,7 +730,7 @@ See :ref:`cli_views`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit headers from CSV/TSV and table/--fmt output + --no-headers Omit CSV headers -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -783,7 +743,6 @@ See :ref:`cli_views`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --ascii Escape non-ASCII characters in JSON output as \uXXXX --columns Include list of columns for each view --schema Include schema for each view --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -818,7 +777,7 @@ See :ref:`cli_rows`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit headers from CSV/TSV and table/--fmt output + --no-headers Omit CSV headers -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, @@ -831,8 +790,6 @@ See :ref:`cli_rows`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --ascii Escape non-ASCII characters in JSON output as - \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -860,7 +817,7 @@ See :ref:`cli_triggers`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit headers from CSV/TSV and table/--fmt output + --no-headers Omit CSV headers -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -873,7 +830,6 @@ See :ref:`cli_triggers`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -901,7 +857,7 @@ See :ref:`cli_indexes`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit headers from CSV/TSV and table/--fmt output + --no-headers Omit CSV headers -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -914,7 +870,6 @@ See :ref:`cli_indexes`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -960,10 +915,10 @@ See :ref:`cli_create_table`. sqlite-utils create-table my.db people \ id integer \ name text \ - height real \ + height float \ photo blob --pk id - Valid column types are text, integer, real, float and blob. + Valid column types are text, integer, float and blob. Options: --pk TEXT Column to use as primary key @@ -1009,29 +964,6 @@ See :ref:`cli_create_index`. -h, --help Show this message and exit. -.. _cli_ref_drop_index: - -drop-index -========== - -See :ref:`cli_drop_index`. - -:: - - Usage: sqlite-utils drop-index [OPTIONS] PATH TABLE INDEX - - Drop an index by index name from the specified table - - Example: - - sqlite-utils drop-index chickens.db chickens idx_chickens_name - - Options: - --ignore Ignore if index does not exist - --load-extension TEXT Path to SQLite extension, with optional :entrypoint - -h, --help Show this message and exit. - - .. _cli_ref_migrate: migrate @@ -1081,7 +1013,7 @@ See :ref:`cli_fts`. Usage: sqlite-utils enable-fts [OPTIONS] PATH TABLE COLUMN... - Enable full-text search for specific table and columns + Enable full-text search for specific table and columns" Example: diff --git a/docs/cli.rst b/docs/cli.rst index 2e506dd..c9389d8 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -29,16 +29,6 @@ The ``sqlite-utils query`` command lets you run queries directly against a SQLit .. note:: In Python: :ref:`db.query() ` CLI reference: :ref:`sqlite-utils query ` -Pass ``-`` as the SQL query to read the query from standard input. This is useful for longer queries that would otherwise require careful shell escaping, or for piping in SQL generated by another tool: - -.. code-block:: bash - - echo "select * from dogs" | sqlite-utils query dogs.db - - -.. code-block:: bash - - sqlite-utils query dogs.db - < query.sql - .. _cli_query_json: Returning JSON @@ -55,8 +45,6 @@ The default format returned for queries is JSON: [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] -If the query returns more than one column with the same name, later occurrences are renamed with a numeric suffix - ``select 1 as id, 2 as id`` returns ``[{"id": 1, "id_2": 2}]``. This only applies to JSON output: :ref:`CSV and TSV ` and :ref:`table ` output keep the duplicate column headers unchanged. - .. _cli_query_nl: Newline-delimited JSON @@ -121,33 +109,6 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] -.. _cli_query_json_ascii: - -Unicode characters in JSON -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -JSON output includes unicode characters directly, without escaping them: - -.. code-block:: bash - - sqlite-utils dogs.db "select '日本語' as text" - -.. code-block:: output - - [{"text": "日本語"}] - -Use ``--ascii`` to escape non-ASCII characters as ``\uXXXX`` sequences instead: - -.. code-block:: bash - - sqlite-utils dogs.db "select '日本語' as text" --ascii - -.. code-block:: output - - [{"text": "\u65e5\u672c\u8a9e"}] - -The ``--ascii`` option can help on systems that cannot display or process UTF-8, such as Windows consoles using a legacy code page. On Windows, setting the ``PYTHONUTF8=1`` environment variable is an alternative fix for ``UnicodeEncodeError`` crashes when redirecting output to a file. - .. _cli_query_binary_json: Binary data in JSON @@ -361,7 +322,7 @@ To return the first column of each result as raw data, separated by newlines, us Using named parameters ---------------------- -You can pass named parameters to the query using ``-p name value``: +You can pass named parameters to the query using ``-p``: .. code-block:: bash @@ -424,9 +385,6 @@ The ``--functions`` option can be used multiple times to load functions from mul from urllib.parse import urlparse return urlparse(url).path' -.. note:: - In Python: :ref:`db.register_function() ` - .. _cli_query_extensions: SQLite extensions @@ -1027,9 +985,6 @@ To show more than 10 common values, use ``--common-limit 20``. To skip the most sqlite-utils analyze-tables github.db tags --common-limit 20 --no-least -.. note:: - In Python: :ref:`table.analyze_column() ` CLI reference: :ref:`sqlite-utils analyze-tables ` - .. _cli_analyze_tables_save: Saving the analyzed table details @@ -1197,9 +1152,6 @@ You can delete all the existing rows in the table before inserting the new recor You can add the ``--analyze`` option to run ``ANALYZE`` against the table after the rows have been inserted. -.. note:: - In Python: :ref:`table.insert_all() ` CLI reference: :ref:`sqlite-utils insert ` - .. _cli_inserting_data_binary: Inserting binary data @@ -1345,8 +1297,6 @@ A progress bar is displayed when inserting data from a file. You can hide the pr By default, column types are automatically detected for CSV or TSV files - resulting in a mix of ``TEXT``, ``INTEGER`` and ``REAL`` columns. To disable type detection and treat all columns as ``TEXT``, use the ``--no-detect-types`` option. -Detected types are only applied when the table is created by the command. Inserting CSV or TSV data into a table that already exists leaves the existing column types unchanged - values are inserted using the table's existing schema. - For example, given a ``creatures.csv`` file containing this: .. code-block:: @@ -1375,25 +1325,6 @@ Will produce this schema with automatically detected types: "weight" REAL ); -.. _cli_insert_csv_tsv_column_types: - -Overriding column types ------------------------ - -Use ``--type column-name type`` to override the type automatically chosen when the table is created. This option can be used more than once, and works with both ``insert`` and ``upsert``: - -.. code-block:: bash - - sqlite-utils insert places.db places places.csv --csv \ - --type zipcode text \ - --type score real - -This is useful for values such as ZIP codes, which may look like integers but should be stored as ``TEXT`` to preserve leading zeros. - -The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or ``BLOB``. Column types are matched case-insensitively. - -As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged. - To disable type detection and treat all columns as TEXT, use ``--no-detect-types``: .. code-block:: bash @@ -1589,27 +1520,6 @@ The result looks like this: COMMIT; -.. _cli_insert_code: - -Inserting rows generated by Python code -======================================= - -Instead of providing a ``FILE`` to import, you can use the ``--code`` option to pass a block of Python code that generates the rows to insert. This is the command-line equivalent of calling ``db["creatures"].insert_all(rows())`` from the :ref:`Python API `. - -Your code should define either a ``rows()`` function that returns or yields dictionaries, or a ``rows`` iterable such as a list of dictionaries: - -.. code-block:: bash - - sqlite-utils insert data.db creatures --code ' - def rows(): - yield {"id": 1, "name": "Cleo"} - yield {"id": 2, "name": "Suna"} - ' --pk id - -``--code`` can also be given a path to a Python ``.py`` file. - -The ``--code`` option works with both ``sqlite-utils insert`` and ``sqlite-utils upsert``, and composes with table options such as ``--pk``, ``--replace``, ``--alter``, ``--not-null`` and ``--default``. It cannot be combined with a ``FILE`` argument or with input format options such as ``--csv`` or ``--convert``. - .. _cli_insert_replace: Insert-replacing data @@ -1624,9 +1534,6 @@ To replace a dog with in ID of 2 with a new record, run the following: echo '{"id": 2, "name": "Pancakes", "age": 3}' | \ sqlite-utils insert dogs.db dogs - --pk=id --replace -.. note:: - In Python: :ref:`table.insert(..., replace=True) ` CLI reference: :ref:`sqlite-utils insert ` - .. _cli_upsert: Upserting data @@ -1645,17 +1552,12 @@ For example: This will update the dog with an ID of 2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is. -If the table already exists and has a primary key, you can omit the ``--pk`` option and ``sqlite-utils`` will use that existing primary key. - The command will fail if you reference columns that do not exist on the table. To automatically create missing columns, use the ``--alter`` option. .. note:: ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. -.. note:: - In Python: :ref:`table.upsert() ` CLI reference: :ref:`sqlite-utils upsert ` - .. _cli_bulk: Executing SQL in bulk @@ -1857,9 +1759,6 @@ You can include named parameters in your where clause and populate them using on The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. -.. note:: - In Python: :ref:`table.convert() ` CLI reference: :ref:`sqlite-utils convert ` - .. _cli_convert_import: Importing additional modules @@ -2158,9 +2057,6 @@ If a table with the same name already exists, you will get an error. You can cho You can also pass ``--transform`` to transform the existing table to match the new schema. See :ref:`python_api_explicit_create` in the Python library documentation for details of how this option works. -.. note:: - In Python: :ref:`table.create() ` CLI reference: :ref:`sqlite-utils create-table ` - .. _cli_renaming_tables: Renaming a table @@ -2174,9 +2070,6 @@ Yo ucan rename a table using the ``rename-table`` command: Pass ``--ignore`` to ignore any errors caused by the table not existing, or the new name already being in use. -.. note:: - In Python: :ref:`db.rename_table() ` CLI reference: :ref:`sqlite-utils rename-table ` - .. _cli_duplicate_table: Duplicating tables @@ -2188,9 +2081,6 @@ The ``duplicate`` command duplicates a table - creating a new table with the sam sqlite-utils duplicate books.db authors authors_copy -.. note:: - In Python: :ref:`table.duplicate() ` CLI reference: :ref:`sqlite-utils duplicate ` - .. _cli_drop_table: Dropping tables @@ -2204,15 +2094,12 @@ You can drop a table using the ``drop-table`` command: Use ``--ignore`` to ignore the error if the table does not exist. -.. note:: - In Python: :ref:`table.drop() ` CLI reference: :ref:`sqlite-utils drop-table ` - .. _cli_transform_table: Transforming tables =================== -The ``transform`` command allows you to apply complex transformations to a table that cannot be implemented using a regular SQLite ``ALTER TABLE`` command. See :ref:`python_api_transform` for details of how this works. By default, the ``transform`` command preserves a table's ``STRICT`` mode. +The ``transform`` command allows you to apply complex transformations to a table that cannot be implemented using a regular SQLite ``ALTER TABLE`` command. See :ref:`python_api_transform` for details of how this works. The ``transform`` command preserves a table's ``STRICT`` mode. .. code-block:: bash @@ -2258,12 +2145,6 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi ``--add-foreign-key column other_table other_column`` Add a foreign key constraint to ``column`` pointing to ``other_table.other_column``. -``--strict`` - Convert the table to a `SQLite STRICT table `__. The command fails if the available SQLite version does not support strict tables. If existing rows contain values that are incompatible with their declared column types the transformation fails and the original table is left unchanged. - -``--no-strict`` - Convert a strict table back to a regular non-strict table. - If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example: .. code-block:: bash @@ -2290,9 +2171,6 @@ If you want to see the SQL that will be executed to make the change without actu DROP TABLE "roadside_attractions"; ALTER TABLE "roadside_attractions_new_4033a60276b9" RENAME TO "roadside_attractions"; -.. note:: - In Python: :ref:`table.transform() ` CLI reference: :ref:`sqlite-utils transform ` - .. _cli_transform_table_add_primary_key_to_rowid: Adding a primary key to a rowid table @@ -2379,8 +2257,6 @@ The ``sqlite-utils extract`` command can be used to extract specified columns in Take a look at the Python API documentation for :ref:`python_api_extract` for a detailed description of how this works, including examples of table schemas before and after running an extraction operation. -Rows where every extracted column is ``null`` are not extracted - those rows get a ``null`` value in their new foreign key column and no record is created for them in the lookup table. - The command takes a database, table and one or more columns that should be extracted. To extract the ``species`` column from the ``trees`` table you would run: .. code-block:: bash @@ -2470,9 +2346,6 @@ After running the above, the command ``sqlite-utils schema global.db`` reveals t CREATE UNIQUE INDEX "idx_countries_country_name" ON "countries" ("country", "name"); -.. note:: - In Python: :ref:`table.extract() ` CLI reference: :ref:`sqlite-utils extract ` - .. _cli_create_view: Creating views @@ -2494,9 +2367,6 @@ You can create a view using the ``create-view`` command: Use ``--replace`` to replace an existing view of the same name, and ``--ignore`` to do nothing if a view already exists. -.. note:: - In Python: :ref:`db.create_view() ` CLI reference: :ref:`sqlite-utils create-view ` - .. _cli_drop_view: Dropping views @@ -2510,9 +2380,6 @@ You can drop a view using the ``drop-view`` command: Use ``--ignore`` to ignore the error if the view does not exist. -.. note:: - In Python: :ref:`view.drop() ` CLI reference: :ref:`sqlite-utils drop-view ` - .. _cli_add_column: Adding columns @@ -2553,9 +2420,6 @@ You can set a ``NOT NULL DEFAULT 'x'`` constraint on the new column using ``--no sqlite-utils add-column mydb.db dogs friends_count integer --not-null-default 0 -.. note:: - In Python: :ref:`table.add_column() ` CLI reference: :ref:`sqlite-utils add-column ` - .. _cli_add_column_alter: Adding columns automatically on insert/update @@ -2567,9 +2431,6 @@ You can use the ``--alter`` option to automatically add new columns if the data sqlite-utils insert dogs.db dogs new-dogs.json --pk=id --alter -.. note:: - In Python: :ref:`table.insert(..., alter=True) ` - .. _cli_add_foreign_key: Adding foreign key constraints @@ -2597,9 +2458,6 @@ Add ``--ignore`` to ignore an existing foreign key (as opposed to returning an e See :ref:`python_api_add_foreign_key` in the Python API documentation for further details, including how the automatic table guessing mechanism works. -.. note:: - In Python: :ref:`table.add_foreign_key() ` CLI reference: :ref:`sqlite-utils add-foreign-key ` - .. _cli_add_foreign_keys: Adding multiple foreign keys at once @@ -2615,9 +2473,6 @@ Adding a foreign key requires a ``VACUUM``. On large databases this can be an ex When you are using this command each foreign key needs to be defined in full, as four arguments - the table, column, other table and other column. -.. note:: - In Python: :ref:`db.add_foreign_keys() ` CLI reference: :ref:`sqlite-utils add-foreign-keys ` - .. _cli_index_foreign_keys: Adding indexes for all foreign keys @@ -2629,9 +2484,6 @@ If you want to ensure that every foreign key column in your database has a corre sqlite-utils index-foreign-keys books.db -.. note:: - In Python: :ref:`db.index_foreign_keys() ` CLI reference: :ref:`sqlite-utils index-foreign-keys ` - .. _cli_defaults_not_null: Setting defaults and not null constraints @@ -2647,9 +2499,6 @@ You can use the ``--not-null`` and ``--default`` options (to both ``insert`` and --default age 2 \ --default score 5 -.. note:: - In Python: :ref:`not_null= and defaults= arguments ` - .. _cli_create_index: Creating indexes @@ -2681,25 +2530,6 @@ If your column names are already prefixed with a hyphen you'll need to manually Add the ``--analyze`` option to run ``ANALYZE`` against the index after it has been created. -.. note:: - In Python: :ref:`table.create_index() ` CLI reference: :ref:`sqlite-utils create-index ` - -.. _cli_drop_index: - -Dropping indexes -================ - -You can drop an index from an existing table using the ``drop-index`` command: - -.. code-block:: bash - - sqlite-utils drop-index mydb.db mytable idx_mytable_col1 - -Use ``--ignore`` to ignore the error if the index does not exist on that table. - -.. note:: - In Python: :ref:`table.drop_index() ` CLI reference: :ref:`sqlite-utils drop-index ` - .. _cli_fts: Configuring full-text search @@ -2753,9 +2583,6 @@ You can rebuild every FTS table by running ``rebuild-fts`` without passing any t sqlite-utils rebuild-fts mydb.db -.. note:: - In Python: :ref:`table.enable_fts() ` CLI reference: :ref:`sqlite-utils enable-fts ` - .. _cli_search: Executing searches @@ -2812,9 +2639,6 @@ Use the ``--sql`` option to output the SQL that would be executed, rather than r order by "documents_fts".rank -.. note:: - In Python: :ref:`table.search() ` CLI reference: :ref:`sqlite-utils search ` - .. _cli_enable_counts: Enabling cached counts @@ -2838,9 +2662,6 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y sqlite-utils reset-counts mydb.db -.. note:: - In Python: :ref:`table.enable_counts() ` CLI reference: :ref:`sqlite-utils enable-counts ` - .. _cli_analyze: Optimizing index usage with ANALYZE @@ -2864,9 +2685,6 @@ You can run it against specific tables, or against specific named indexes, by pa You can also run ``ANALYZE`` as part of another command using the ``--analyze`` option. This is supported by the ``create-index``, ``insert`` and ``upsert`` commands. -.. note:: - In Python: :ref:`db.analyze() ` CLI reference: :ref:`sqlite-utils analyze ` - .. _cli_vacuum: Vacuum @@ -2878,9 +2696,6 @@ You can run VACUUM to optimize your database like so: sqlite-utils vacuum mydb.db -.. note:: - In Python: :ref:`db.vacuum() ` CLI reference: :ref:`sqlite-utils vacuum ` - .. _cli_optimize: Optimize @@ -2904,9 +2719,6 @@ To optimize specific tables rather than every FTS table, pass those tables as ex sqlite-utils optimize mydb.db table_1 table_2 -.. note:: - In Python: :ref:`table.optimize() ` CLI reference: :ref:`sqlite-utils optimize ` - .. _cli_wal: WAL mode @@ -2926,9 +2738,6 @@ You can disable WAL mode using ``disable-wal``: Both of these commands accept one or more database files as arguments. -.. note:: - In Python: :ref:`db.enable_wal() and db.disable_wal() ` CLI reference: :ref:`sqlite-utils enable-wal ` - .. _cli_dump: Dumping the database to SQL @@ -2944,9 +2753,6 @@ The ``dump`` command outputs a SQL dump of the schema and full contents of the s ... COMMIT; -.. note:: - In Python: :ref:`db.iterdump() ` CLI reference: :ref:`sqlite-utils dump ` - .. _cli_load_extension: Loading SQLite extensions @@ -2994,9 +2800,6 @@ Eight (case-insensitive) types are allowed: * GEOMETRYCOLLECTION * GEOMETRY -.. note:: - In Python: :ref:`table.add_geometry_column() ` CLI reference: :ref:`sqlite-utils add-geometry-column ` - .. _cli_spatialite_indexes: Adding spatial indexes @@ -3010,9 +2813,6 @@ Once you have a geometry column, you can speed up bounding box queries by adding See this `SpatiaLite Cookbook recipe `__ for examples of how to use a spatial index. -.. note:: - In Python: :ref:`table.create_spatial_index() ` CLI reference: :ref:`sqlite-utils create-spatial-index ` - .. _cli_install: Installing packages diff --git a/docs/conf.py b/docs/conf.py index 4f29b39..04a2301 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,8 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import inspect -from pathlib import Path -from subprocess import Popen, PIPE, check_output -import sys +from subprocess import Popen, PIPE +from beanbag_docutils.sphinx.ext.github import github_linkcode_resolve # This file is execfile()d with the current directory set to its # containing dir. @@ -47,52 +45,14 @@ extlinks = { } -def _linkcode_git_ref(): - try: - return check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip() - except Exception: - return "main" - - def linkcode_resolve(domain, info): - if domain != "py": - return None - - module_name = info.get("module") - if not module_name or module_name.split(".")[0] != "sqlite_utils": - return None - - module = sys.modules.get(module_name) - if module is None: - return None - - obj = module - for part in info.get("fullname", "").split("."): - obj = getattr(obj, part, None) - if obj is None: - return None - - if isinstance(obj, property): - obj = obj.fget - - try: - obj = inspect.unwrap(obj) - source_file = inspect.getsourcefile(obj) - _, line_number = inspect.getsourcelines(obj) - except Exception: - return None - - if source_file is None: - return None - - try: - filename = Path(source_file).resolve().relative_to(Path(__file__).parent.parent) - except ValueError: - return None - - return ( - "https://github.com/simonw/sqlite-utils/blob/" - f"{_linkcode_git_ref()}/{filename}#L{line_number}" + return github_linkcode_resolve( + domain=domain, + info=info, + allowed_module_names=["sqlite_utils"], + github_org_id="simonw", + github_repo_id="sqlite-utils", + branch="main", ) diff --git a/docs/installation.rst b/docs/installation.rst index 1333f5d..beb6d4a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -52,15 +52,15 @@ On some platforms the ability to load additional extensions (via ``conn.load_ext You may also see the error ``sqlite3.OperationalError: table sqlite_master may not be modified`` when trying to alter an existing table. -You can work around these limitations by installing the `pysqlite3 `__ package, which provides a drop-in replacement for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions. +You can work around these limitations by installing either the `pysqlite3 `__ package or the `sqlean.py `__ package, both of which provide drop-in replacements for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions. -To install ``pysqlite3`` run the following: +To install ``sqlean.py`` (which has compiled binary wheels available for all major platforms) run the following: .. code-block:: bash - sqlite-utils install pysqlite3 + sqlite-utils install sqlean.py -``pysqlite3`` does not provide an implementation of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package: +``pysqlite3`` and ``sqlean.py`` do not provide implementations of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package: .. code-block:: bash @@ -87,4 +87,4 @@ For ``zsh``: Add this code to ``~/.zshrc`` or ``~/.bashrc`` to automatically run it when you start a new shell. -See `the Click documentation `__ for more details. +See `the Click documentation `__ for more details. \ No newline at end of file diff --git a/docs/migrations.rst b/docs/migrations.rst index 23aa9d8..e685936 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -27,7 +27,7 @@ Here is a simple example of a ``migrations.py`` file which creates a table, then .. code-block:: python - from sqlite_utils import Migrations + from sqlite_utils import Database, Migrations migrations = Migrations("creatures") @@ -51,8 +51,6 @@ Once you have a ``Migrations(name)`` collection with one or more migrations regi .. code-block:: python - from sqlite_utils import Database - db = Database("creatures.db") migrations.apply(db) @@ -159,7 +157,7 @@ You can also target a specific migration set using ``migration_set:migration_nam The ``--stop-before`` option can be passed more than once. -If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. Naming a migration that has already been applied is also an error - stopping before it is impossible to honor - and no pending migrations are applied. +If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. Verbose output ============== diff --git a/docs/python-api.rst b/docs/python-api.rst index 43b734d..3f7e12c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -109,14 +109,6 @@ You can also create a named in-memory database. Unlike regular memory databases db = Database(memory_name="my_shared_database") -After creating a ``Database`` you can use ``db.memory`` and ``db.memory_name`` to tell whether it is backed by an in-memory database and to read the shared cache name. ``db.memory`` is ``True`` for any in-memory database and ``db.memory_name`` holds the name passed to ``memory_name=``, or ``None`` otherwise. - -.. code-block:: python - - db = Database(memory_name="shared") - db.memory # True - db.memory_name # "shared" - Connections use ``PRAGMA recursive_triggers=on`` by default. If you don't want to use `recursive triggers `__ you can turn them off using: .. code-block:: python @@ -184,9 +176,6 @@ You can attach an additional database using the ``.attach()`` method, providing You can reference tables in the attached database using the alias value you passed to ``db.attach(alias, filepath)`` as a prefix, for example the ``second.table_in_second`` reference in the SQL query above. -.. note:: - In the CLI: :ref:`sqlite-utils --attach ` - .. _python_api_tracing: Tracing queries @@ -244,22 +233,6 @@ The SQL query is executed as soon as ``db.query()`` is called. The resulting row ``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. The rejected statement is rolled back, so it has no effect on the database. Use :ref:`db.execute() ` for those statements instead. -There is one exception to the rolled-back guarantee: a ``PRAGMA`` statement that returns no rows, such as ``PRAGMA user_version = 5``, still raises a ``ValueError`` but will already have taken effect. Some PRAGMA statements refuse to run inside a transaction, so PRAGMAs are executed outside the savepoint that is used to roll back other rejected statements. Use ``db.execute()`` for PRAGMA statements that do not return rows. - -If a query returns more than one column with the same name - a join between two tables that share column names, for example - later occurrences are renamed with a numeric suffix, so every value is included in the dictionary: - -.. code-block:: python - - row = next(db.query("select 1 as id, 2 as id, 3 as id")) - print(row) - # Outputs: - # {'id': 1, 'id_2': 2, 'id_3': 3} - -A suffix that would collide with another column in the query is skipped - ``select 1 as id, 2 as id, 3 as id_2`` returns ``{'id': 1, 'id_3': 2, 'id_2': 3}``. The same renaming is applied by ``table.rows_where()`` and ``table.search()``. - -.. note:: - In the CLI: :ref:`sqlite-utils query ` - .. _python_api_execute: db.execute(sql, params) @@ -326,15 +299,11 @@ Every method in this library that writes to the database - ``insert()``, ``upser The same applies to raw SQL executed with :ref:`db.execute() ` - a write statement is committed as soon as it has run. -Another way to think about this is that each sqlite-utils method call is its own unit of work. If several method calls must either all succeed or all fail, use ``db.atomic()`` to turn them into a single unit of work. - You never need to call ``commit()``, and you do not need to close the database to persist your changes. There are exactly two situations where you need to think about transactions: 1. You want to group several write operations together, so they either all succeed or all fail - use :ref:`db.atomic() `. 2. You are :ref:`managing a transaction yourself ` with ``db.begin()``, in which case nothing is committed until you commit - the library will never commit a transaction you opened. -``with Database(...) as db:`` is not a transaction block. It manages the lifetime of the database connection and closes it on exit. Use ``with db.atomic():`` for a transaction. - .. _python_api_atomic: Grouping changes with db.atomic() @@ -350,27 +319,6 @@ Use ``db.atomic()`` to group multiple operations in a single transaction: The transaction commits when the block exits. If an exception is raised, changes made inside the block will be rolled back. -This matters when several operations represent a single logical change. Without ``db.atomic()``, an earlier method call remains committed if a later one fails: - -.. code-block:: python - - # These are two separate transactions - db.table("accounts").update(1, {"balance": 90}) - db.table("accounts").update(2, {"balance": 110}) - - # These updates either both succeed or both fail - with db.atomic(): - db.table("accounts").update(1, {"balance": 90}) - db.table("accounts").update(2, {"balance": 110}) - -Transactions can also improve performance. Calling ``insert()`` repeatedly outside ``db.atomic()`` creates and commits a separate transaction for every call. For bulk inserts, prefer :ref:`insert_all() `. If you need to call several different methods in a loop, wrap the loop in ``db.atomic()``: - -.. code-block:: python - - with db.atomic(): - for row in rows: - db.table("events").insert(row) - ``db.atomic()`` can be nested. Nested blocks use SQLite savepoints, so an exception in an inner block can roll back to that savepoint without rolling back the entire outer transaction: .. code-block:: python @@ -399,8 +347,6 @@ Write statements executed with :ref:`db.execute() ` follow t db.execute("insert into news (headline) values (?)", ["Dog wins award"]) # Already committed -``db.execute()`` participates in sqlite-utils transaction handling. Calling ``db.conn.execute()`` directly bypasses that policy and leaves transaction handling to Python's underlying ``sqlite3.Connection``. Prefer ``db.execute()`` unless you deliberately need the lower-level API. - If a transaction is open - because the call happens inside a ``db.atomic()`` block, or after ``db.begin()`` - the statement becomes part of that transaction instead, and commits when the transaction commits: .. code-block:: python @@ -432,12 +378,9 @@ You can take full manual control using the ``db.begin()``, ``db.commit()`` and ` The library will never commit a transaction you opened. If you call write methods such as ``insert()`` - or use ``db.atomic()`` - while your transaction is open, they participate in it using SQLite savepoints instead of committing: exiting an ``atomic()`` block releases its savepoint, but nothing is saved to disk until you commit the outer transaction yourself. If you roll back, their changes are rolled back too. -Prefer ``db.atomic()`` or ``db.begin()``, ``db.commit()`` and ``db.rollback()`` over mixing sqlite-utils transaction methods with calls to ``db.conn.commit()``, ``db.conn.rollback()`` or raw transaction-control SQL. Mixing the two layers makes it much harder to tell which layer owns the current transaction. - -Some related safeguards to be aware of: +Two related safeguards to be aware of: - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. -- ``table.transform()`` raises a ``sqlite_utils.db.TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions, because the pragma cannot be turned off mid-transaction to protect those referencing rows - see :ref:`python_api_transform_foreign_keys_transactions`. - Closing the database - explicitly with ``db.close()``, or by exiting a ``with Database(...) as db:`` block - rolls back any transaction that is still open, see :ref:`python_api_close`. .. _python_api_transactions_modes: @@ -445,11 +388,9 @@ Some related safeguards to be aware of: Supported connection modes -------------------------- -``db.atomic()`` and the automatic per-method transactions currently require a connection using Python's legacy transaction control mode (``sqlite3.LEGACY_TRANSACTION_CONTROL`` on Python 3.12 and later). Passing a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options to ``Database()`` raises a ``sqlite_utils.db.TransactionError``. +``db.atomic()`` and the automatic per-method transactions require a connection in Python's default transaction handling mode. Passing a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options to ``Database()`` raises a ``sqlite_utils.db.TransactionError``. -Connections using ``autocommit=False`` are not supported because Python keeps a transaction open continuously. sqlite-utils uses ``Connection.in_transaction`` to distinguish its own transactions from transactions opened by its caller, and that distinction is not available in this mode. - -Connections using ``autocommit=True`` are also currently rejected because sqlite-utils has not formally exposed that as a supported configuration. +This is because ``commit()`` and ``rollback()`` behave differently on those connections - under ``autocommit=True`` they are documented no-ops - which would cause every write made by this library to be silently discarded when the connection closed, rather than failing loudly. .. _python_api_table: @@ -504,9 +445,6 @@ You can also iterate through the table objects themselves using the ``.tables`` >>> db.tables [] -.. note:: - In the CLI: :ref:`sqlite-utils tables ` - .. _python_api_views: Listing views @@ -532,9 +470,6 @@ View objects are similar to Table objects, except that any attempts to insert or * ``rows_where(where, where_args, order_by, select)`` * ``drop()`` -.. note:: - In the CLI: :ref:`sqlite-utils views ` - .. _python_api_rows: Listing rows @@ -590,9 +525,6 @@ This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} -.. note:: - In the CLI: :ref:`sqlite-utils rows ` - .. _python_api_rows_count_where: Counting rows @@ -677,9 +609,6 @@ The ``db.schema`` property returns the full SQL schema for the database as a str "name" TEXT ); -.. note:: - In the CLI: :ref:`sqlite-utils schema ` - .. _python_api_creating_tables: Creating tables @@ -828,9 +757,6 @@ You can pass ``strict=True`` to create a table in ``STRICT`` mode: "name": str, }, strict=True) -.. note:: - In the CLI: :ref:`sqlite-utils create-table ` - .. _python_api_compound_primary_keys: Compound primary keys @@ -891,61 +817,6 @@ You can leave off the third item in the tuple to have the referenced column auto ("author_id", "authors") ]) -.. _python_api_compound_foreign_keys: - -Compound foreign keys -~~~~~~~~~~~~~~~~~~~~~ - -To create a compound (multi-column) foreign key, use tuples of column names in place of the single column names: - -.. code-block:: python - - db.table("courses").create({ - "course_code": str, - "campus_name": str, - "dept_code": str, - }, pk="course_code", foreign_keys=[ - (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) - ]) - -This creates a table-level constraint: - -.. code-block:: sql - - CREATE TABLE "courses" ( - "course_code" TEXT PRIMARY KEY, - "campus_name" TEXT, - "dept_code" TEXT, - FOREIGN KEY ("campus_name", "dept_code") REFERENCES "departments"("campus_name", "dept_code") - ) - -As with single columns, you can leave off the tuple of other columns to reference the compound primary key of the other table: - -.. code-block:: python - - foreign_keys=[ - (("campus_name", "dept_code"), "departments") - ] - -To specify ``ON DELETE`` or ``ON UPDATE`` actions, pass ``ForeignKey`` objects instead: - -.. code-block:: python - - from sqlite_utils.db import ForeignKey - - db.table("books").create({ - "id": int, - "author_id": int, - }, pk="id", foreign_keys=[ - ForeignKey( - table="books", column="author_id", - other_table="authors", other_column="id", - on_delete="CASCADE", - ) - ]) - -Foreign key actions are preserved by :ref:`table.transform() ` - prior to sqlite-utils 4.0 they were silently dropped when a table was transformed. - .. _python_api_table_configuration: Table configuration options @@ -1011,9 +882,6 @@ Here's an example that uses these features: # ) -.. note:: - In the CLI: :ref:`sqlite-utils insert --not-null and --default ` - .. _python_api_rename_table: Renaming a table @@ -1031,9 +899,6 @@ This executes the following SQL: ALTER TABLE [my_table] RENAME TO [new_name_for_my_table] -.. note:: - In the CLI: :ref:`sqlite-utils rename-table ` - .. _python_api_duplicate: Duplicating tables @@ -1049,9 +914,6 @@ The new ``authors_copy`` table will now contain a duplicate copy of the data fro This method raises ``sqlite_utils.db.NoTable`` if the table does not exist. -.. note:: - In the CLI: :ref:`sqlite-utils duplicate ` - .. _python_api_bulk_inserts: Bulk inserts @@ -1094,9 +956,6 @@ You can delete all the existing rows in the table before inserting the new recor Pass ``analyze=True`` to run ``ANALYZE`` against the table after inserting the new records. -.. note:: - In the CLI: :ref:`sqlite-utils insert ` - .. _python_api_insert_lists: Inserting data from a list or tuple iterator @@ -1174,9 +1033,6 @@ To replace any existing records that have a matching primary key, use the ``repl .. note:: Prior to sqlite-utils 2.0 the ``.upsert()`` and ``.upsert_all()`` methods worked the same way as ``.insert(replace=True)`` does today. See :ref:`python_api_upsert` for the new behaviour of those methods introduced in 2.0. -.. note:: - In the CLI: :ref:`sqlite-utils insert --replace ` - .. _python_api_update: Updating a specific record @@ -1262,9 +1118,6 @@ Every record passed to ``upsert()`` or ``upsert_all()`` must include a value for .. note:: ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 `__ for details of this change. -.. note:: - In the CLI: :ref:`sqlite-utils upsert ` - .. _python_api_old_upsert: Alternative upserts using INSERT OR IGNORE @@ -1360,8 +1213,6 @@ To create a species record with a note on when it was first seen, you can use th The first time this is called the record will be created for ``name="Palm"``. Any subsequent calls with that name will ignore the second argument, even if it includes different values. -``None`` values are matched correctly: calling ``.lookup()`` a second time with the same values will return the primary key of the existing row even if some of those values are ``None``. - ``.lookup()`` also accepts keyword arguments, which are passed through to the :ref:`insert() method ` and can be used to influence the shape of the created table. Supported parameters are: - ``pk`` - which defaults to ``id`` @@ -1407,8 +1258,6 @@ To extract the ``species`` column out to a separate ``Species`` table, you can d "species": "Common Juniper" }, extracts={"species": "Species"}) -``None`` values are not extracted: no record is created for them in the lookup table and the column value stays ``null``. - .. _python_api_m2m: Working with many-to-many relationships @@ -1616,9 +1465,6 @@ You can set a ``NOT NULL DEFAULT 'x'`` constraint on the new column using ``not_ db.table("dogs").add_column("friends_count", int, not_null_default=0) -.. note:: - In the CLI: :ref:`sqlite-utils add-column ` - .. _python_api_add_column_alter: Adding columns automatically on insert/update @@ -1642,9 +1488,6 @@ You can insert or update data that includes new columns and have the table autom new_table = db.table("new_table", alter=True) new_table.insert({"name": "Gareth", "age": 32, "shoe_size": 11}) -.. note:: - In the CLI: :ref:`sqlite-utils insert --alter ` - .. _python_api_add_foreign_key: Adding foreign key constraints @@ -1683,29 +1526,6 @@ To ignore the case where the key already exists, use ``ignore=True``: db.table("books").add_foreign_key("author_id", "authors", "id", ignore=True) -To add a compound foreign key, pass tuples of columns: - -.. code-block:: python - - db.table("courses").add_foreign_key( - ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") - ) - -As with single columns, omitting the other columns will use the compound primary key of the other table. ``other_table`` must always be specified for a compound foreign key. - -Use ``on_delete=`` and ``on_update=`` to specify ``ON DELETE`` and ``ON UPDATE`` actions for the foreign key: - -.. code-block:: python - - db.table("books").add_foreign_key( - "author_id", "authors", "id", on_delete="CASCADE" - ) - -This creates a foreign key with an ``ON DELETE CASCADE`` clause, so deleting an author will also delete their books (provided foreign key enforcement is enabled with ``PRAGMA foreign_keys = ON``). Valid actions are ``"SET NULL"``, ``"SET DEFAULT"``, ``"CASCADE"``, ``"RESTRICT"`` and the default ``"NO ACTION"``. - -.. note:: - In the CLI: :ref:`sqlite-utils add-foreign-key ` - .. _python_api_add_foreign_keys: Adding multiple foreign key constraints at once @@ -1724,11 +1544,6 @@ Here's an example adding two foreign keys at once: This method runs the same checks as ``.add_foreign_keys()`` and will raise ``sqlite_utils.db.AlterError`` if those checks fail. -Foreign keys that already exist are silently skipped, so repeated calls are idempotent - but only if they match exactly. Requesting a foreign key that exists with different ``ON DELETE``/``ON UPDATE`` actions raises ``AlterError``: use ``table.transform()`` to change the actions of an existing foreign key. - -.. note:: - In the CLI: :ref:`sqlite-utils add-foreign-keys ` - .. _python_api_index_foreign_keys: Adding indexes for all foreign keys @@ -1740,11 +1555,6 @@ If you want to ensure that every foreign key column in your database has a corre db.index_foreign_keys() -Compound foreign keys get a single composite index across their columns. - -.. note:: - In the CLI: :ref:`sqlite-utils index-foreign-keys ` - .. _python_api_drop: Dropping a table or view @@ -1766,9 +1576,6 @@ Pass ``ignore=True`` if you want to ignore the error caused by the table or view db.table("my_table").drop(ignore=True) -.. note:: - In the CLI: :ref:`sqlite-utils drop-table ` and :ref:`sqlite-utils drop-view ` - .. _python_api_transform: Transforming a table @@ -1797,9 +1604,6 @@ To keep the original table around instead of dropping it, pass the ``keep_table= This method raises a ``sqlite_utils.db.TransformError`` exception if the table cannot be transformed, usually because there are existing constraints or indexes that are incompatible with modifications to the columns. -.. note:: - In the CLI: :ref:`sqlite-utils transform ` - .. _python_api_transform_alter_column_types: Altering column types @@ -1814,29 +1618,6 @@ To alter the type of a column, use the ``types=`` argument: See :ref:`python_api_add_column` for a list of available types. -.. _python_api_transform_strict: - -Changing strict mode --------------------- - -The optional ``strict=`` parameter can change whether a table uses `SQLite STRICT mode `__. Pass ``strict=True`` to convert a regular table to a strict table: - -.. code-block:: python - - table.transform(strict=True) - -Pass ``strict=False`` to convert a strict table back to a regular non-strict table: - -.. code-block:: python - - table.transform(strict=False) - -The default is ``strict=None``, which preserves the table's existing strict mode. - -Passing ``strict=True`` raises ``sqlite_utils.db.TransformError`` if the available SQLite version does not support strict tables. - -Converting to a strict table validates all existing rows as they are copied into the replacement table. If a value is incompatible with its declared column type, SQLite raises ``sqlite3.IntegrityError`` and the transformation is rolled back, leaving the original table and its data unchanged. - .. _python_api_transform_rename_columns: Renaming columns @@ -1976,16 +1757,6 @@ This example drops two foreign keys - the one from ``places.country`` to ``count drop_foreign_keys=("country", "continent") ) -A bare column name drops any foreign key that column participates in, including compound foreign keys. To target a compound foreign key precisely, pass a tuple of its columns: - -.. code-block:: python - - db.table("courses").transform( - drop_foreign_keys=[("campus_name", "dept_code")] - ) - -Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint. - .. _python_api_transform_sql: Custom transformations with .transform_sql() @@ -1997,36 +1768,6 @@ If you want to do something more advanced, you can call the ``table.transform_sq This method will return a list of SQL statements that should be executed to implement the change. You can then make modifications to that SQL - or add additional SQL statements - before executing it yourself. -.. _python_api_transform_foreign_keys_transactions: - -Foreign keys and transactions ------------------------------ - -Because ``.transform()`` drops the old table, running it with ``PRAGMA foreign_keys`` enabled could fire ``ON DELETE`` actions on any tables that reference it - an inbound ``ON DELETE CASCADE`` foreign key would silently delete those referencing rows. To prevent this, ``.transform()`` turns ``PRAGMA foreign_keys`` off for the duration of the operation and restores it afterwards, running ``PRAGMA foreign_key_check`` before committing. - -``PRAGMA foreign_keys`` cannot be changed inside a transaction, so this protection is impossible if you call ``.transform()`` while a transaction is already open - for example inside a ``with db.atomic():`` block or after ``db.begin()``. If ``PRAGMA foreign_keys`` is on and another table references the table being transformed with a destructive ``ON DELETE`` action - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT`` - the method will refuse to run and raise a ``sqlite_utils.db.TransactionError``: - -.. code-block:: python - - from sqlite_utils.db import TransactionError - - try: - with db.atomic(): - db["authors"].transform(types={"id": str}) - except TransactionError as ex: - print("Could not transform in transaction:", ex) - -To transform such a table either call ``.transform()`` outside of the transaction, or execute ``PRAGMA foreign_keys = off`` before opening it: - -.. code-block:: python - - db.execute("PRAGMA foreign_keys = off") - with db.atomic(): - db["authors"].transform(types={"id": str}) - db.execute("PRAGMA foreign_keys = on") - -Tables referenced by foreign keys without a destructive action (the default ``NO ACTION``, or ``RESTRICT``) can still be transformed inside a transaction - sqlite-utils uses ``PRAGMA defer_foreign_keys`` to postpone the foreign key checks until the transaction commits. - .. _python_api_extract: Extracting columns into a separate table @@ -2183,11 +1924,6 @@ This produces a lookup table like so: "latin" TEXT ) -Rows where every extracted column is ``null`` are not extracted: no record is created for them in the lookup table and their foreign key column is left as ``null``. When extracting multiple columns, rows where at least one of the extracted columns has a value will be extracted as usual. - -.. note:: - In the CLI: :ref:`sqlite-utils extract ` - .. _python_api_hash: Setting an ID based on the hash of the row contents @@ -2249,9 +1985,6 @@ You can pass ``ignore=True`` to silently ignore an existing view and do nothing, select * from dogs where is_good_dog = 1 """, replace=True) -.. note:: - In the CLI: :ref:`sqlite-utils create-view ` - Storing JSON ============ @@ -2364,15 +2097,12 @@ The ``db.iterdump()`` method returns a sequence of SQL strings representing a co This uses the `sqlite3.Connection.iterdump() `__ method. -If you are using ``pysqlite3`` the underlying method may be missing. If you install the `sqlite-dump `__ package then the ``db.iterdump()`` method will use that implementation instead: +If you are using ``pysqlite3`` or ``sqlean.py`` the underlying method may be missing. If you install the `sqlite-dump `__ package then the ``db.iterdump()`` method will use that implementation instead: .. code-block:: bash pip install sqlite-dump -.. note:: - In the CLI: :ref:`sqlite-utils dump ` - .. _python_api_introspection: Introspecting tables and views @@ -2474,39 +2204,17 @@ Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly .foreign_keys ------------- -The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey`` objects. It is not available on views. - -Each ``ForeignKey`` has the following attributes: - -``table`` - The table the foreign key is defined on. -``column`` - The column on this table, or ``None`` for a compound foreign key. -``other_table`` - The table being referenced. -``other_column`` - The referenced column, or ``None`` for a compound foreign key. -``columns`` - A tuple of the columns on this table, always populated (a one-item tuple for single-column foreign keys). -``other_columns`` - A tuple of the referenced columns. -``is_compound`` - ``True`` if this is a compound (multi-column) foreign key. -``on_delete`` - The ``ON DELETE`` action, e.g. ``"CASCADE"`` - ``"NO ACTION"`` if not set. -``on_update`` - The ``ON UPDATE`` action - ``"NO ACTION"`` if not set. - -``ForeignKey`` was a ``namedtuple`` prior to sqlite-utils 4.0. It is now a dataclass and can no longer be unpacked or indexed as a tuple - access its fields by name instead. See :ref:`upgrading_3_to_4` for details. +The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views. :: >>> db.table("Street_Tree_List").foreign_keys - [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=('qLegalStatus',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), - ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=('qCareAssistant',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), - ...] - -Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` tuples. + [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id'), + ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id'), + ForeignKey(table='Street_Tree_List', column='qSiteInfo', other_table='qSiteInfo', other_column='id'), + ForeignKey(table='Street_Tree_List', column='qSpecies', other_table='qSpecies', other_column='id'), + ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'), + ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')] .. _python_api_introspection_schema: @@ -2572,9 +2280,6 @@ The ``.indexes`` property returns all indexes created for a table, as a list of Index(seq=4, name='"Street_Tree_List_qCaretaker"', unique=0, origin='c', partial=0, columns=['qCaretaker']), Index(seq=5, name='"Street_Tree_List_PlantType"', unique=0, origin='c', partial=0, columns=['PlantType'])] -.. note:: - In the CLI: :ref:`sqlite-utils indexes ` - .. _python_api_introspection_xindexes: .xindexes @@ -2618,9 +2323,6 @@ The ``.triggers`` property lists database triggers. It can be used on both datab >>> db.triggers ... similar output to db.table("authors").triggers -.. note:: - In the CLI: :ref:`sqlite-utils triggers ` - .. _python_api_introspection_triggers_dict: .triggers_dict @@ -2769,9 +2471,6 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab db.table("dogs").disable_fts() -.. note:: - In the CLI: :ref:`sqlite-utils enable-fts ` - .. _python_api_quote_fts: Quoting characters for use in search @@ -2836,9 +2535,6 @@ To return just the title and published columns for three matches for ``"dog"`` w ): print(article) -.. note:: - In the CLI: :ref:`sqlite-utils search ` - .. _python_api_fts_search_sql: Building SQL queries with table.search_sql() @@ -2923,9 +2619,6 @@ This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("rebuild"); -.. note:: - In the CLI: :ref:`sqlite-utils rebuild-fts ` - .. _python_api_fts_optimize: Optimizing a full-text search table @@ -2941,9 +2634,6 @@ This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("optimize"); -.. note:: - In the CLI: :ref:`sqlite-utils optimize ` - .. _python_api_cached_table_counts: Cached table counts using triggers @@ -3006,9 +2696,6 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y db.reset_counts() -.. note:: - In the CLI: :ref:`sqlite-utils enable-counts ` - .. _python_api_create_index: Creating indexes @@ -3052,17 +2739,6 @@ Use ``if_not_exists=True`` to do nothing if an index with that name already exis Pass ``analyze=True`` to run ``ANALYZE`` against the new index after creating it. -You can drop an index from a table using ``.drop_index(index_name)``: - -.. code-block:: python - - db.table("dogs").drop_index("idx_dogs_name") - -Use ``ignore=True`` to ignore the error if the index does not exist. - -.. note:: - In the CLI: :ref:`sqlite-utils create-index ` and :ref:`sqlite-utils drop-index ` - .. _python_api_analyze: Optimizing index usage with ANALYZE @@ -3090,9 +2766,6 @@ To run against all indexes attached to a specific table, you can either pass the db.table("dogs").analyze() -.. note:: - In the CLI: :ref:`sqlite-utils analyze ` - .. _python_api_vacuum: Vacuum @@ -3104,9 +2777,6 @@ You can optimize your database by running VACUUM against it like so: Database("my_database.db").vacuum() -.. note:: - In the CLI: :ref:`sqlite-utils vacuum ` - .. _python_api_wal: WAL mode @@ -3134,9 +2804,6 @@ You can check the current journal mode for a database using the ``journal_mode`` This will usually be ``wal`` or ``delete`` (meaning WAL is disabled), but can have other values - see the `PRAGMA journal_mode `__ documentation. -.. note:: - In the CLI: :ref:`sqlite-utils enable-wal and disable-wal ` - .. _python_api_suggest_column_types: Suggesting column types @@ -3277,9 +2944,6 @@ You can cause ``sqlite3`` to return more useful errors, including the traceback sqlite3.enable_callback_tracebacks(True) -.. note:: - In the CLI: :ref:`sqlite-utils query --functions ` - .. _python_api_quote: Quoting strings for use in SQL @@ -3412,9 +3076,6 @@ Initialize SpatiaLite .. automethod:: sqlite_utils.db.Database.init_spatialite :noindex: -.. note:: - In the CLI: :ref:`sqlite-utils create-database --init-spatialite ` - .. _python_api_gis_find_spatialite: Finding SpatiaLite @@ -3430,9 +3091,6 @@ Adding geometry columns .. automethod:: sqlite_utils.db.Table.add_geometry_column :noindex: -.. note:: - In the CLI: :ref:`sqlite-utils add-geometry-column ` - .. _python_api_gis_create_spatial_index: Creating a spatial index @@ -3440,6 +3098,3 @@ Creating a spatial index .. automethod:: sqlite_utils.db.Table.create_spatial_index :noindex: - -.. note:: - In the CLI: :ref:`sqlite-utils create-spatial-index ` diff --git a/docs/reference.rst b/docs/reference.rst index a9fdf29..5b5fd25 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -70,13 +70,6 @@ sqlite_utils.db.ColumnDetails .. autoclass:: sqlite_utils.db.ColumnDetails -.. _reference_db_other_foreign_key: - -sqlite_utils.db.ForeignKey --------------------------- - -.. autoclass:: sqlite_utils.db.ForeignKey - sqlite_utils.utils ================== diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 92b582a..1974ad6 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -46,18 +46,6 @@ Two related things have been removed: Python API changes ------------------ -**db.query() now rejects SQL that does not return rows.** This is likely the most common change you will need to make to existing code. ``db.query()`` used to accept any SQL statement - passing one that returns no rows, such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause or a ``CREATE TABLE``, did nothing at all, silently. Those statements now raise a ``ValueError``, and are rolled back so they have no effect on the database. Transaction control statements (``BEGIN``, ``COMMIT``, ``END``, ``ROLLBACK``, ``SAVEPOINT``, ``RELEASE``) plus ``VACUUM``, ``ATTACH`` and ``DETACH`` are also rejected with a ``ValueError``, without being executed at all. Use ``db.execute()`` for statements that do not return rows: - -.. code-block:: python - - # 3.x accepted this but silently did nothing: - db.query("update dogs set name = 'Cleopaws'") - - # In 4.0 use execute() for SQL that does not return rows: - db.execute("update dogs set name = 'Cleopaws'") - -**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily, but errors in your SQL raise at the ``db.query()`` call site rather than on first iteration, and a write with a ``RETURNING`` clause takes effect even if you never iterate over its results. - **db.table() no longer returns views.** ``db.table(name)`` now raises a ``sqlite_utils.db.NoTable`` exception if ``name`` is a SQL view. Use the new ``db.view(name)`` method for views: .. code-block:: python @@ -67,6 +55,11 @@ Python API changes ``db["name"]`` still returns either a ``Table`` or a ``View`` depending on what exists in the database. +**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily. Two consequences: + +- Errors in your SQL now raise at the ``db.query()`` call site rather than on first iteration. +- Passing a statement that returns no rows - such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause - previously did nothing at all, silently. It now raises a ``ValueError``, and the statement is rolled back so it has no effect on the database. Use ``db.execute()`` for statements that do not return rows. + **Upserts use INSERT ... ON CONFLICT.** Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax rather than the previous ``INSERT OR IGNORE`` followed by ``UPDATE``. If your code depends on the old behavior, pass ``use_old_upsert=True`` to the ``Database()`` constructor - see :ref:`python_api_old_upsert`. **Upsert records must include their primary keys.** ``table.upsert()`` and ``table.upsert_all()`` now raise ``sqlite_utils.db.PrimaryKeyRequired`` if a record is missing a value for any primary key column (or has ``None`` for one). Previously such records were quietly inserted as new rows. Relatedly, ``pk=`` is now optional when the table already exists with a primary key - it is detected automatically. @@ -77,30 +70,8 @@ Python API changes **table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values. -**Null values are no longer extracted into lookup tables.** ``table.extract()`` and the ``sqlite-utils extract`` command leave rows alone if every extracted column is ``null`` - the new foreign key column is left as ``null`` instead of pointing at an all-``null`` record in the lookup table. The ``extracts=`` insert option similarly keeps ``None`` values as ``null``. Relatedly, ``table.lookup()`` now compares values using ``IS`` so that looking up a value containing ``None`` returns the existing matching row - previously it inserted a duplicate row on every call. - -**ensure_autocommit_off() is now ensure_autocommit_on().** The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``. The old name described the opposite of what the method did: it temporarily puts the connection into driver-level autocommit mode (by setting ``isolation_level = None``), so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. The behavior is unchanged - update any calls to use the new name. - **View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. -**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained new fields - ``columns``, ``other_columns``, ``is_compound``, ``on_delete`` and ``on_update`` - so that compound (multi-column) foreign keys and foreign key actions can be represented. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: - -.. code-block:: python - - # 3.x - tuple unpacking, no longer works: - for table, column, other_table, other_column in db["courses"].foreign_keys: - ... - - # 4.0 - access fields by name: - for fk in db["courses"].foreign_keys: - fk.table, fk.column, fk.other_table, fk.other_column - -Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. Like the old namedtuple, ``ForeignKey`` instances are immutable and hashable - they can be collected into sets and used as dictionary keys. Note that equality now includes the ``on_delete`` and ``on_update`` actions: a ``ForeignKey`` with ``ON DELETE CASCADE`` is not equal to one without. - -Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` tuples instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are one-item tuples. - -Two related behavior changes to ``table.transform()``: compound foreign keys now survive a transform (previously they were split into separate single-column keys), and ``ON DELETE``/``ON UPDATE`` actions such as ``ON DELETE CASCADE`` are now preserved (previously they were silently stripped from the schema). - **Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``. **Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: diff --git a/mypy.ini b/mypy.ini index 2f6a875..de0dc83 100644 --- a/mypy.ini +++ b/mypy.ini @@ -16,6 +16,9 @@ ignore_errors = True [mypy-pysqlite3.*] ignore_missing_imports = True +[mypy-sqlean.*] +ignore_missing_imports = True + [mypy-sqlite_dump.*] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 003322c..4ad1bd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.1.1" +version = "4.0rc1" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -53,6 +53,7 @@ dev = [ "tabulate>=0.10.0", ] docs = [ + "beanbag-docutils>=2.0", "codespell", "furo", "pygments-csv-lexer", diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e0b8969..a30f79a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -13,10 +13,8 @@ from sqlite_utils.db import ( BadMultiValues, DEFAULT, DescIndex, - InvalidColumns, NoTable, NoView, - PrimaryKeyRequired, quote_identifier, ) from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins @@ -36,7 +34,6 @@ from .utils import ( OperationalError, _compile_code, chunks, - dedupe_keys, file_progress, find_spatialite, flatten as _flatten, @@ -113,11 +110,7 @@ def output_options(fn): ), click.option("--csv", is_flag=True, help="Output CSV"), click.option("--tsv", is_flag=True, help="Output TSV"), - click.option( - "--no-headers", - is_flag=True, - help="Omit headers from CSV/TSV and table/--fmt output", - ), + click.option("--no-headers", is_flag=True, help="Omit CSV headers"), click.option( "-t", "--table", is_flag=True, help="Output as a formatted table" ), @@ -133,13 +126,6 @@ def output_options(fn): is_flag=True, default=False, ), - click.option( - "--ascii", - "ascii_", - help="Escape non-ASCII characters in JSON output as \\uXXXX", - is_flag=True, - default=False, - ), ) ): fn = decorator(fn) @@ -154,17 +140,6 @@ def load_extension_option(fn): )(fn) -def functions_option(fn): - return click.option( - "--functions", - help=( - "Python code or a file path defining custom SQL functions; " - "can be used multiple times" - ), - multiple=True, - )(fn) - - @click.group( cls=DefaultGroup, default="query", @@ -219,7 +194,6 @@ def tables( table, fmt, json_cols, - ascii_, columns, schema, load_extension, @@ -265,13 +239,7 @@ def tables( yield row if table or fmt: - print( - tabulate.tabulate( - _iter(), - headers=() if no_headers else headers, - tablefmt=fmt or "simple", - ) - ) + print(tabulate.tabulate(_iter(), headers=headers, tablefmt=fmt or "simple")) elif csv or tsv: writer = csv_std.writer(sys.stdout, dialect="excel-tab" if tsv else "excel") if not no_headers: @@ -279,7 +247,7 @@ def tables( for row in _iter(): writer.writerow(row) else: - for line in output_rows(_iter(), headers, nl, arrays, json_cols, ascii_): + for line in output_rows(_iter(), headers, nl, arrays, json_cols): click.echo(line) @@ -317,7 +285,6 @@ def views( table, fmt, json_cols, - ascii_, columns, schema, load_extension, @@ -343,7 +310,6 @@ def views( table=table, fmt=fmt, json_cols=json_cols, - ascii_=ascii_, columns=columns, schema=schema, load_extension=load_extension, @@ -692,34 +658,6 @@ def create_index( ) -@cli.command(name="drop-index") -@click.argument( - "path", - type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), - required=True, -) -@click.argument("table") -@click.argument("index") -@click.option("--ignore", help="Ignore if index does not exist", is_flag=True) -@load_extension_option -def drop_index(path, table, index, ignore, load_extension): - """ - Drop an index by index name from the specified table - - Example: - - \b - sqlite-utils drop-index chickens.db chickens idx_chickens_name - """ - db = sqlite_utils.Database(path) - _register_db_for_cleanup(db) - _load_extensions(db, load_extension) - try: - db.table(table).drop_index(index, ignore=ignore) - except OperationalError as ex: - raise click.ClickException(str(ex)) - - @cli.command(name="enable-fts") @click.argument( "path", @@ -746,7 +684,7 @@ def drop_index(path, table, index, ignore, load_extension): def enable_fts( path, table, column, fts4, fts5, tokenize, create_triggers, replace, load_extension ): - """Enable full-text search for specific table and columns + """Enable full-text search for specific table and columns" Example: @@ -972,19 +910,13 @@ def insert_upsert_options(*, require_pk=False): required=True, ), click.argument("table"), - click.argument( - "file", type=click.File("rb", lazy=True), required=False - ), + click.argument("file", type=click.File("rb", lazy=True), required=True), click.option( "--pk", help="Columns to use as the primary key, e.g. id", multiple=True, required=require_pk, ), - click.option( - "--code", - help="Python code defining a rows() function or iterable of rows to insert", - ), ) + _import_options + ( @@ -1008,16 +940,6 @@ def insert_upsert_options(*, require_pk=False): type=(str, str), help="Default value that should be set for a column", ), - click.option( - "--type", - "types", - type=( - str, - click.Choice(list(VALID_COLUMN_TYPES), case_sensitive=False), - ), - multiple=True, - help="Column types to use when creating the table", - ), click.option( "--no-detect-types", is_flag=True, @@ -1072,7 +994,6 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, - types=None, no_detect_types=False, analyze=False, load_extension=None, @@ -1080,123 +1001,11 @@ def insert_upsert_implementation( bulk_sql=None, functions=None, strict=False, - code=None, ): db = sqlite_utils.Database(path) _register_db_for_cleanup(db) _load_extensions(db, load_extension) _maybe_register_functions(db, functions) - column_type_overrides = {column: ctype.upper() for column, ctype in (types or [])} - - def _insert_docs(docs, tracker=None): - extra_kwargs = { - "ignore": ignore, - "replace": replace, - "truncate": truncate, - "analyze": analyze, - "strict": strict, - } - if not_null: - extra_kwargs["not_null"] = set(not_null) - if default: - extra_kwargs["defaults"] = dict(default) - if column_type_overrides: - extra_kwargs["columns"] = column_type_overrides - if upsert: - extra_kwargs["upsert"] = upsert - - # docs should all be dictionaries - docs = (verify_is_dict(doc) for doc in docs) - - # Apply {"$base64": true, ...} decoding, if needed - docs = (decode_base64_values(doc) for doc in docs) - - # For bulk_sql= we use cursor.executemany() instead - if bulk_sql: - if batch_size: - doc_chunks = chunks(docs, batch_size) - else: - doc_chunks = [docs] - for doc_chunk in doc_chunks: - with db.atomic(): - db.conn.cursor().executemany(bulk_sql, doc_chunk) - return - - # table_names() rather than db.table(), which raises NoTable for - # views before the error handling below can deal with them - table_existed_before_insert = table in db.table_names() - try: - db.table(table).insert_all( - docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs - ) - except (NoTable, InvalidColumns, PrimaryKeyRequired) as e: - raise click.ClickException(str(e)) - except Exception as e: - if ( - isinstance(e, OperationalError) - and e.args - and ( - "has no column named" in e.args[0] or "no such column" in e.args[0] - ) - ): - raise click.ClickException( - "{}\n\nTry using --alter to add additional columns".format( - e.args[0] - ) - ) - # If we can find sql= and parameters= arguments, show those - variables = _find_variables(e.__traceback__, ["sql", "parameters"]) - if "sql" in variables and "parameters" in variables: - raise click.ClickException( - "{}\n\nsql = {}\nparameters = {}".format( - str(e), variables["sql"], variables["parameters"] - ) - ) - else: - raise - # Apply detected types only to a table this command created - - # transforming a pre-existing table would rewrite its column types - # and corrupt values such as TEXT zip codes with leading zeros - if ( - tracker is not None - and not table_existed_before_insert - and db.table(table).exists() - ): - detected_types = tracker.types - detected_types.update(column_type_overrides) - db.table(table).transform(types=detected_types) - - if code is not None: - if file is not None: - raise click.ClickException("--code cannot be used with a FILE argument") - if any( - [ - flatten, - nl, - csv, - tsv, - empty_null, - lines, - text, - convert, - sniff, - no_headers, - delimiter, - quotechar, - encoding, - ] - ): - raise click.ClickException( - "--code cannot be used with input format options" - ) - _insert_docs(_rows_from_code(code)) - return - - if file is None: - raise click.ClickException( - "Provide either a FILE argument or --code to specify rows to insert" - ) - if (delimiter or quotechar or sniff or no_headers) and not tsv: csv = True if (nl + csv + tsv) >= 2: @@ -1304,7 +1113,68 @@ def insert_upsert_implementation( else: docs = (fn(doc) or doc for doc in docs) - _insert_docs(docs, tracker=tracker) + extra_kwargs = { + "ignore": ignore, + "replace": replace, + "truncate": truncate, + "analyze": analyze, + "strict": strict, + } + if not_null: + extra_kwargs["not_null"] = set(not_null) + if default: + extra_kwargs["defaults"] = dict(default) + if upsert: + extra_kwargs["upsert"] = upsert + + # docs should all be dictionaries + docs = (verify_is_dict(doc) for doc in docs) + + # Apply {"$base64": true, ...} decoding, if needed + docs = (decode_base64_values(doc) for doc in docs) + + # For bulk_sql= we use cursor.executemany() instead + if bulk_sql: + if batch_size: + doc_chunks = chunks(docs, batch_size) + else: + doc_chunks = [docs] + for doc_chunk in doc_chunks: + with db.atomic(): + db.conn.cursor().executemany(bulk_sql, doc_chunk) + return + + try: + db.table(table).insert_all( + docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs + ) + except NoTable as e: + raise click.ClickException(str(e)) + except Exception as e: + if ( + isinstance(e, OperationalError) + and e.args + and ( + "has no column named" in e.args[0] or "no such column" in e.args[0] + ) + ): + raise click.ClickException( + "{}\n\nTry using --alter to add additional columns".format( + e.args[0] + ) + ) + # If we can find sql= and parameters= arguments, show those + variables = _find_variables(e.__traceback__, ["sql", "parameters"]) + if "sql" in variables and "parameters" in variables: + raise click.ClickException( + "{}\n\nsql = {}\nparameters = {}".format( + str(e), variables["sql"], variables["parameters"] + ) + ) + else: + raise + if tracker is not None and db.table(table).exists(): + db.table(table).transform(types=tracker.types) # Clean up open file-like objects if sniff_buffer: @@ -1347,7 +1217,6 @@ def insert( table, file, pk, - code, flatten, nl, csv, @@ -1374,7 +1243,6 @@ def insert( truncate, not_null, default, - types, strict, ): """ @@ -1393,9 +1261,6 @@ def insert( - Use --lines to write each incoming line to a column called "line" - Use --text to write the entire input to a column called "text" - Use --type column-name type to override the type automatically chosen - when the table is created. - You can also use --convert to pass a fragment of Python code that will be used to convert each input. @@ -1423,17 +1288,6 @@ def insert( \b echo 'A bunch of words' | sqlite-utils insert words.db words - \\ --text --convert '({"word": w} for w in text.split())' - - Instead of a FILE you can use --code to provide a block of Python code - that defines the rows to insert, as either a rows() function that yields - dictionaries or a "rows" iterable. --code can also be a path to a .py file: - - \b - sqlite-utils insert data.db creatures --code ' - def rows(): - yield {"id": 1, "name": "Cleo"} - yield {"id": 2, "name": "Suna"} - ' --pk id """ try: insert_upsert_implementation( @@ -1468,22 +1322,19 @@ def insert( silent=silent, not_null=not_null, default=default, - types=types, strict=strict, - code=code, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) @cli.command() -@insert_upsert_options() +@insert_upsert_options(require_pk=True) def upsert( path, table, file, pk, - code, flatten, nl, csv, @@ -1503,7 +1354,6 @@ def upsert( alter, not_null, default, - types, no_detect_types, analyze, load_extension, @@ -1515,11 +1365,6 @@ def upsert( an incoming record has a primary key that matches an existing record the existing record will be updated. - If the table already exists and has a primary key, --pk can be omitted. - - Use --type column-name type to override the type automatically chosen - when the table is created. - Example: \b @@ -1554,13 +1399,11 @@ def upsert( upsert=True, not_null=not_null, default=default, - types=types, no_detect_types=no_detect_types, analyze=analyze, load_extension=load_extension, silent=silent, strict=strict, - code=code, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) @@ -1575,7 +1418,11 @@ def upsert( @click.argument("sql") @click.argument("file", type=click.File("rb"), required=True) @click.option("--batch-size", type=int, default=100, help="Commit every X records") -@functions_option +@click.option( + "--functions", + help="Python code or file path defining custom SQL functions", + multiple=True, +) @import_options @load_extension_option def bulk( @@ -1755,10 +1602,10 @@ def create_table( sqlite-utils create-table my.db people \\ id integer \\ name text \\ - height real \\ + height float \\ photo blob --pk id - Valid column types are text, integer, real, float and blob. + Valid column types are text, integer, float and blob. """ db = sqlite_utils.Database(path) _register_db_for_cleanup(db) @@ -1981,7 +1828,11 @@ def drop_view(path, view, ignore, load_extension): type=(str, str), help="Named :parameters for SQL query", ) -@functions_option +@click.option( + "--functions", + help="Python code or file path defining custom SQL functions", + multiple=True, +) @load_extension_option def query( path, @@ -1995,7 +1846,6 @@ def query( table, fmt, json_cols, - ascii_, raw, raw_lines, param, @@ -2010,15 +1860,7 @@ def query( sqlite-utils data.db \\ "select * from chickens where age > :age" \\ -p age 1 - - Pass "-" as the SQL to read the query from standard input: - - \b - echo "select * from chickens" | sqlite-utils data.db - """ - if sql == "-": - # Read SQL from standard input - sql = sys.stdin.read() db = sqlite_utils.Database(path) _register_db_for_cleanup(db) for alias, attach_path in attach: @@ -2042,7 +1884,6 @@ def query( nl, arrays, json_cols, - ascii_, ) @@ -2054,7 +1895,11 @@ def query( nargs=-1, ) @click.argument("sql") -@functions_option +@click.option( + "--functions", + help="Python code or file path defining custom SQL functions", + multiple=True, +) @click.option( "--attach", type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), @@ -2113,7 +1958,6 @@ def memory( table, fmt, json_cols, - ascii_, raw, raw_lines, param, @@ -2253,7 +2097,6 @@ def memory( nl, arrays, json_cols, - ascii_, ) @@ -2271,7 +2114,6 @@ def _execute_query( nl, arrays, json_cols, - ascii_, ): with db.conn: try: @@ -2302,9 +2144,7 @@ def _execute_query( elif fmt or table: print( tabulate.tabulate( - list(cursor), - headers=() if no_headers else headers, - tablefmt=fmt or "simple", + list(cursor), headers=headers, tablefmt=fmt or "simple" ) ) elif csv or tsv: @@ -2314,7 +2154,7 @@ def _execute_query( for row in cursor: writer.writerow(row) else: - for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_): + for line in output_rows(cursor, headers, nl, arrays, json_cols): click.echo(line) @@ -2358,7 +2198,6 @@ def search( table, fmt, json_cols, - ascii_, load_extension, ): """Execute a full-text search against this table @@ -2405,7 +2244,6 @@ def search( table=table, fmt=fmt, json_cols=json_cols, - ascii_=ascii_, param=[("query", q)], load_extension=load_extension, ) @@ -2466,7 +2304,6 @@ def rows( table, fmt, json_cols, - ascii_, load_extension, ): """Output all rows in the specified table @@ -2501,7 +2338,6 @@ def rows( fmt=fmt, param=param, json_cols=json_cols, - ascii_=ascii_, load_extension=load_extension, ) @@ -2528,7 +2364,6 @@ def triggers( table, fmt, json_cols, - ascii_, load_extension, ): """Show triggers configured in this database @@ -2558,7 +2393,6 @@ def triggers( table=table, fmt=fmt, json_cols=json_cols, - ascii_=ascii_, load_extension=load_extension, ) @@ -2587,7 +2421,6 @@ def indexes( table, fmt, json_cols, - ascii_, load_extension, ): """Show indexes for the whole database or specific tables @@ -2629,7 +2462,6 @@ def indexes( table=table, fmt=fmt, json_cols=json_cols, - ascii_=ascii_, load_extension=load_extension, ) @@ -2718,11 +2550,6 @@ def schema( multiple=True, help="Drop foreign key constraint for this column", ) -@click.option( - "--strict/--no-strict", - default=None, - help="Enable or disable STRICT mode (default: preserve current mode)", -) @click.option("--sql", is_flag=True, help="Output SQL without executing it") @load_extension_option def transform( @@ -2740,7 +2567,6 @@ def transform( default_none, add_foreign_keys, drop_foreign_keys, - strict, sql, load_extension, ): @@ -2802,7 +2628,6 @@ def transform( defaults=default_dict, drop_foreign_keys=drop_foreign_keys_value, add_foreign_keys=add_foreign_keys_value, - strict=strict, ): click.echo(line) else: @@ -2816,7 +2641,6 @@ def transform( defaults=default_dict, drop_foreign_keys=drop_foreign_keys_value, add_foreign_keys=add_foreign_keys_value, - strict=strict, ) @@ -2864,10 +2688,7 @@ def extract( fk_column=fk_column, rename=dict(rename), ) - try: - db.table(table).extract(**kwargs) - except (NoTable, InvalidColumns) as e: - raise click.ClickException(str(e)) + db.table(table).extract(**kwargs) @cli.command(name="insert-files") @@ -3176,12 +2997,6 @@ def _generate_convert_help(): "value" is a variable with the column value to be converted. - CODE can also be a reference to a callable that takes the value, for example: - - \b - sqlite-utils convert my.db mytable date r.parsedate - sqlite-utils convert my.db mytable data json.loads --import json - Use "-" for CODE to read Python code from standard input. The following common operations are available as recipe functions: @@ -3195,8 +3010,9 @@ def _generate_convert_help(): ] for name in recipe_names: fn = getattr(recipes, name) - doc = textwrap.dedent(fn.__doc__.rstrip()).replace("\b\n", "") - help += "\n\nr.{}{}\n\n\b{}".format(name, str(inspect.signature(fn)), doc) + help += "\n\nr.{}{}\n\n\b{}".format( + name, str(inspect.signature(fn)), textwrap.dedent(fn.__doc__.rstrip()) + ) help += "\n\n" help += textwrap.dedent(""" You can use these recipes like so: @@ -3283,7 +3099,7 @@ def convert( if multi: def preview(v): - return json.dumps(fn(v), default=repr, ensure_ascii=False) if v else v + return json.dumps(fn(v), default=repr) if v else v else: @@ -3568,14 +3384,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): # Listing is read-only - don't create the database file db = sqlite_utils.Database(memory=True) _register_db_for_cleanup(db) - # Legacy sqlite-migrate classes create the migrations table from - # their pending()/applied() methods - run the listing inside a - # transaction and roll it back so --list stays read-only - db.begin() - try: - _display_migration_list(db, migration_sets) - finally: - db.rollback() + _display_migration_list(db, migration_sets) return db = sqlite_utils.Database(db_path) @@ -3607,10 +3416,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): for migration_set in migration_sets: matches = _stop_before_for_migration_set(stop_before, migration_set.name) if isinstance(migration_set, sqlite_utils.Migrations): - try: - migration_set.apply(db, stop_before=matches) - except ValueError as e: - raise click.ClickException(str(e)) + migration_set.apply(db, stop_before=matches) else: # Legacy sqlite-migrate Migrations objects take a single string # for stop_before, not a list @@ -3639,7 +3445,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): @cli.command(name="plugins") def plugins_list(): "List installed plugins" - click.echo(json.dumps(get_plugins(), indent=2, ensure_ascii=False)) + click.echo(json.dumps(get_plugins(), indent=2)) ensure_plugins_loaded() @@ -3686,11 +3492,7 @@ FILE_COLUMNS = { } -def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False): - # Duplicate column names would collide as dictionary keys, so rename - # later occurrences id, id -> id, id_2 - CSV and table output keep - # the original duplicate headers since they never build dictionaries - headers = dedupe_keys(headers) +def output_rows(iterator, headers, nl, arrays, json_cols): # We have to iterate two-at-a-time so we can know if we # should output a trailing comma or if we have reached # the last row. @@ -3707,7 +3509,7 @@ def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False): data = dict(zip(headers, data)) line = "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar=("[" if first else " ") if not nl else "", - serialized=json.dumps(data, default=json_binary, ensure_ascii=ascii_), + serialized=json.dumps(data, default=json_binary), maybecomma="," if (not nl and not is_last) else "", lastchar="]" if (is_last and not nl) else "", ) @@ -3788,32 +3590,3 @@ def _maybe_register_functions(db, functions_list): for functions in functions_list: if isinstance(functions, str) and functions.strip(): _register_functions(db, functions) - - -def _rows_from_code(code): - # code may be a path to a .py file - if "\n" not in code and code.endswith(".py"): - try: - code = pathlib.Path(code).read_text() - except FileNotFoundError: - raise click.ClickException("File not found: {}".format(code)) - namespace = {} - try: - exec(code, namespace) - except SyntaxError as ex: - raise click.ClickException("Error in --code: {}".format(ex)) - rows = namespace.get("rows") - if callable(rows): - rows = rows() - if isinstance(rows, dict): - rows = [rows] - error = click.ClickException( - "--code must define a 'rows' function or iterable of rows to insert" - ) - if rows is None or isinstance(rows, (str, bytes)): - raise error - try: - iter(rows) - except TypeError: - raise error - return rows diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e97b7d9..8e6a194 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,6 +1,5 @@ from .utils import ( chunks, - dedupe_keys, hash_record, sqlite3, OperationalError, @@ -12,7 +11,6 @@ from .utils import ( ) import binascii from collections import namedtuple -from dataclasses import dataclass, field from collections.abc import Mapping import contextlib import datetime @@ -53,11 +51,6 @@ except ImportError: SQLITE_MAX_VARS = 999 -# Names that refer to a rowid table's implicit integer primary key. These are -# valid primary key targets even though they are not listed among a table's -# columns. See https://www.sqlite.org/lang_createtable.html#rowid -ROWID_ALIASES = frozenset({"rowid", "_rowid_", "oid"}) - _quote_fts_re = re.compile(r'\s+|(".*?")') _virtual_table_using_re = re.compile( @@ -90,34 +83,6 @@ def quote_identifier(identifier: str) -> str: return '"{}"'.format(identifier.replace('"', '""')) -_IDENTIFIER_CASEFOLD = str.maketrans( - "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" -) - - -def fold_identifier_case(identifier: str) -> str: - """ - Lowercase an identifier using the same rules SQLite uses - only ASCII - characters are folded, other characters are left unchanged. - """ - return identifier.translate(_IDENTIFIER_CASEFOLD) - - -def resolve_casing(name: str, candidates: Iterable[str]) -> str: - """ - SQLite treats identifiers as case-insensitive. Return the entry in - ``candidates`` that matches ``name`` case-insensitively, preferring an - exact match. If nothing matches, return ``name`` unchanged. - """ - if name in candidates: - return name - folded = fold_identifier_case(name) - for candidate in candidates: - if fold_identifier_case(candidate) == folded: - return candidate - return name - - pd: Any = None try: pd = importlib.import_module("pandas") @@ -196,75 +161,9 @@ Summary information about a column, see :ref:`python_api_analyze_column`. The ``N`` least common values as a list of ``(value, count)`` tuples, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) """ - - -@dataclass(order=True, frozen=True) -class ForeignKey: - """ - A foreign key defined on a table. - - For single-column foreign keys ``column`` and ``other_column`` hold the - column names, and ``columns``/``other_columns`` are one-item tuples. - - For compound (multi-column) foreign keys ``column`` and ``other_column`` - are ``None`` - use ``columns`` and ``other_columns`` instead, and check - ``is_compound``. - - ``on_delete`` and ``on_update`` hold the foreign key actions, e.g. - ``"CASCADE"`` - ``"NO ACTION"`` if not set. - - Instances are immutable and hashable, so they can be collected into - sets and used as dictionary keys. Equality covers every compared field, - including ``on_delete`` and ``on_update`` - two foreign keys differing - only in their actions are different constraints. - - Prior to sqlite-utils 4.0 this was a ``namedtuple`` and could be unpacked - or indexed as ``(table, column, other_table, other_column)``. It is now a - dataclass - access its fields by name instead. - """ - - table: str - # column/other_column are None for compound keys, which would break - # ordering against str values - comparison uses columns/other_columns - column: Optional[str] = field(compare=False) - other_table: str - other_column: Optional[str] = field(compare=False) - columns: Tuple[str, ...] = () - other_columns: Tuple[str, ...] = () - is_compound: bool = False - on_delete: str = "NO ACTION" - on_update: str = "NO ACTION" - - def __post_init__(self): - # Populate columns/other_columns for single-column foreign keys, - # normalizing any lists to tuples. object.__setattr__ because the - # dataclass is frozen - if self.columns: - object.__setattr__(self, "columns", tuple(self.columns)) - else: - object.__setattr__( - self, "columns", (self.column,) if self.column is not None else () - ) - if self.other_columns: - object.__setattr__(self, "other_columns", tuple(self.other_columns)) - else: - object.__setattr__( - self, - "other_columns", - (self.other_column,) if self.other_column is not None else (), - ) - - -def _fk_actions_sql(fk: ForeignKey) -> str: - "ON UPDATE/ON DELETE clauses for a foreign key, or an empty string." - actions = "" - if fk.on_update and fk.on_update != "NO ACTION": - actions += " ON UPDATE {}".format(fk.on_update) - if fk.on_delete and fk.on_delete != "NO ACTION": - actions += " ON DELETE {}".format(fk.on_delete) - return actions - - +ForeignKey = namedtuple( + "ForeignKey", ("table", "column", "other_table", "other_column") +) Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) XIndex = namedtuple("XIndex", ("name", "columns")) XIndexColumn = namedtuple( @@ -277,18 +176,12 @@ class TransformError(Exception): pass -# A single column name, or a tuple of columns for a compound foreign key -ForeignKeyColumns = Union[str, Tuple[str, ...], List[str]] - -# (table, column(s), other_table, other_column(s)) -ForeignKeyTuple = Tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] - ForeignKeyIndicator = Union[ str, ForeignKey, - Tuple[ForeignKeyColumns, str], - Tuple[ForeignKeyColumns, str, ForeignKeyColumns], - ForeignKeyTuple, + Tuple[str, str], + Tuple[str, str, str], + Tuple[str, str, str, str], ] ForeignKeysType = Union[Iterable[ForeignKeyIndicator], List[ForeignKeyIndicator]] @@ -431,52 +324,14 @@ CREATE TABLE IF NOT EXISTS "{}"( """.strip() -_TRANSACTION_CONTROL_KEYWORDS = { +_TRANSACTION_CONTROL_PREFIXES = ( "BEGIN", "COMMIT", "END", "ROLLBACK", "SAVEPOINT", "RELEASE", -} - -# Statements that never return rows and cannot run inside (or would break -# out of) the savepoint guard used by query() -_QUERY_REJECTED_KEYWORDS = _TRANSACTION_CONTROL_KEYWORDS | { - "VACUUM", - "ATTACH", - "DETACH", -} - - -def _first_keyword(sql: str) -> str: - """ - Return the first keyword of a SQL statement, uppercased, skipping - everything the sqlite3 driver tolerates before the first real token: - whitespace, ``--`` or ``/* ... */`` comments, empty statements - (bare ``;``) and a UTF-8 byte order mark. Returns an empty string if - there is no leading keyword. - """ - i, n = 0, len(sql) - while i < n: - if sql[i].isspace() or sql[i] in (";", "\ufeff"): - i += 1 - elif sql.startswith("--", i): - newline = sql.find("\n", i) - if newline == -1: - return "" - i = newline + 1 - elif sql.startswith("/*", i): - end = sql.find("*/", i + 2) - if end == -1: - return "" - i = end + 2 - else: - break - j = i - while j < n and (sql[j].isalpha() or sql[j] == "_"): - j += 1 - return sql[i:j].upper() +) class Database: @@ -607,13 +462,8 @@ class Database: try: yield self except BaseException: - # An error such as a RAISE(ROLLBACK) trigger can destroy - # the whole transaction, savepoints included - cleaning up - # anyway would mask the original exception with - # "no such savepoint" - if self.conn.in_transaction: - self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) - self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) + self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) raise else: self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) @@ -622,15 +472,13 @@ class Database: try: yield self except BaseException: - # rollback() is a no-op if the error already destroyed the - # transaction, so the original exception propagates - self.rollback() + self.conn.execute("ROLLBACK") raise else: try: self.conn.execute("COMMIT") except BaseException: - self.rollback() + self.conn.execute("ROLLBACK") raise def begin(self) -> None: @@ -661,32 +509,17 @@ class Database: self.conn.execute("ROLLBACK") @contextlib.contextmanager - def ensure_autocommit_on(self) -> Generator[None, None, None]: + def ensure_autocommit_off(self) -> Generator[None, None, None]: """ - Ensure the connection is in driver-level autocommit mode for the - duration of a block of code. - - This temporarily sets ``isolation_level = None`` on the underlying - ``sqlite3`` connection, so the driver does not open implicit - transactions. This is useful for statements such as - ``PRAGMA journal_mode=wal`` which cannot run inside a transaction. + Ensure autocommit is off for this database connection. Example usage:: - with db.ensure_autocommit_on(): + with db.ensure_autocommit_off(): # do stuff here - The previous ``isolation_level`` is restored at the end of the block. - - :raises TransactionError: if a transaction is open - assigning - ``isolation_level`` would commit it as a side effect, silently - breaking the caller's ability to roll back + This will reset to the previous autocommit state at the end of the block. """ - if self.conn.in_transaction: - raise TransactionError( - "ensure_autocommit_on() cannot be used inside a transaction - " - "changing isolation_level would commit the open transaction" - ) old_isolation_level = self.conn.isolation_level try: self.conn.isolation_level = None @@ -828,35 +661,30 @@ class Database: parameters, or a dictionary for ``where id = :id`` :raises ValueError: if the SQL statement does not return rows - use :meth:`execute` for those statements instead. The rejected statement - is rolled back, so it has no effect on the database. One exception: - a row-less ``PRAGMA`` statement takes effect despite the - ``ValueError``, because PRAGMAs run outside the savepoint guard - - some of them refuse to run inside a transaction + is rolled back, so it has no effect on the database """ message = ( "query() can only be used with SQL that returns rows - " "use execute() for other statements" ) - keyword = _first_keyword(sql) - if keyword in _QUERY_REJECTED_KEYWORDS: + prefix = sql.lstrip().upper() + if prefix.startswith( + _TRANSACTION_CONTROL_PREFIXES + ("VACUUM", "ATTACH", "DETACH") + ): # None of these return rows - reject them without executing anything raise ValueError(message) if self._tracer: self._tracer(sql, params) args: tuple = (params,) if params is not None else () - if keyword == "PRAGMA": + if prefix.startswith("PRAGMA"): # Some PRAGMA statements refuse to run inside a transaction, so - # execute these without the savepoint guard used below. Some - # adapters open an implicit transaction before comment-prefixed - # PRAGMAs, so temporarily use driver autocommit when it is safe. - if self.conn.in_transaction: - cursor = self.conn.execute(sql, *args) - else: - with self.ensure_autocommit_on(): - cursor = self.conn.execute(sql, *args) + # execute these without the savepoint guard used below. PRAGMAs + # never open an implicit transaction, so there is nothing to + # undo if this one turns out not to return rows + cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) - keys = dedupe_keys(d[0] for d in cursor.description) + keys = [d[0] for d in cursor.description] return (dict(zip(keys, row)) for row in cursor) # Execute inside a savepoint, so a statement that turns out not to # return rows can be rolled back before the ValueError is raised @@ -866,7 +694,7 @@ class Database: cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) - keys = dedupe_keys(d[0] for d in cursor.description) + keys = [d[0] for d in cursor.description] try: self.conn.execute('RELEASE "sqlite_utils_query"') released = True @@ -882,11 +710,8 @@ class Database: return (dict(zip(keys, row)) for row in fetched) return (dict(zip(keys, row)) for row in cursor) finally: - if not released and self.conn.in_transaction: - # An error occurred - undo anything the statement changed. - # If the error itself destroyed the transaction (such as a - # RAISE(ROLLBACK) trigger) the savepoint is already gone - # and there is nothing left to undo + if not released: + # An error occurred - undo anything the statement changed self.conn.execute('ROLLBACK TO "sqlite_utils_query"') self.conn.execute('RELEASE "sqlite_utils_query"') @@ -908,23 +733,15 @@ class Database: if self._tracer: self._tracer(sql, parameters) was_in_transaction = self.conn.in_transaction - try: - if parameters is not None: - cursor = self.conn.execute(sql, parameters) - else: - cursor = self.conn.execute(sql) - except Exception: - if not was_in_transaction and self.conn.in_transaction: - # The failed statement opened an implicit transaction that - # nothing would ever commit - roll it back, otherwise it - # would capture every subsequent write - self.conn.execute("ROLLBACK") - raise + if parameters is not None: + cursor = self.conn.execute(sql, parameters) + else: + cursor = self.conn.execute(sql) if ( not was_in_transaction and self.conn.in_transaction and cursor.description is None - and _first_keyword(sql) not in _TRANSACTION_CONTROL_KEYWORDS + and not sql.lstrip().upper().startswith(_TRANSACTION_CONTROL_PREFIXES) ): # The statement opened an implicit transaction - commit it, so # that execute() behaves consistently with the rest of the @@ -1029,11 +846,6 @@ class Database: if str(value).upper() in ("CURRENT_TIME", "CURRENT_DATE", "CURRENT_TIMESTAMP"): return value - if isinstance(value, str) and value.upper() in ("TRUE", "FALSE", "NULL"): - # Keyword literals must stay unquoted; quoting them would turn the - # default into a string ('TRUE' instead of 1, 'NULL' instead of null). - return value - if str(value).endswith(")"): # Expr return "({})".format(value) @@ -1170,7 +982,7 @@ class Database: """ if self.journal_mode != "wal": self._ensure_no_open_transaction("enable_wal()") - with self.ensure_autocommit_on(): + with self.ensure_autocommit_off(): self.execute("PRAGMA journal_mode=wal;") def disable_wal(self) -> None: @@ -1182,7 +994,7 @@ class Database: """ if self.journal_mode != "delete": self._ensure_no_open_transaction("disable_wal()") - with self.ensure_autocommit_on(): + with self.ensure_autocommit_off(): self.execute("PRAGMA journal_mode=delete;") def _ensure_no_open_transaction(self, operation: str) -> None: @@ -1254,31 +1066,26 @@ class Database: :param name: Name of table that foreign keys are being defined for :param foreign_keys: List of foreign keys, each of which can be a - string, a ForeignKey() object, a tuple of (column, other_table), + string, a ForeignKey() named tuple, a tuple of (column, other_table), or a tuple of (column, other_table, other_column), or a tuple of - (table, column, other_table, other_column). For compound foreign - keys the column elements can be tuples of column names, e.g. - (("campus_name", "dept_code"), "departments") or - (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) + (table, column, other_table, other_column) """ table = self.table(name) - fks = [] - for fk in foreign_keys: - if isinstance(fk, ForeignKey): - fks.append(fk) - continue - if isinstance(fk, str): - # A bare column name - guess the other table and column - other_table = table.guess_foreign_table(fk) + if all(isinstance(fk, ForeignKey) for fk in foreign_keys): + return cast(List[ForeignKey], foreign_keys) + if all(isinstance(fk, str) for fk in foreign_keys): + # It's a list of columns + fks = [] + for column in foreign_keys: + column = cast(str, column) + other_table = table.guess_foreign_table(column) other_column = table.guess_foreign_column(other_table) - fks.append(ForeignKey(name, fk, other_table, other_column)) - continue - if not isinstance(fk, (tuple, list)): - raise ValueError( - "foreign_keys= should be a list of tuples, " - "ForeignKey objects or column name strings" - ) - tuple_or_list = cast(Sequence[Any], fk) + fks.append(ForeignKey(name, column, other_table, other_column)) + return fks + if not all(isinstance(fk, (tuple, list)) for fk in foreign_keys): + raise ValueError("foreign_keys= should be a list of tuples") + fks = [] + for tuple_or_list in foreign_keys: if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( @@ -1286,106 +1093,32 @@ class Database: tuple_or_list, name ) ) - tuple_or_list = tuple_or_list[1:] - if len(tuple_or_list) not in (2, 3): + if len(tuple_or_list) not in (2, 3, 4): raise ValueError( "foreign_keys= should be a list of tuple pairs or triples" ) - column_or_columns = tuple_or_list[0] - other_table = tuple_or_list[1] - if isinstance(column_or_columns, (list, tuple)): - # Compound foreign key - columns = tuple(column_or_columns) - if len(tuple_or_list) == 3: - if not isinstance(tuple_or_list[2], (list, tuple)): - raise ValueError( - "Compound foreign key {} should reference a tuple " - "of other columns".format(tuple(tuple_or_list)) - ) - other_columns = tuple(tuple_or_list[2]) + if len(tuple_or_list) in (3, 4): + if len(tuple_or_list) == 4: + tuple_or_list = cast(Tuple[str, str, str], tuple_or_list[1:]) else: - # Guess the compound primary key of the other table - other_columns = tuple(self.table(other_table).pks) - if len(columns) != len(other_columns): - raise ValueError( - "Compound foreign key {} should have the same number " - "of columns on both sides".format(tuple(tuple_or_list)) - ) - if len(columns) == 1: - # Single-column key passed as a one-item list - fks.append( - ForeignKey(name, columns[0], other_table, other_columns[0]) - ) - else: - fks.append( - ForeignKey( - name, - None, - other_table, - None, - columns=columns, - other_columns=other_columns, - is_compound=True, - ) - ) - elif len(tuple_or_list) == 3: + tuple_or_list = cast(Tuple[str, str, str], tuple_or_list) fks.append( - ForeignKey(name, column_or_columns, other_table, tuple_or_list[2]) + ForeignKey( + name, tuple_or_list[0], tuple_or_list[1], tuple_or_list[2] + ) ) else: # Guess the primary key fks.append( ForeignKey( name, - column_or_columns, - other_table, - table.guess_foreign_column(other_table), + tuple_or_list[0], + tuple_or_list[1], + table.guess_foreign_column(tuple_or_list[1]), ) ) return fks - def _resolve_foreign_key_casing( - self, fk: ForeignKey, columns: Iterable[str] - ) -> ForeignKey: - """ - Return ``fk`` with its column references resolved to match the casing - of the actual columns. ``columns`` provides the column names of - ``fk.table``, which may be a table that is still being created. - """ - resolved_columns = tuple(resolve_casing(c, columns) for c in fk.columns) - if fk.other_table == fk.table: - other_candidates: Iterable[str] = columns - else: - other_candidates = self[fk.other_table].columns_dict - resolved_other_columns = tuple( - resolve_casing(c, other_candidates) for c in fk.other_columns - ) - if ( - resolved_columns == fk.columns - and resolved_other_columns == fk.other_columns - ): - return fk - if fk.is_compound: - return ForeignKey( - fk.table, - None, - fk.other_table, - None, - columns=resolved_columns, - other_columns=resolved_other_columns, - is_compound=True, - on_delete=fk.on_delete, - on_update=fk.on_update, - ) - return ForeignKey( - fk.table, - resolved_columns[0], - fk.other_table, - resolved_other_columns[0], - on_delete=fk.on_delete, - on_update=fk.on_update, - ) - def create_table_sql( self, name: str, @@ -1419,15 +1152,8 @@ class Database: """ if hash_id_columns and (hash_id is None): hash_id = "id" - resolved_fks: List[ForeignKey] = [ - self._resolve_foreign_key_casing(fk, columns) - for fk in self.resolve_foreign_keys(name, foreign_keys or []) - ] - # Compound foreign keys are rendered as table-level constraints; - # single-column ones as inline REFERENCES on their column - foreign_keys_by_column = { - fk.column: fk for fk in resolved_fks if not fk.is_compound - } + foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) + foreign_keys_by_column = {fk.column: fk for fk in foreign_keys} # any extracts will be treated as integer columns with a foreign key extracts = resolve_extracts(extracts) for extract_column, extract_table in extracts.items(): @@ -1441,10 +1167,8 @@ class Database: name, extract_column, extract_table, "id" ) # Soundness check not_null, and defaults if provided - not_null = {resolve_casing(n, columns) for n in not_null or set()} - defaults = {resolve_casing(n, columns): v for n, v in (defaults or {}).items()} - if column_order is not None: - column_order = [resolve_casing(c, columns) for c in column_order] + not_null = not_null or set() + defaults = defaults or {} if not columns: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): @@ -1470,28 +1194,25 @@ class Database: column_items.insert(0, (hash_id, str)) pk = hash_id # Soundness check foreign_keys point to existing tables - for fk in resolved_fks: - for other_column in fk.other_columns: - if fk.other_table == name and columns.get(other_column): - continue - if other_column != "rowid" and not any( - c for c in self[fk.other_table].columns if c.name == other_column - ): - raise AlterError( - "No such column: {}.{}".format(fk.other_table, other_column) - ) + for fk in foreign_keys: + if fk.other_table == name and columns.get(fk.other_column): + continue + if fk.other_column != "rowid" and not any( + c for c in self[fk.other_table].columns if c.name == fk.other_column + ): + raise AlterError( + "No such column: {}.{}".format(fk.other_table, fk.other_column) + ) column_defs = [] # ensure pk is a tuple single_pk = None - if isinstance(pk, (list, tuple)) and len(pk) == 1 and isinstance(pk[0], str): + if isinstance(pk, list) and len(pk) == 1 and isinstance(pk[0], str): pk = pk[0] if isinstance(pk, str): - single_pk = pk = resolve_casing(pk, [c[0] for c in column_items]) + single_pk = pk if pk not in [c[0] for c in column_items]: column_items.insert(0, (pk, int)) - elif pk: - pk = [resolve_casing(p, [c[0] for c in column_items]) for p in pk] for column_name, column_type in column_items: column_extras = [] if column_name == single_pk: @@ -1503,12 +1224,14 @@ class Database: "DEFAULT {}".format(self.quote_default_value(defaults[column_name])) ) if column_name in foreign_keys_by_column: - fk = foreign_keys_by_column[column_name] column_extras.append( - "REFERENCES {}({}){}".format( - quote_identifier(fk.other_table), - quote_identifier(cast(str, fk.other_column)), - _fk_actions_sql(fk), + "REFERENCES {}({})".format( + quote_identifier( + foreign_keys_by_column[column_name].other_table + ), + quote_identifier( + foreign_keys_by_column[column_name].other_column + ), ) ) column_type_str = COLUMN_TYPE_MAPPING[column_type] @@ -1530,26 +1253,6 @@ class Database: extra_pk = ",\n PRIMARY KEY ({pks})".format( pks=", ".join([quote_identifier(p) for p in pk]) ) - # Compound foreign keys become table-level FOREIGN KEY constraints - column_names = [c[0] for c in column_items] - for fk in resolved_fks: - if not fk.is_compound: - continue - missing = [c for c in fk.columns if c not in column_names] - if missing: - raise AlterError( - "No such column: {}".format(", ".join(sorted(missing))) - ) - column_defs.append( - " FOREIGN KEY ({columns}) REFERENCES {other_table}({other_columns}){actions}".format( - columns=", ".join(quote_identifier(c) for c in fk.columns), - other_table=quote_identifier(fk.other_table), - other_columns=", ".join( - quote_identifier(c) for c in fk.other_columns - ), - actions=_fk_actions_sql(fk), - ) - ) columns_sql = ",\n".join(column_defs) sql = """CREATE TABLE {if_not_exists}{table} ( {columns_sql}{extra_pk} @@ -1613,11 +1316,6 @@ class Database: should_transform = False # First add missing columns and figure out columns to drop existing_columns = table.columns_dict - # Match existing columns case-insensitively, the way SQLite does - columns = { - resolve_casing(col_name, existing_columns): col_type - for col_name, col_type in columns.items() - } missing_columns = dict( (col_name, col_type) for col_name, col_type in columns.items() @@ -1641,28 +1339,18 @@ class Database: current_pks = table.pks desired_pk = None if isinstance(pk, str): - desired_pk = [resolve_casing(pk, existing_columns)] + desired_pk = [pk] elif pk: - desired_pk = [resolve_casing(p, existing_columns) for p in pk] + desired_pk = list(pk) if desired_pk and current_pks != desired_pk: should_transform = True # Any not-null changes? current_not_null = {c.name for c in table.columns if c.notnull} - desired_not_null = ( - {resolve_casing(n, existing_columns) for n in not_null} - if not_null - else set() - ) + desired_not_null = set(not_null) if not_null else set() if current_not_null != desired_not_null: should_transform = True # How about defaults? - if ( - defaults - and { - resolve_casing(c, existing_columns): v for c, v in defaults.items() - } - != table.default_values - ): + if defaults and defaults != table.default_values: should_transform = True # Only run .transform() if there is something to do if should_transform: @@ -1762,121 +1450,57 @@ class Database: return candidates def add_foreign_keys( - self, foreign_keys: Iterable[Union[ForeignKey, ForeignKeyTuple]] + self, foreign_keys: Iterable[Tuple[str, str, str, str]] ) -> None: """ See :ref:`python_api_add_foreign_keys`. :param foreign_keys: A list of ``(table, column, other_table, other_column)`` - tuples - for compound foreign keys, ``column`` and ``other_column`` can - be tuples of column names + tuples """ # foreign_keys is a list of explicit 4-tuples if not all( - isinstance(fk, ForeignKey) - or (isinstance(fk, (list, tuple)) and len(fk) == 4) - for fk in foreign_keys + len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys ): raise ValueError( "foreign_keys must be a list of 4-tuples, " "(table, column, other_table, other_column)" ) - foreign_keys_to_create: List[ForeignKey] = [] + foreign_keys_to_create = [] # Verify that all tables and columns exist - for fk in foreign_keys: - if isinstance(fk, ForeignKey): - fk_object = fk - else: - table, column_or_columns, other_table, other_column_or_columns = fk - # Compound foreign keys use tuples of columns - columns = ( - (column_or_columns,) - if isinstance(column_or_columns, str) - else tuple(column_or_columns) - ) - other_columns = ( - (other_column_or_columns,) - if isinstance(other_column_or_columns, str) - else tuple(other_column_or_columns) - ) - if len(columns) != len(other_columns): - raise ValueError( - "Compound foreign key must have the same number of " - "columns on both sides" - ) - if len(columns) == 1: - fk_object = ForeignKey( - table, columns[0], other_table, other_columns[0] - ) - else: - fk_object = ForeignKey( - table, - None, - other_table, - None, - columns=columns, - other_columns=other_columns, - is_compound=True, - ) - table = fk_object.table - other_table = fk_object.other_table + for table, column, other_table, other_column in foreign_keys: if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) - fk_object = self._resolve_foreign_key_casing( - fk_object, table_obj.columns_dict - ) - columns = fk_object.columns - other_columns = fk_object.other_columns - for column in columns: - if column not in table_obj.columns_dict: - raise AlterError("No such column: {} in {}".format(column, table)) + if column not in table_obj.columns_dict: + raise AlterError("No such column: {} in {}".format(column, table)) if not self[other_table].exists(): raise AlterError("No such other_table: {}".format(other_table)) - for other_column in other_columns: - if ( - other_column != "rowid" - and other_column not in self[other_table].columns_dict - ): - raise AlterError( - "No such other_column: {} in {}".format( - other_column, other_table - ) - ) - # Silently skip foreign keys that exist already - but only if - # they match exactly, including ON DELETE/ON UPDATE actions - columns_folded = tuple(fold_identifier_case(c) for c in columns) - other_columns_folded = tuple(fold_identifier_case(c) for c in other_columns) - existing = [ - fk - for fk in table_obj.foreign_keys - if tuple(fold_identifier_case(c) for c in fk.columns) == columns_folded - and fold_identifier_case(fk.other_table) - == fold_identifier_case(other_table) - and tuple(fold_identifier_case(c) for c in fk.other_columns) - == other_columns_folded - ] - if not existing: - foreign_keys_to_create.append(fk_object) - elif any( - fk.on_delete != fk_object.on_delete - or fk.on_update != fk_object.on_update - for fk in existing + if ( + other_column != "rowid" + and other_column not in self[other_table].columns_dict ): raise AlterError( - "Foreign key already exists for {} => {}.{} but with " - "different ON DELETE/ON UPDATE actions - use " - "table.transform() to change them".format( - ", ".join(columns), other_table, ", ".join(other_columns) - ) + "No such other_column: {} in {}".format(other_column, other_table) + ) + # We will silently skip foreign keys that exist already + if not any( + fk + for fk in table_obj.foreign_keys + if fk.column == column + and fk.other_table == other_table + and fk.other_column == other_column + ): + foreign_keys_to_create.append( + (table, column, other_table, other_column) ) # Group them by table - by_table: Dict[str, List[ForeignKey]] = {} - for fk_object in foreign_keys_to_create: - by_table.setdefault(fk_object.table, []).append(fk_object) + by_table: Dict[str, List] = {} + for fk in foreign_keys_to_create: + by_table.setdefault(fk[0], []).append(fk) for table, fks in by_table.items(): self.table(table).transform(add_foreign_keys=fks) @@ -1888,12 +1512,12 @@ class Database: "Create indexes for every foreign key column on every table in the database." for table_name in self.table_names(): table = self.table(table_name) - existing_indexes = {tuple(i.columns) for i in table.indexes} + existing_indexes = { + i.columns[0] for i in table.indexes if len(i.columns) == 1 + } for fk in table.foreign_keys: - # A compound foreign key gets a single composite index - if fk.columns not in existing_indexes: - table.create_index(fk.columns, find_unique_name=True) - existing_indexes.add(fk.columns) + if fk.column not in existing_indexes: + table.create_index([fk.column], find_unique_name=True) def vacuum(self) -> None: "Run a SQLite ``VACUUM`` against the database." @@ -2043,7 +1667,7 @@ class Queryable: if offset is not None: sql += " offset {}".format(offset) cursor = self.db.execute(sql, where_args or []) - columns = dedupe_keys(c[0] for c in cursor.description) + columns = [c[0] for c in cursor.description] for row in cursor: yield dict(zip(columns, row)) @@ -2066,22 +1690,12 @@ class Queryable: :param limit: Integer number of rows to limit to :param offset: Integer for SQL offset """ - # This method is defined on Queryable so it serves views too, which - # have no pks property - sort pk columns into declaration order here - pk_columns = sorted( - (column for column in self.columns if column.is_pk), - key=lambda column: column.is_pk, - ) - pks = [column.name for column in pk_columns] - select_parts = [quote_identifier(column.name) for column in self.columns] + column_names = [column.name for column in self.columns] + pks = [column.name for column in self.columns if column.is_pk] if not pks: - # rowid is left unquoted: it is not a real column, and SQLite - # turns a double-quoted identifier that does not resolve into a - # string literal - on a view that would silently select the - # string 'rowid' instead of raising an error - select_parts.insert(0, "rowid") + column_names.insert(0, "rowid") pks = ["rowid"] - select = ",".join(select_parts) + select = ",".join(quote_identifier(column_name) for column_name in column_names) for row in self.rows_where( select=select, where=where, @@ -2213,18 +1827,8 @@ class Table(Queryable): @property def pks(self) -> List[str]: - """ - Primary key columns for this table, in PRIMARY KEY declaration order - - ``PRAGMA table_info`` sets ``is_pk`` to the 1-based position of each - column within the primary key, which can differ from the order of the - columns in the table. SQLite uses the declaration order to resolve - implicit foreign key references, so this order matters. - """ - pk_columns = sorted( - (column for column in self.columns if column.is_pk), - key=lambda column: column.is_pk, - ) - names = [column.name for column in pk_columns] + "Primary key columns for this table." + names = [column.name for column in self.columns if column.is_pk] if not names: names = ["rowid"] return names @@ -2264,50 +1868,21 @@ class Table(Queryable): @property def foreign_keys(self) -> List["ForeignKey"]: - """ - List of foreign keys defined on this table. - - Compound (multi-column) foreign keys are returned as a single - ``ForeignKey`` with ``is_compound=True`` and populated - ``columns``/``other_columns`` lists. - """ - # PRAGMA foreign_key_list returns one row per column, grouped by "id" - # with "seq" giving the column order within a compound foreign key. - by_id: Dict[int, list] = {} + "List of foreign keys defined on this table." + fks = [] for row in self.db.execute( "PRAGMA foreign_key_list({})".format(quote_identifier(self.name)) ).fetchall(): if row is not None: id, seq, table_name, from_, to_, on_update, on_delete, match = row - by_id.setdefault(id, []).append( - (seq, table_name, from_, to_, on_update, on_delete) + fks.append( + ForeignKey( + table=self.name, + column=from_, + other_table=table_name, + other_column=to_, + ) ) - fks = [] - for id in sorted(by_id): - rows = sorted(by_id[id]) # order columns by seq - other_table = rows[0][1] - columns = tuple(row[2] for row in rows) - other_columns = tuple(row[3] for row in rows) - if all(c is None for c in other_columns): - # "REFERENCES other_table" with no columns - the pragma - # returns None, meaning the other table's primary key - other_table_pks = tuple(self.db.table(other_table).pks) - if len(other_table_pks) == len(columns): - other_columns = other_table_pks - is_compound = len(rows) > 1 - fks.append( - ForeignKey( - table=self.name, - column=None if is_compound else columns[0], - other_table=other_table, - other_column=None if is_compound else other_columns[0], - columns=columns, - other_columns=other_columns, - is_compound=is_compound, - on_update=rows[0][4], - on_delete=rows[0][5], - ) - ) return fks @property @@ -2514,7 +2089,6 @@ class Table(Queryable): foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, keep_table: Optional[str] = None, - strict: Optional[bool] = None, ) -> "Table": """ Apply an advanced alter table, including operations that are not supported by @@ -2522,28 +2096,19 @@ class Table(Queryable): See :ref:`python_api_transform` for full details. - Raises :py:class:`sqlite_utils.db.TransactionError` if called while a - transaction is open with ``PRAGMA foreign_keys`` enabled and the table - is referenced by foreign keys with destructive ``ON DELETE`` actions - - see :ref:`python_api_transform_foreign_keys_transactions`. - :param types: Columns that should have their type changed, for example ``{"weight": float}`` :param rename: Columns to rename, for example ``{"headline": "title"}`` :param drop: Columns to drop :param pk: New primary key for the table :param not_null: Columns to set as ``NOT NULL`` :param defaults: Default values for columns - :param drop_foreign_keys: Foreign key constraints to remove - a column name - drops any foreign key that column participates in, a tuple of column names - drops the compound foreign key with exactly those columns + :param drop_foreign_keys: Names of columns that should have their foreign key constraints removed :param add_foreign_keys: List of foreign keys to add to the table :param foreign_keys: List of foreign keys to set for the table, replacing any existing foreign keys :param column_order: List of strings specifying a full or partial column order to use when creating the table :param keep_table: If specified, the existing table will be renamed to this and will not be dropped - :param strict: Set to ``True`` to make the table strict or ``False`` to make it - non-strict. Defaults to ``None``, which preserves the existing strict mode. """ if not self.exists(): raise ValueError("Cannot transform a table that doesn't exist yet") @@ -2559,7 +2124,6 @@ class Table(Queryable): foreign_keys=foreign_keys, column_order=column_order, keep_table=keep_table, - strict=strict, ) pragma_foreign_keys_was_on = bool( self.db.execute("PRAGMA foreign_keys").fetchone()[0] @@ -2571,36 +2135,6 @@ class Table(Queryable): should_defer_foreign_keys = ( pragma_foreign_keys_was_on and already_in_transaction ) - if should_defer_foreign_keys: - # PRAGMA foreign_keys is a no-op inside a transaction, and - # defer_foreign_keys only defers violation checks, not ON DELETE - # actions - so dropping the old table would still fire destructive - # actions on any tables that reference it. Refuse rather than - # silently modify or delete those rows. - destructive_fks = [ - (table.name, fk) - for table in self.db.tables - for fk in table.foreign_keys - if fk.other_table == self.name - and fk.on_delete in ("CASCADE", "SET NULL", "SET DEFAULT") - ] - if destructive_fks: - raise TransactionError( - "Cannot transform table {table} while a transaction is open: " - "PRAGMA foreign_keys cannot be changed inside a transaction, " - "and the table is referenced by foreign keys with ON DELETE " - "actions that would fire when the old table is dropped: " - "{fks}. Call transform() outside of the transaction, or " - 'execute "PRAGMA foreign_keys = off" before opening it.'.format( - table=self.name, - fks=", ".join( - "{}.{} (ON DELETE {})".format( - table_name, ", ".join(fk.columns), fk.on_delete - ) - for table_name, fk in destructive_fks - ), - ) - ) defer_foreign_keys_was_on = False try: if should_disable_foreign_keys: @@ -2626,8 +2160,6 @@ class Table(Queryable): self.db.execute("PRAGMA defer_foreign_keys=OFF;") if should_disable_foreign_keys: self.db.execute("PRAGMA foreign_keys=1;") - if strict is not None: - self._defaults["strict"] = strict return self def transform_sql( @@ -2645,7 +2177,6 @@ class Table(Queryable): column_order: Optional[List[str]] = None, tmp_suffix: Optional[str] = None, keep_table: Optional[str] = None, - strict: Optional[bool] = None, ) -> List[str]: """ Return a list of SQL statements that should be executed in order to apply this transformation. @@ -2656,9 +2187,7 @@ class Table(Queryable): :param pk: New primary key for the table :param not_null: Columns to set as ``NOT NULL`` :param defaults: Default values for columns - :param drop_foreign_keys: Foreign key constraints to remove - a column name - drops any foreign key that column participates in, a tuple of column names - drops the compound foreign key with exactly those columns + :param drop_foreign_keys: Names of columns that should have their foreign key constraints removed :param add_foreign_keys: List of foreign keys to add to the table :param foreign_keys: List of foreign keys to set for the table, replacing any existing foreign keys :param column_order: List of strings specifying a full or partial column order @@ -2666,40 +2195,11 @@ class Table(Queryable): :param tmp_suffix: Suffix to use for the temporary table name :param keep_table: If specified, the existing table will be renamed to this and will not be dropped - :param strict: Set to ``True`` to make the table strict or ``False`` to make it - non-strict. Defaults to ``None``, which preserves the existing strict mode. """ - if strict is True and not self.db.supports_strict: - raise TransformError("SQLite does not support STRICT tables") types = types or {} rename = rename or {} drop = drop or set() - # Resolve column references against the existing schema, matching - # case-insensitively the way SQLite does - existing_columns = self.columns_dict - types = {resolve_casing(c, existing_columns): t for c, t in types.items()} - rename = {resolve_casing(c, existing_columns): v for c, v in rename.items()} - drop = {resolve_casing(c, existing_columns) for c in drop} - if pk is not DEFAULT and pk is not None: - if isinstance(pk, str): - pk = resolve_casing(pk, existing_columns) - else: - pk = [resolve_casing(p, existing_columns) for p in pk] - if isinstance(not_null, dict): - not_null = { - resolve_casing(c, existing_columns): v - for c, v in cast(Dict[str, Any], not_null).items() - } - elif isinstance(not_null, set): - not_null = {resolve_casing(c, existing_columns) for c in not_null} - if defaults is not None: - defaults = { - resolve_casing(c, existing_columns): v for c, v in defaults.items() - } - if column_order is not None: - column_order = [resolve_casing(c, existing_columns) for c in column_order] - create_table_foreign_keys: List[ForeignKeyIndicator] = [] if foreign_keys is not None: @@ -2714,68 +2214,29 @@ class Table(Queryable): create_table_foreign_keys.extend(foreign_keys) else: # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys - # The casing of columns in a foreign key definition can differ - # from the casing of the columns themselves, so these comparisons - # are all case-folded - dropped_columns_folded = {fold_identifier_case(c) for c in drop} - renamed_columns_folded = { - fold_identifier_case(k): v for k, v in rename.items() - } - - def fk_should_be_dropped(fk: ForeignKey) -> bool: - fk_columns_folded = tuple(fold_identifier_case(c) for c in fk.columns) - if drop_foreign_keys is not None: - for spec in drop_foreign_keys: - if isinstance(spec, str): - # A column name matches any foreign key it participates in - if fold_identifier_case(spec) in fk_columns_folded: - return True - elif ( - tuple(fold_identifier_case(s) for s in spec) - == fk_columns_folded - ): - # A tuple/list must match a compound key's columns exactly - return True - # Dropping any of a foreign key's columns drops the whole key - return any( - column in dropped_columns_folded for column in fk_columns_folded - ) - - def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: - columns = tuple( - renamed_columns_folded.get(fold_identifier_case(column)) or column - for column in fk.columns - ) - if fk.is_compound: - return ForeignKey( - self.name, - None, - fk.other_table, - None, - columns=columns, - other_columns=fk.other_columns, - is_compound=True, - on_delete=fk.on_delete, - on_update=fk.on_update, - ) - return ForeignKey( - self.name, - columns[0], - fk.other_table, - fk.other_columns[0], - on_delete=fk.on_delete, - on_update=fk.on_update, - ) - create_table_foreign_keys = [] - # Copy over old foreign keys, unless we are dropping them - for fk in self.foreign_keys: - if not fk_should_be_dropped(fk): - create_table_foreign_keys.append(fk_with_renamed_columns(fk)) + for table, column, other_table, other_column in self.foreign_keys: + # Copy over old foreign keys, unless we are dropping them + if (drop_foreign_keys is None) or (column not in drop_foreign_keys): + create_table_foreign_keys.append( + ForeignKey( + table, + rename.get(column) or column, + other_table, + other_column, + ) + ) # Add new foreign keys if add_foreign_keys is not None: for fk in self.db.resolve_foreign_keys(self.name, add_foreign_keys): - create_table_foreign_keys.append(fk_with_renamed_columns(fk)) + create_table_foreign_keys.append( + ForeignKey( + self.name, + rename.get(fk.column) or fk.column, + fk.other_table, + fk.other_column, + ) + ) new_table_name = "{}_new_{}".format( self.name, tmp_suffix or os.urandom(6).hex() @@ -2794,8 +2255,7 @@ class Table(Queryable): if pk is DEFAULT: pks_renamed = tuple( - rename.get(pk_name) or pk_name - for pk_name in (self.pks if not self.use_rowid else []) + rename.get(p.name) or p.name for p in self.columns if p.is_pk ) if len(pks_renamed) == 1: pk = pks_renamed[0] @@ -2852,7 +2312,7 @@ class Table(Queryable): defaults=create_table_defaults, foreign_keys=create_table_foreign_keys, column_order=column_order, - strict=self.strict if strict is None else strict, + strict=self.strict, ).strip() ) @@ -2934,8 +2394,6 @@ class Table(Queryable): rename = rename or {} if isinstance(columns, str): columns = [columns] - columns = [resolve_casing(c, self.columns_dict) for c in columns] - rename = {resolve_casing(k, self.columns_dict): v for k, v in rename.items()} if not set(columns).issubset(self.columns_dict.keys()): raise InvalidColumns( "Invalid columns {} for table with columns {}".format( @@ -2975,36 +2433,14 @@ class Table(Queryable): ) lookup_columns = [(rename.get(col) or col) for col in columns] lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) - # Rows where every extracted column is null are left alone - they - # get a null foreign key and no lookup table record, see #186 - all_columns_are_null = " AND ".join( - "{} IS NULL".format(quote_identifier(c)) for c in columns - ) - # INSERT OR IGNORE dedupes against the unique index, but unique - # indexes treat NULLs as distinct - the NOT EXISTS guard uses IS - # comparison so NULL-containing rows match existing lookup rows - # instead of being inserted again - already_in_lookup = " AND ".join( - "{lookup}.{lookup_col} IS {source}.{source_col}".format( - lookup=quote_identifier(table), - lookup_col=quote_identifier(rename.get(column) or column), - source=quote_identifier(self.name), - source_col=quote_identifier(column), - ) - for column in columns - ) self.db.execute( - "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {} " - "WHERE NOT ({all_null}) AND NOT EXISTS (SELECT 1 FROM {lookup} WHERE {already_in_lookup})".format( + "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {}".format( quote_identifier(table), quote_identifier(self.name), lookup_columns=", ".join( quote_identifier(c) for c in lookup_columns ), table_cols=", ".join(quote_identifier(c) for c in columns), - all_null=all_columns_are_null, - lookup=quote_identifier(table), - already_in_lookup=already_in_lookup, ) ) @@ -3013,7 +2449,7 @@ class Table(Queryable): # And populate it self.db.execute( - "UPDATE {} SET {} = (SELECT id FROM {} WHERE {where}) WHERE NOT ({all_null})".format( + "UPDATE {} SET {} = (SELECT id FROM {} WHERE {where})".format( quote_identifier(self.name), quote_identifier(magic_lookup_column), quote_identifier(table), @@ -3026,7 +2462,6 @@ class Table(Queryable): ) for column in columns ), - all_null=all_columns_are_null, ) ) # Figure out the right column order @@ -3126,22 +2561,6 @@ class Table(Queryable): self.db.analyze(created_index_name) return self - def drop_index(self, index_name: str, ignore: bool = False): - """ - Drop an index on this table. - - :param index_name: Name of the index to drop - :param ignore: Set to ``True`` to ignore the error if the index does not exist - """ - if index_name not in {index.name for index in self.indexes}: - if ignore: - return self - raise OperationalError( - "No index named {} on table {}".format(index_name, self.name) - ) - self.db.execute("DROP INDEX {}".format(quote_identifier(index_name))) - return self - def add_column( self, col_name: str, @@ -3166,15 +2585,11 @@ class Table(Queryable): raise AlterError("table '{}' does not exist".format(fk)) # if fk_col specified, must be a valid column if fk_col is not None: - fk_col = resolve_casing(fk_col, self.db[fk].columns_dict) if fk_col not in self.db[fk].columns_dict: raise AlterError("table '{}' has no column {}".format(fk, fk_col)) else: # automatically set fk_col to first primary_key of fk table - pks = sorted( - (c for c in self.db[fk].columns if c.is_pk), - key=lambda c: c.is_pk, - ) + pks = [c for c in self.db[fk].columns if c.is_pk] if pks: fk_col = pks[0].name fk_col_type = pks[0].type @@ -3254,106 +2669,52 @@ class Table(Queryable): def add_foreign_key( self, - column: ForeignKeyColumns, + column: str, other_table: Optional[str] = None, - other_column: Optional[ForeignKeyColumns] = None, + other_column: Optional[str] = None, ignore: bool = False, - on_delete: str = "NO ACTION", - on_update: str = "NO ACTION", ): """ Alter the schema to mark the specified column as a foreign key to another table. - :param column: The column to mark as a foreign key - use a tuple of columns - for a compound foreign key. + :param column: The column to mark as a foreign key. :param other_table: The table it refers to - if omitted, will be guessed based on the column name. :param other_column: The column on the other table it - if omitted, will be guessed. - Use a tuple of columns for a compound foreign key. :param ignore: Set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. - :param on_delete: ``ON DELETE`` action for the foreign key, e.g. ``"CASCADE"`` - or ``"SET NULL"``. - :param on_update: ``ON UPDATE`` action for the foreign key. """ - columns = (column,) if isinstance(column, str) else tuple(column) - columns = tuple(resolve_casing(c, self.columns_dict) for c in columns) - # Ensure columns exist - for col in columns: - if col not in self.columns_dict: - raise AlterError("No such column: {}".format(col)) + # Ensure column exists + if column not in self.columns_dict: + raise AlterError("No such column: {}".format(column)) # If other_table is not specified, attempt to guess it from the column if other_table is None: - if len(columns) > 1: - raise ValueError( - "other_table must be specified for a compound foreign key" - ) - other_table = self.guess_foreign_table(columns[0]) + other_table = self.guess_foreign_table(column) # If other_column is not specified, detect the primary key on other_table if other_column is None: - if len(columns) > 1: - other_columns = tuple(self.db.table(other_table).pks) - else: - other_columns = (self.guess_foreign_column(other_table),) - elif isinstance(other_column, str): - other_columns = (other_column,) - else: - other_columns = tuple(other_column) - other_columns = tuple( - resolve_casing(c, self.db[other_table].columns_dict) for c in other_columns - ) - if len(columns) != len(other_columns): - raise ValueError( - "Compound foreign key must have the same number of columns " - "on both sides" - ) + other_column = self.guess_foreign_column(other_table) - # Soundness check that the other columns exist - for other_col in other_columns: - if ( - not [c for c in self.db[other_table].columns if c.name == other_col] - and other_col != "rowid" - ): - raise AlterError("No such column: {}.{}".format(other_table, other_col)) + # Soundness check that the other column exists + if ( + not [c for c in self.db[other_table].columns if c.name == other_column] + and other_column != "rowid" + ): + raise AlterError("No such column: {}.{}".format(other_table, other_column)) # Check we do not already have an existing foreign key if any( fk for fk in self.foreign_keys - if tuple(fold_identifier_case(c) for c in fk.columns) - == tuple(fold_identifier_case(c) for c in columns) - and fold_identifier_case(fk.other_table) - == fold_identifier_case(other_table) - and tuple(fold_identifier_case(c) for c in fk.other_columns) - == tuple(fold_identifier_case(c) for c in other_columns) + if fk.column == column + and fk.other_table == other_table + and fk.other_column == other_column ): if ignore: return self else: raise AlterError( "Foreign key already exists for {} => {}.{}".format( - ", ".join(columns), other_table, ", ".join(other_columns) + column, other_table, other_column ) ) - if len(columns) == 1: - fk_object = ForeignKey( - self.name, - columns[0], - other_table, - other_columns[0], - on_delete=on_delete, - on_update=on_update, - ) - else: - fk_object = ForeignKey( - self.name, - None, - other_table, - None, - columns=columns, - other_columns=other_columns, - is_compound=True, - on_delete=on_delete, - on_update=on_update, - ) - self.db.add_foreign_keys([fk_object]) + self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) return self def enable_counts(self) -> None: @@ -3729,7 +3090,7 @@ class Table(Queryable): ), args, ) - columns = dedupe_keys(c[0] for c in cursor.description) + columns = [c[0] for c in cursor.description] for row in cursor: yield dict(zip(columns, row)) @@ -3871,7 +3232,6 @@ class Table(Queryable): """ if isinstance(columns, str): columns = [columns] - columns = [resolve_casing(c, self.columns_dict) for c in columns] if multi: return self._convert_multi( @@ -3886,7 +3246,6 @@ class Table(Queryable): if output is not None: if len(columns) != 1: raise ValueError("output= can only be used with a single column") - output = resolve_casing(output, self.columns_dict) if output not in self.columns_dict: self.add_column(output, output_type or "text") @@ -3927,7 +3286,9 @@ class Table(Queryable): # First we execute the function pk_to_values = {} new_column_types: Dict[str, Set[type]] = {} - pks = self.pks + pks = [column.name for column in self.columns if column.is_pk] + if not pks: + pks = ["rowid"] with progressbar( length=self.count, silent=not show_progress, label="1: Evaluating" @@ -4032,7 +3393,7 @@ class Table(Queryable): # Only process extracts if there are any if has_extracts: for i, key in enumerate(all_columns): - if key in extracts and record_values[i] is not None: + if key in extracts: record_values[i] = self.db.table(extracts[key]).lookup( {"value": record_values[i]} ) @@ -4052,7 +3413,7 @@ class Table(Queryable): ), ) ) - if key in extracts and value is not None: + if key in extracts: extract_table = extracts[key] value = self.db.table(extract_table).lookup({"value": value}) record_values.append(value) @@ -4084,8 +3445,6 @@ class Table(Queryable): # Everything from here on is for upsert=True pk_cols = [pk] if isinstance(pk, str) else list(pk) - # The records may use different casing for the pk columns than pk= - pk_cols = [resolve_casing(c, all_columns) for c in pk_cols] # Every record must provide a value for every primary key column - a # NULL primary key never matches ON CONFLICT, so the record would be # inserted as a brand new row instead of upserted @@ -4136,7 +3495,10 @@ class Table(Queryable): # At this point we need compatibility UPSERT for SQLite < 3.24.0 # (INSERT OR IGNORE + second UPDATE stage) queries_and_params = [] - pks = pk_cols + if isinstance(pk, str): + pks = [pk] + else: + pks = pk self.last_pk = None for record_values in values: record = dict(zip(all_columns, record_values)) @@ -4403,35 +3765,6 @@ class Table(Queryable): if hash_id: pk = hash_id - # pk columns missing from an existing table are an error - unless - # alter=True, where a pk column supplied by the records will be - # added, so validation waits until the record keys are known - deferred_invalid_pk_check = None - if pk and not hash_id and self.exists(): - pk_cols = [pk] if isinstance(pk, str) else list(pk) - existing_columns = self.columns_dict - # rowid and its aliases are valid primary keys for a rowid table - # even though they are not listed among the table's columns - rowid_aliases = ROWID_ALIASES if self.use_rowid else frozenset() - missing_pk_cols = [ - col - for col in pk_cols - if col.lower() not in rowid_aliases - and resolve_casing(col, existing_columns) not in existing_columns - ] - if missing_pk_cols: - invalid_pk_error = InvalidColumns( - "Invalid primary key column{} {} for table {} with columns {}".format( - "s" if len(missing_pk_cols) > 1 else "", - missing_pk_cols, - self.name, - list(existing_columns), - ) - ) - if not alter: - raise invalid_pk_error - deferred_invalid_pk_check = (missing_pk_cols, invalid_pk_error) - if ignore and replace: raise ValueError("Use either ignore=True or replace=True, not both") all_columns = [] @@ -4541,16 +3874,6 @@ class Table(Queryable): all_columns = list(sorted(all_columns_set)) if hash_id: all_columns.insert(0, hash_id) - if deferred_invalid_pk_check is not None: - # alter=True - pk columns the table lacks are valid if - # the records supply them, otherwise raise the error - missing_pk_cols, invalid_pk_error = deferred_invalid_pk_check - record_columns = {column: True for column in all_columns} - if any( - resolve_casing(col, record_columns) not in record_columns - for col in missing_pk_cols - ): - raise invalid_pk_error else: if not list_mode: for record in cast(List[Dict[str, Any]], chunk): @@ -4581,86 +3904,18 @@ class Table(Queryable): if num_records_processed == 1: # For an insert we need to use result.lastrowid if not upsert and result is not None: - ignored_insert = ignore and result.rowcount == 0 - if ignored_insert: - # The row was not inserted because it conflicts with an - # existing row. Point last_pk / last_rowid at that existing - # row when we can identify it from the record's primary key - # values, rather than leaving them stale or unset. - if list_mode: - first_record_dict = dict( - zip(column_names, cast(Sequence[Any], first_record)) - ) - else: - first_record_dict = cast(Dict[str, Any], first_record) + self.last_rowid = result.lastrowid + if (hash_id or pk) and self.last_rowid: + # Set self.last_pk to the pk(s) for that rowid + row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] if hash_id: - self.last_pk = hash_record(first_record_dict, hash_id_columns) + self.last_pk = row[hash_id] elif isinstance(pk, str): - self.last_pk = first_record_dict[ - resolve_casing(pk, first_record_dict) - ] - elif pk: - self.last_pk = tuple( - first_record_dict[resolve_casing(p, first_record_dict)] - for p in pk - ) - # Locate the existing conflicting row using its primary key - # columns so we can report its rowid (and pk if not already - # known). Falls back to leaving them unset if the conflict - # cannot be resolved to a pk lookup (e.g. a UNIQUE column). - key_cols: Optional[List[str]] = None - if isinstance(pk, str): - key_cols = [pk] - elif pk: - key_cols = list(pk) - elif not hash_id and not self.use_rowid: - key_cols = self.pks - if key_cols: - try: - key_values = [ - first_record_dict[resolve_casing(c, first_record_dict)] - for c in key_cols - ] - except KeyError: - key_values = None - if key_values is not None: - where = " and ".join( - "{} = ?".format(quote_identifier(c)) for c in key_cols - ) - existing = self.db.execute( - "select rowid from {} where {} limit 1".format( - quote_identifier(self.name), where - ), - key_values, - ).fetchone() - if existing is not None: - self.last_rowid = existing[0] - # On a primary key conflict the record's pk - # values identify the existing row - if self.last_pk is None: - self.last_pk = ( - key_values[0] - if len(key_cols) == 1 - else tuple(key_values) - ) - else: - self.last_rowid = result.lastrowid - # A rowid-alias pk resolves directly to the rowid, so there - # is no separate pk column to look up - rowid_pk = isinstance(pk, str) and pk.lower() in ROWID_ALIASES - if (hash_id or (pk and not rowid_pk)) and self.last_rowid: - # Set self.last_pk to the pk(s) for that rowid - row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] - if hash_id: - self.last_pk = row[hash_id] - elif isinstance(pk, str): - self.last_pk = row[resolve_casing(pk, row)] - else: - self.last_pk = tuple( - row[resolve_casing(p, row)] for p in pk - ) + self.last_pk = row[pk] else: - self.last_pk = self.last_rowid + self.last_pk = tuple(row[p] for p in pk) + else: + self.last_pk = self.last_rowid else: # For an upsert use first_record from earlier if list_mode: @@ -4670,14 +3925,11 @@ class Table(Queryable): # hash_id not supported in list mode for last_pk pass elif isinstance(pk, str): - pk_index = column_names.index(resolve_casing(pk, column_names)) + pk_index = column_names.index(pk) self.last_pk = first_record_list[pk_index] else: self.last_pk = tuple( - first_record_list[ - column_names.index(resolve_casing(p, column_names)) - ] - for p in pk + first_record_list[column_names.index(p)] for p in pk ) else: first_record_dict = cast(Dict[str, Any], first_record) @@ -4685,12 +3937,9 @@ class Table(Queryable): self.last_pk = hash_record(first_record_dict, hash_id_columns) else: self.last_pk = ( - first_record_dict[resolve_casing(pk, first_record_dict)] + first_record_dict[pk] if isinstance(pk, str) - else tuple( - first_record_dict[resolve_casing(p, first_record_dict)] - for p in pk - ) + else tuple(first_record_dict[p] for p in pk) ) if analyze: @@ -4835,16 +4084,11 @@ class Table(Queryable): combined_values.update(extra_values) if self.exists(): self.add_missing_columns([combined_values]) - unique_column_sets = [ - {fold_identifier_case(c) for c in i.columns} for i in self.indexes - ] - if { - fold_identifier_case(c) for c in lookup_values - } not in unique_column_sets: + unique_column_sets = [set(i.columns) for i in self.indexes] + if set(lookup_values.keys()) not in unique_column_sets: self.create_index(lookup_values.keys(), unique=True) - # IS rather than = so that null values are matched correctly wheres = [ - "{} IS ?".format(quote_identifier(column)) for column in lookup_values + "{} = ?".format(quote_identifier(column)) for column in lookup_values ] rows = list( self.rows_where( @@ -4852,7 +4096,7 @@ class Table(Queryable): ) ) try: - return rows[0][resolve_casing(pk, rows[0])] + return rows[0][pk] except IndexError: return self.insert( combined_values, diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 00d0fa5..36e053d 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -96,34 +96,14 @@ class Migrations: changes are rolled back, no record is written and the migration stays pending. Migrations registered with ``transactional=False`` run outside of a transaction. - - :raises ValueError: if a ``stop_before`` name matches a migration in - this set that has already been applied - stopping before it is - impossible to honor, and no pending migrations are applied """ + self.ensure_migrations_table(db) if stop_before is None: stop_before_names = set() elif isinstance(stop_before, str): stop_before_names = {stop_before} else: stop_before_names = set(stop_before) - # A stop_before naming an already-applied migration cannot be - # honored - error rather than applying everything after it. Names - # not in this set at all are ignored, because unqualified CLI - # values are offered to every migration set - already_applied = stop_before_names.intersection( - migration.name for migration in self.applied(db) - ) - if already_applied: - raise ValueError( - "Cannot stop before migration{} {} in set '{}' - already " - "been applied".format( - "s" if len(already_applied) > 1 else "", - ", ".join(sorted(already_applied)), - self.name, - ) - ) - self.ensure_migrations_table(db) for migration in self.pending(db): name = migration.name if name in stop_before_names: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index b39b117..0ca98fc 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -43,10 +43,15 @@ else: dbapi2 = importlib.import_module("pysqlite3.dbapi2") OperationalError = dbapi2.OperationalError except ImportError: - import sqlite3 # noqa: F401 - from sqlite3 import dbapi2 # noqa: F401 + try: + sqlite3 = importlib.import_module("sqlean") + dbapi2 = importlib.import_module("sqlean.dbapi2") + OperationalError = dbapi2.OperationalError + except ImportError: + import sqlite3 # noqa: F401 + from sqlite3 import dbapi2 # noqa: F401 - OperationalError = dbapi2.OperationalError + OperationalError = dbapi2.OperationalError SPATIALITE_PATHS = ( @@ -613,37 +618,6 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> ).hexdigest() -def dedupe_keys(keys: Iterable[str]) -> List[str]: - """ - Rename duplicates in a list of column names so every name is unique, - by appending ``_2``, ``_3``... to later occurrences - skipping any - suffix that would collide with another column in the list. - - Used when converting SQL query rows to dictionaries, where duplicate - column names would otherwise silently overwrite each other. - - :param keys: List of column names, possibly containing duplicates - """ - keys = list(keys) - taken = set(keys) - if len(taken) == len(keys): - # No duplicates - the common case - return keys - seen: set = set() - result = [] - for key in keys: - if key in seen: - new_key = key - suffix = 2 - while new_key in seen or new_key in taken: - new_key = "{}_{}".format(key, suffix) - suffix += 1 - key = new_key - seen.add(key) - result.append(key) - return result - - def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]: for key, value in d.items(): if isinstance(value, dict): diff --git a/tests/test_atomic.py b/tests/test_atomic.py index c3fd02f..a8e3563 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -247,81 +247,6 @@ def test_execute_write_respects_explicit_transaction(fresh_db): assert [r["id"] for r in fresh_db["t"].rows] == [1] -def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): - # A BEGIN hidden behind a leading comment must not be auto-committed - # out from under the caller - fresh_db["t"].insert({"id": 1}, pk="id") - fresh_db.execute("-- start a transaction\nbegin") - assert fresh_db.conn.in_transaction - fresh_db.execute("insert into t (id) values (2)") - fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] - - -def _sqlite_accepts_bom(): - try: - sqlite3.connect(":memory:").execute("\ufeffselect 1") - return True - except sqlite3.OperationalError: - return False - - -@pytest.mark.parametrize("begin_sql", ["; begin", "\ufeffbegin"]) -def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql): - # sqlite3 tolerates empty statements and a UTF-8 BOM before the first - # real token, so a BEGIN behind either must not be auto-committed - # out from under the caller - if begin_sql.startswith("\ufeff") and not _sqlite_accepts_bom(): - pytest.skip("This SQLite version rejects a leading byte order mark") - fresh_db["t"].insert({"id": 1}, pk="id") - fresh_db.execute(begin_sql) - assert fresh_db.conn.in_transaction - fresh_db.execute("insert into t (id) values (2)") - fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] - - -def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): - # A failed write must not leave the driver's implicit transaction open - - # that would silently disable auto-commit for every subsequent write - path = str(tmpdir / "test.db") - db = Database(path) - db["t"].insert({"id": 1}, pk="id") - with pytest.raises(sqlite3.IntegrityError): - db.execute("insert into t (id) values (1)") - assert not db.conn.in_transaction - # Subsequent writes commit as normal and survive closing the connection - db["other"].insert({"id": 2}) - db.close() - db2 = Database(path) - assert db2["other"].exists() - db2.close() - - -def test_execute_failed_write_preserves_explicit_transaction(fresh_db): - # A failed write inside an explicit transaction must not roll back - # the caller's earlier work - only the caller decides that - fresh_db["t"].insert({"id": 1}, pk="id") - fresh_db.begin() - fresh_db.execute("insert into t (id) values (2)") - with pytest.raises(sqlite3.IntegrityError): - fresh_db.execute("insert into t (id) values (1)") - assert fresh_db.conn.in_transaction - fresh_db.commit() - assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] - - -def test_execute_failed_write_inside_atomic_preserves_block(fresh_db): - # A caught failure inside an atomic() block must leave the block's - # transaction open so its other work still commits - fresh_db["t"].insert({"id": 1}, pk="id") - with fresh_db.atomic(): - fresh_db.execute("insert into t (id) values (2)") - with pytest.raises(sqlite3.IntegrityError): - fresh_db.execute("insert into t (id) values (1)") - assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] - - def test_query_returning_commits_after_iteration(tmpdir): if sqlite3.sqlite_version_info < (3, 35, 0): import pytest as _pytest @@ -337,46 +262,3 @@ def test_query_returning_commits_after_iteration(tmpdir): assert other.execute("select count(*) from t").fetchone()[0] == 2 other.close() db.close() - - -TRIGGER_SQL = """ -create trigger no_bad before insert on t -when new.v = 'bad' -begin - select raise(rollback, 'trigger says no'); -end -""" - - -def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db): - # RAISE(ROLLBACK) rolls back the whole transaction and destroys every - # savepoint - atomic()'s cleanup must not mask the IntegrityError - # with "cannot rollback - no transaction is active" - fresh_db.execute("create table t (id integer primary key, v text)") - fresh_db.execute(TRIGGER_SQL) - with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): - with fresh_db.atomic(): - fresh_db.execute("insert into t (v) values ('bad')") - assert not fresh_db.conn.in_transaction - - -def test_nested_atomic_preserves_error_from_transaction_destroying_trigger( - fresh_db, -): - # The nested savepoint branch previously raised - # "no such savepoint" from ROLLBACK TO SAVEPOINT - fresh_db.execute("create table t (id integer primary key, v text)") - fresh_db.execute(TRIGGER_SQL) - with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): - with fresh_db.atomic(): - with fresh_db.atomic(): - fresh_db.execute("insert into t (v) values ('bad')") - assert not fresh_db.conn.in_transaction - - -def test_atomic_preserves_error_from_insert_or_rollback(fresh_db): - fresh_db["t"].insert({"id": 1}, pk="id") - with pytest.raises(sqlite3.IntegrityError): - with fresh_db.atomic(): - fresh_db.execute("insert or rollback into t (id) values (1)") - assert not fresh_db.conn.in_transaction diff --git a/tests/test_cli.py b/tests/test_cli.py index a2135b0..d26e4dd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,7 +3,6 @@ from sqlite_utils.db import Index, ForeignKey from click.testing import CliRunner from pathlib import Path import subprocess -import sqlite3 import sys import json import os @@ -196,50 +195,6 @@ def test_output_table(db_path, options, expected): assert expected == result.output.strip() -@pytest.mark.parametrize( - "fmt_option", [["--fmt", "simple"], ["-t"], ["--fmt", "github"]] -) -def test_output_table_no_headers(db_path, fmt_option): - # --no-headers should omit the header row from --fmt/--table output too, not - # just from --csv/--tsv (#566). Previously the flag was silently ignored for - # tabulate formats and the column names were always printed. - db = Database(db_path) - with db.conn: - db["dogs"].insert_all( - [ - {"id": 1, "name": "Cleo", "age": 4}, - {"id": 2, "name": "Pancakes", "age": 2}, - ] - ) - sql = "select id, name, age from dogs order by id" - - with_headers = CliRunner().invoke(cli.cli, ["query", db_path, sql] + fmt_option) - without_headers = CliRunner().invoke( - cli.cli, ["query", db_path, sql] + fmt_option + ["--no-headers"] - ) - assert with_headers.exit_code == 0 - assert without_headers.exit_code == 0 - - # The column names appear when headers are shown, and must not appear at all - # once --no-headers is passed. - assert "name" in with_headers.output - for header in ("id", "name", "age"): - assert ( - header not in without_headers.output - ), f"header {header!r} leaked into --no-headers output" - # The data is still all present. - for value in ("Cleo", "Pancakes", "1", "2", "4"): - assert value in without_headers.output - - # The rows command shares the same code path. - rows_no_headers = CliRunner().invoke( - cli.cli, ["rows", db_path, "dogs"] + fmt_option + ["--no-headers"] - ) - assert rows_no_headers.exit_code == 0 - assert "name" not in rows_no_headers.output - assert "Cleo" in rows_no_headers.output - - def test_create_index(db_path): db = Database(db_path) assert [] == db["Gosh"].indexes @@ -292,24 +247,6 @@ def test_create_index(db_path): ) -def test_drop_index(db_path): - db = Database(db_path) - db["Gosh"].create_index(["c1"]) - assert [index.name for index in db["Gosh"].indexes] == ["idx_Gosh_c1"] - result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) - assert result.exit_code == 0 - assert db["Gosh"].indexes == [] - - result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) - assert result.exit_code == 1 - assert "No index named idx_Gosh_c1" in result.output - - result = CliRunner().invoke( - cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1", "--ignore"] - ) - assert result.exit_code == 0 - - def test_create_index_analyze(db_path): db = Database(db_path) assert "sqlite_stat1" not in db.table_names() @@ -801,25 +738,6 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() -def test_query_sql_from_stdin(db_path): - # https://github.com/simonw/sqlite-utils/issues/765 - db = Database(db_path) - with db.conn: - db["dogs"].insert_all( - [ - {"id": 1, "age": 4, "name": "Cleo"}, - {"id": 2, "age": 2, "name": "Pancakes"}, - ] - ) - result = CliRunner().invoke( - cli.cli, - ["query", db_path, "-"], - input="select name from dogs order by name", - ) - assert result.exit_code == 0, result.output - assert json.loads(result.output) == [{"name": "Cleo"}, {"name": "Pancakes"}] - - def test_query_json_empty(db_path): result = CliRunner().invoke( cli.cli, @@ -828,26 +746,6 @@ def test_query_json_empty(db_path): assert result.output.strip() == "[]" -def test_query_json_duplicate_columns_are_deduped(db_path): - # https://github.com/simonw/sqlite-utils/issues/624 - result = CliRunner().invoke( - cli.cli, - [db_path, "select 1 as id, 2 as id, 'x' as value, 'y' as value"], - ) - assert result.output.strip() == ( - '[{"id": 1, "id_2": 2, "value": "x", "value_2": "y"}]' - ) - - -def test_query_csv_duplicate_columns_are_preserved(db_path): - # CSV output should keep the duplicate headers, not rename them - result = CliRunner().invoke( - cli.cli, - [db_path, "select 1 as id, 2 as id", "--csv"], - ) - assert result.output.replace("\r", "").strip() == "id,id\n1,2" - - def test_query_invalid_function(db_path): result = CliRunner().invoke( cli.cli, [db_path, "select bad()", "--functions", "def invalid_python"] @@ -1085,34 +983,6 @@ def test_query_json_with_json_cols(db_path): assert expected == result_rows.output.strip() -def test_query_json_unicode_not_escaped_by_default(db_path): - db = Database(db_path) - with db.conn: - db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") - result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text"]) - assert result.exit_code == 0 - assert result.output.strip() == '[{"id": 1, "text": "Japanese 日本語"}]' - # Same for --nl - result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text", "--nl"]) - assert result.exit_code == 0 - assert result.output.strip() == '{"id": 1, "text": "Japanese 日本語"}' - - -@pytest.mark.parametrize("command", ["query", "rows"]) -def test_query_json_ascii_option(db_path, command): - db = Database(db_path) - with db.conn: - db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") - if command == "query": - args = [db_path, "select id, text from text", "--ascii"] - else: - args = ["rows", db_path, "text", "--ascii"] - result = CliRunner().invoke(cli.cli, args) - assert result.exit_code == 0 - expected = '[{"id": 1, "text": "Japanese ' + "\\u65e5\\u672c\\u8a9e" + '"}]' - assert result.output.strip() == expected - - @pytest.mark.parametrize( "content,is_binary", [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], @@ -1254,38 +1124,20 @@ def test_upsert(db_path, tmpdir): ] -def test_upsert_pk_inferred_from_existing_table(db_path, tmpdir): +def test_upsert_pk_required(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") - db = Database(db_path) insert_dogs = [ {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Nixie", "age": 4}, ] write_json(json_path, insert_dogs) - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "dogs", json_path, "--pk", "id"], - catch_exceptions=False, - ) - assert result.exit_code == 0, result.output - - write_json( - json_path, - [ - {"id": 1, "age": 5}, - {"id": 2, "age": 5}, - ], - ) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path], catch_exceptions=False, ) - assert result.exit_code == 0, result.output - assert list(db.query("select * from dogs order by id")) == [ - {"id": 1, "name": "Cleo", "age": 5}, - {"id": 2, "name": "Nixie", "age": 5}, - ] + assert result.exit_code == 2 + assert "Error: Missing option '--pk'" in result.output def test_upsert_analyze(db_path, tmpdir): @@ -1940,64 +1792,6 @@ def test_transform_sql(db_path): assert db["dogs"].schema == original_schema -@pytest.mark.parametrize( - "initial_strict,args,expected_strict", - ( - (False, [], False), - (True, [], True), - (False, ["--strict"], True), - (True, ["--no-strict"], False), - ), -) -def test_transform_strict_option(db_path, initial_strict, args, expected_strict): - db = Database(db_path) - if not db.supports_strict: - pytest.skip("SQLite version does not support strict tables") - db["dogs"].create({"id": int}, strict=initial_strict) - - result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs"] + args) - - assert result.exit_code == 0, result.output - assert db["dogs"].strict is expected_strict - - -@pytest.mark.parametrize( - "initial_strict,flag,sql_is_strict", - ( - (False, "--strict", True), - (True, "--no-strict", False), - ), -) -def test_transform_strict_option_sql(db_path, initial_strict, flag, sql_is_strict): - db = Database(db_path) - if not db.supports_strict: - pytest.skip("SQLite version does not support strict tables") - db["dogs"].create({"id": int}, strict=initial_strict) - - result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", flag, "--sql"]) - - assert result.exit_code == 0, result.output - assert (") STRICT;" in result.output) is sql_is_strict - assert db["dogs"].strict is initial_strict - - -def test_transform_strict_option_with_invalid_data(db_path): - db = Database(db_path) - if not db.supports_strict: - pytest.skip("SQLite version does not support strict tables") - dogs = db["dogs"] - dogs.create({"id": int}) - dogs.insert({"id": "not-an-integer"}) - - result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", "--strict"]) - - assert result.exit_code == 1 - assert isinstance(result.exception, sqlite3.IntegrityError) - assert dogs.strict is False - assert list(dogs.rows) == [{"id": "not-an-integer"}] - assert not any(name.startswith("dogs_new_") for name in db.table_names()) - - @pytest.mark.parametrize( "extra_args,expected_schema", ( @@ -2872,22 +2666,3 @@ def test_insert_upsert_strict(tmpdir, method, strict): assert result.exit_code == 0 db = Database(db_path) assert db["items"].strict == strict or not db.supports_strict - - -def test_extract_bad_column_clean_error(db_path): - db = Database(db_path) - db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") - result = CliRunner().invoke(cli.cli, ["extract", db_path, "trees", "nope"]) - assert result.exit_code == 1 - assert result.exception is None or isinstance(result.exception, SystemExit) - assert result.output.startswith("Error: Invalid columns") - - -def test_extract_view_clean_error(db_path): - db = Database(db_path) - db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") - db.create_view("v", "select * from trees") - result = CliRunner().invoke(cli.cli, ["extract", db_path, "v", "species"]) - assert result.exit_code == 1 - assert result.exception is None or isinstance(result.exception, SystemExit) - assert result.output.startswith("Error:") diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 6c3f5c5..443e72c 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -215,25 +215,6 @@ def test_convert_multi_dryrun(test_db_and_path): ) -def test_convert_multi_dryrun_unicode_not_escaped(test_db_and_path): - db_path = test_db_and_path[1] - result = CliRunner().invoke( - cli.cli, - [ - "convert", - db_path, - "example", - "dt", - "{'text': 'Japanese 日本語'}", - "--dry-run", - "--multi", - ], - ) - assert result.exit_code == 0 - # Preview should match what jsonify_if_needed() would actually store - assert '{"text": "Japanese 日本語"}' in result.output - - @pytest.mark.parametrize("drop", (True, False)) def test_convert_output_column(test_db_and_path, drop): db, db_path = test_db_and_path diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index df6f80c..2df1e0c 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -628,263 +628,3 @@ def test_insert_into_view_errors(tmpdir): ) assert result.exit_code == 1 assert result.output.strip() == "Error: Table v is actually a view" - - -def test_insert_csv_detect_types_leaves_existing_table_alone(db_path): - # Type detection is the default for CSV/TSV inserts, but it must only - # apply to tables created by this command - transforming a pre-existing - # table would rewrite its column types and corrupt data such as - # TEXT zip codes with leading zeros - db = Database(db_path) - db["places"].insert({"name": "Boston", "zip": "01234"}) - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "places", "-", "--csv"], - catch_exceptions=False, - input="name,zip\nSF,94107", - ) - assert result.exit_code == 0, result.output - assert db["places"].columns_dict["zip"] is str - assert list(db["places"].rows) == [ - {"name": "Boston", "zip": "01234"}, - {"name": "SF", "zip": "94107"}, - ] - - -def test_insert_csv_detect_types_new_table(db_path): - # A table created by the insert still gets detected types - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "data", "-", "--csv"], - catch_exceptions=False, - input="name,age,weight\nCleo,5,12.5", - ) - assert result.exit_code == 0, result.output - db = Database(db_path) - assert db["data"].columns_dict == {"name": str, "age": int, "weight": float} - - -@pytest.mark.parametrize( - "command,extra_args,input_text,expected_row", - ( - ( - "insert", - [], - "zipcode,score\n01234,9.5\n", - {"zipcode": "01234", "score": 9.5}, - ), - ( - "upsert", - ["--pk", "id"], - "id,zipcode,score\n1,01234,9.5\n", - {"id": 1, "zipcode": "01234", "score": 9.5}, - ), - ), -) -def test_insert_upsert_csv_type_overrides_detected_types( - db_path, command, extra_args, input_text, expected_row -): - result = CliRunner().invoke( - cli.cli, - [ - command, - db_path, - "places", - "-", - "--csv", - ] - + extra_args - + [ - "--type", - "zipcode", - "text", - ], - catch_exceptions=False, - input=input_text, - ) - assert result.exit_code == 0, result.output - db = Database(db_path) - expected_columns = {"zipcode": str, "score": float} - if command == "upsert": - expected_columns = {"id": int, **expected_columns} - assert db["places"].columns_dict == expected_columns - assert list(db["places"].rows) == [expected_row] - - -def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): - db = Database(db_path) - db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id") - result = CliRunner().invoke( - cli.cli, - ["upsert", db_path, "places", "-", "--csv", "--pk", "id"], - catch_exceptions=False, - input="id,name,zip\n2,SF,94107", - ) - assert result.exit_code == 0, result.output - assert db["places"].columns_dict["zip"] is str - assert db["places"].get(1)["zip"] == "01234" - - -def test_insert_invalid_pk_clean_error(db_path): - # An invalid --pk against an existing table should be a clean CLI - # error, not a raw InvalidColumns traceback - db = Database(db_path) - db["t"].insert({"a": 1}) - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "t", "-", "--pk", "badcol"], - input='{"a": 2}', - ) - assert result.exit_code == 1 - assert result.exception is None or isinstance(result.exception, SystemExit) - assert result.output.startswith("Error: Invalid primary key column") - - -# --code tests, see https://github.com/simonw/sqlite-utils/issues/684 -CODE_ROWS_FUNCTION = """ -def rows(): - yield {"id": 1, "name": "Cleo"} - yield {"id": 2, "name": "Suna"} -""" - -CODE_ROWS_ITERABLE = """ -rows = [ - {"id": 1, "name": "Cleo"}, - {"id": 2, "name": "Suna"}, -] -""" - - -@pytest.mark.parametrize("code", (CODE_ROWS_FUNCTION, CODE_ROWS_ITERABLE)) -def test_insert_code(tmpdir, code): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "creatures", "--code", code, "--pk", "id"], - ) - assert result.exit_code == 0, result.output - db = Database(db_path) - assert db["creatures"].pks == ["id"] - assert list(db["creatures"].rows) == [ - {"id": 1, "name": "Cleo"}, - {"id": 2, "name": "Suna"}, - ] - - -def test_insert_code_from_file(tmpdir): - db_path = str(tmpdir / "dogs.db") - code_path = str(tmpdir / "gen.py") - with open(code_path, "w") as fp: - fp.write(CODE_ROWS_FUNCTION) - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "creatures", "--code", code_path], - ) - assert result.exit_code == 0, result.output - assert list(Database(db_path)["creatures"].rows) == [ - {"id": 1, "name": "Cleo"}, - {"id": 2, "name": "Suna"}, - ] - - -def test_upsert_code(tmpdir): - db_path = str(tmpdir / "dogs.db") - db = Database(db_path) - db["creatures"].insert_all( - [{"id": 1, "name": "old"}, {"id": 2, "name": "Suna"}], pk="id" - ) - result = CliRunner().invoke( - cli.cli, - ["upsert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--pk", "id"], - ) - assert result.exit_code == 0, result.output - assert list(db["creatures"].rows) == [ - {"id": 1, "name": "Cleo"}, - {"id": 2, "name": "Suna"}, - ] - - -def test_insert_code_requires_file_or_code(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke(cli.cli, ["insert", db_path, "creatures"]) - assert result.exit_code == 1 - assert "Provide either a FILE argument or --code" in result.output - - -def test_insert_code_mutually_exclusive_with_file(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "creatures", "-", "--code", CODE_ROWS_FUNCTION], - input="{}", - ) - assert result.exit_code == 1 - assert "--code cannot be used with a FILE argument" in result.output - - -def test_insert_code_rejects_input_format_options(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--csv"], - ) - assert result.exit_code == 1 - assert "--code cannot be used with input format options" in result.output - - -def test_insert_code_missing_rows(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "creatures", "--code", "x = 1"], - ) - assert result.exit_code == 1 - assert "must define a 'rows' function or iterable" in result.output - - -def test_insert_code_single_dict(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - [ - "insert", - db_path, - "creatures", - "--code", - 'rows = {"id": 1, "name": "Cleo"}', - "--pk", - "id", - ], - ) - assert result.exit_code == 0, result.output - assert list(Database(db_path)["creatures"].rows) == [{"id": 1, "name": "Cleo"}] - - -def test_insert_code_not_iterable(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "creatures", "--code", "rows = 5"], - ) - assert result.exit_code == 1 - assert "must define a 'rows' function or iterable" in result.output - - -def test_insert_code_syntax_error(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "creatures", "--code", "def rows(:"], - ) - assert result.exit_code == 1 - assert "Error in --code" in result.output - - -def test_insert_code_file_not_found(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "creatures", "--code", "missing.py"], - ) - assert result.exit_code == 1 - assert "File not found: missing.py" in result.output diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 0f29e36..1cdf8e7 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -463,45 +463,3 @@ def test_list_does_not_upgrade_legacy_migrations_table(two_migrations): db2 = sqlite_utils.Database(db_path) assert db2["_sqlite_migrations"].pks == ["migration_set", "name"] db2.close() - - -def test_stop_before_applied_migration_errors(two_migrations): - path, _ = two_migrations - db_path = str(path / "test.db") - migrations_path = str(path / "foo" / "migrations.py") - # Apply everything first - first = CliRunner().invoke( - sqlite_utils.cli.cli, - ["migrate", db_path, migrations_path, "--stop-before", "bar"], - ) - assert first.exit_code == 0 - # foo is now applied - stopping before it is an error, and bar - # must not be applied as a side effect - result = CliRunner().invoke( - sqlite_utils.cli.cli, - ["migrate", db_path, migrations_path, "--stop-before", "foo"], - ) - assert result.exit_code != 0 - assert "already been applied" in result.output - db = sqlite_utils.Database(db_path) - assert not db["bar"].exists() - - -def test_list_with_legacy_class_is_read_only(tmpdir): - # Legacy sqlite-migrate classes create the _sqlite_migrations table - # from their pending()/applied() methods - --list must roll that - # back so it stays a read-only operation as documented - path = pathlib.Path(tmpdir) - (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") - db_path = str(path / "test.db") - db = sqlite_utils.Database(db_path) - db["existing"].insert({"id": 1}) - db.close() - result = CliRunner().invoke( - sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"] - ) - assert result.exit_code == 0, result.output - assert "first" in result.output - db2 = sqlite_utils.Database(db_path) - assert "_sqlite_migrations" not in db2.table_names() - db2.close() diff --git a/tests/test_column_casing.py b/tests/test_column_casing.py deleted file mode 100644 index ce11345..0000000 --- a/tests/test_column_casing.py +++ /dev/null @@ -1,233 +0,0 @@ -""" -SQLite treats column names as case-insensitive. These tests exercise the -places where sqlite-utils performs Python-side lookups of column names -provided by the caller, which should match the schema case-insensitively. - -https://github.com/simonw/sqlite-utils/issues/760 -""" - -import pytest - -from sqlite_utils import Database -from sqlite_utils.db import ForeignKey - - -def test_insert_populates_last_pk_case_insensitively(fresh_db): - books = fresh_db["books"] - books.create({"Id": int, "Title": str}, pk="Id") - books.insert({"Id": 1, "Title": "One"}, pk="id") - assert books.last_pk == 1 - - -def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db): - books = fresh_db["books"] - books.create({"Author": str, "Position": int, "Title": str}) - books.insert( - {"Author": "Sue", "Position": 1, "Title": "One"}, pk=("author", "position") - ) - assert books.last_pk == ("Sue", 1) - - -@pytest.mark.parametrize("use_old_upsert", (False, True)) -def test_upsert_pk_case_differs_from_schema(use_old_upsert): - db = Database(memory=True, use_old_upsert=use_old_upsert) - books = db["books"] - books.create({"Id": int, "Title": str}, pk="Id") - books.insert({"Id": 1, "Title": "One"}) - books.upsert({"id": 1, "title": "Won"}, pk="id") - assert list(books.rows) == [{"Id": 1, "Title": "Won"}] - assert books.last_pk == 1 - - -@pytest.mark.parametrize("use_old_upsert", (False, True)) -def test_upsert_record_key_case_differs_from_pk(use_old_upsert): - # all_columns comes from the record keys, pk= from the caller - db = Database(memory=True, use_old_upsert=use_old_upsert) - books = db["books"] - books.create({"Id": int, "Title": str}, pk="Id") - books.upsert({"ID": 1, "Title": "One"}, pk="id") - assert list(books.rows) == [{"Id": 1, "Title": "One"}] - assert books.last_pk == 1 - - -def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db): - # pk is inferred from the existing schema as "Id", records use "id" - books = fresh_db["books"] - books.create({"Id": int, "Title": str}, pk="Id") - books.upsert({"id": 1, "title": "One"}) - assert list(books.rows) == [{"Id": 1, "Title": "One"}] - assert books.last_pk == 1 - - -def test_upsert_list_mode_pk_case_insensitive(fresh_db): - books = fresh_db["books"] - books.create({"Id": int, "Title": str}, pk="Id") - books.upsert_all([["id", "title"], [1, "One"]], pk="Id") - assert list(books.rows) == [{"Id": 1, "Title": "One"}] - assert books.last_pk == 1 - - -def test_lookup_pk_case_insensitive(fresh_db): - fresh_db["species"].create({"ID": int, "Name": str}, pk="ID") - fresh_db["species"].insert({"ID": 5, "Name": "Palm"}) - fresh_db["species"].create_index(["Name"], unique=True) - assert fresh_db["species"].lookup({"Name": "Palm"}, pk="id") == 5 - - -def test_lookup_does_not_create_redundant_index(fresh_db): - fresh_db["species"].create({"id": int, "Name": str}, pk="id") - fresh_db["species"].create_index(["Name"], unique=True) - fresh_db["species"].lookup({"name": "Palm"}) - assert len(fresh_db["species"].indexes) == 1 - - -def test_create_table_transform_same_columns_different_case(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db["t"].insert({"Name": "Cleo", "Age": 5}) - fresh_db.create_table("t", {"name": str, "age": int}, transform=True) - # Schema casing is preserved - SQLite considers these the same columns - assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} - assert list(fresh_db["t"].rows) == [{"Name": "Cleo", "Age": 5}] - - -def test_create_table_transform_case_insensitive_with_changes(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db.create_table("t", {"name": str, "age": str, "size": int}, transform=True) - # age changed type, size added, Name untouched - assert fresh_db["t"].columns_dict == {"Name": str, "Age": str, "size": int} - - -def test_transform_types_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str, "Age": str}) - fresh_db["t"].transform(types={"age": int}) - assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} - - -def test_transform_rename_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str}) - fresh_db["t"].transform(rename={"name": "title"}) - assert fresh_db["t"].columns_dict == {"title": str} - - -def test_transform_drop_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db["t"].transform(drop=["name"]) - assert fresh_db["t"].columns_dict == {"Age": int} - - -def test_transform_not_null_and_defaults_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db["t"].transform(not_null={"name"}, defaults={"age": 3}) - columns = {c.name: c for c in fresh_db["t"].columns} - assert columns["Name"].notnull - assert fresh_db["t"].default_values == {"Age": 3} - - -def test_transform_pk_case_insensitive(fresh_db): - fresh_db["t"].create({"Id": int, "Name": str}) - fresh_db["t"].transform(pk="id") - assert fresh_db["t"].pks == ["Id"] - assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} - - -def test_transform_drop_foreign_keys_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create( - {"id": int, "Parent_ID": int}, - pk="id", - foreign_keys=[("Parent_ID", "parent", "Id")], - ) - fresh_db["child"].transform(drop_foreign_keys=["parent_id"]) - assert fresh_db["child"].foreign_keys == [] - - -def test_add_foreign_key_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") - fresh_db["child"].add_foreign_key("parent_id", "parent", "id") - fks = fresh_db["child"].foreign_keys - assert len(fks) == 1 - # The foreign key should use the schema casing of the columns - assert fks[0].column == "Parent_ID" - assert fks[0].other_column == "Id" - - -def test_add_foreign_keys_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") - fresh_db.add_foreign_keys([("child", "parent_id", "parent", "id")]) - fks = fresh_db["child"].foreign_keys - assert len(fks) == 1 - assert fks[0].column == "Parent_ID" - assert fks[0].other_column == "Id" - - -def test_add_foreign_key_detects_existing_case_insensitively(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create( - {"id": int, "Parent_ID": int}, - pk="id", - foreign_keys=[("Parent_ID", "parent", "Id")], - ) - # ignore=True should treat this as already existing, not add a duplicate - fresh_db["child"].add_foreign_key("parent_id", "parent", "id", ignore=True) - assert len(fresh_db["child"].foreign_keys) == 1 - - -def test_add_column_fk_col_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create({"id": int}, pk="id") - fresh_db["child"].add_column("parent_id", int, fk="parent", fk_col="id") - fks = fresh_db["child"].foreign_keys - assert len(fks) == 1 - assert fks[0].other_column == "Id" - - -def test_extract_case_insensitive(fresh_db): - fresh_db["trees"].insert({"id": 1, "Species": "Palm"}, pk="id") - fresh_db["trees"].extract("species") - assert fresh_db["trees"].columns_dict == {"id": int, "Species_id": int} - assert list(fresh_db["Species"].rows) == [{"id": 1, "Species": "Palm"}] - - -def test_convert_multi_case_insensitive(fresh_db): - fresh_db["t"].insert({"id": 1, "Name": "Cleo"}, pk="id") - fresh_db["t"].convert("name", lambda v: {"upper": v.upper()}, multi=True) - assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "upper": "CLEO"}] - - -def test_convert_output_case_insensitive(fresh_db): - fresh_db["t"].insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id") - fresh_db["t"].convert("name", lambda v: v.upper(), output="upper") - assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "Upper": "CLEO"}] - - -def test_create_table_sql_pk_case_insensitive(fresh_db): - fresh_db["t"].create({"Id": int, "Name": str}, pk="id") - # Should not have created an extra lowercase "id" column - assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} - assert fresh_db["t"].pks == ["Id"] - - -def test_create_table_not_null_and_defaults_case_insensitive(fresh_db): - fresh_db["t"].create( - {"Name": str, "Age": int}, not_null={"name"}, defaults={"age": 1} - ) - columns = {c.name: c for c in fresh_db["t"].columns} - assert columns["Name"].notnull - assert fresh_db["t"].default_values == {"Age": 1} - - -def test_create_table_foreign_keys_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create( - {"id": int, "Parent_ID": int}, - pk="id", - foreign_keys=[("parent_id", "parent", "id")], - ) - fks = fresh_db["child"].foreign_keys - assert fks == [ - ForeignKey( - table="child", column="Parent_ID", other_table="parent", other_column="Id" - ) - ] diff --git a/tests/test_constructor.py b/tests/test_constructor.py index a619fba..7714f26 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -87,27 +87,3 @@ def test_legacy_transaction_control_connection_is_accepted(tmpdir): db["t"].insert({"id": 1}, pk="id") assert [r["id"] for r in db["t"].rows] == [1] db.close() - - -def test_memory_attribute_for_memory_true(): - db = Database(memory=True) - assert db.memory is True - assert db.memory_name is None - - -def test_memory_attribute_for_memory_name(): - db = Database(memory_name="shared_attr") - assert db.memory is True - assert db.memory_name == "shared_attr" - - -def test_memory_attribute_for_memory_string_path(): - db = Database(":memory:") - assert db.memory is True - assert db.memory_name is None - - -def test_memory_attribute_for_file_path(tmpdir): - db = Database(str(tmpdir / "file.db")) - assert db.memory is False - assert db.memory_name is None diff --git a/tests/test_create.py b/tests/test_create.py index d281eb4..decefcf 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -3,7 +3,6 @@ from sqlite_utils.db import ( Database, DescIndex, AlterError, - InvalidColumns, NoObviousTable, OperationalError, ForeignKey, @@ -809,34 +808,6 @@ def test_create_index_if_not_exists(fresh_db): dogs.create_index(["name"], if_not_exists=True) -def test_drop_index(fresh_db): - dogs = fresh_db["dogs"] - dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) - dogs.create_index(["name"]) - assert [index.name for index in dogs.indexes] == ["idx_dogs_name"] - dogs.drop_index("idx_dogs_name") - assert dogs.indexes == [] - - -def test_drop_index_ignore(fresh_db): - dogs = fresh_db["dogs"] - dogs.insert({"name": "Cleo"}) - with pytest.raises(OperationalError, match="No index named idx_dogs_name"): - dogs.drop_index("idx_dogs_name") - dogs.drop_index("idx_dogs_name", ignore=True) - - -def test_drop_index_wrong_table(fresh_db): - dogs = fresh_db["dogs"] - cats = fresh_db["cats"] - dogs.insert({"name": "Cleo"}) - cats.insert({"name": "Misty"}) - dogs.create_index(["name"]) - with pytest.raises(OperationalError, match="No index named idx_dogs_name"): - cats.drop_index("idx_dogs_name") - assert [index.name for index in dogs.indexes] == ["idx_dogs_name"] - - def test_create_index_desc(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) @@ -954,58 +925,6 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db assert rows == [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] -@pytest.mark.parametrize("num_rows", (0, 1, 2, 3, 10)) -def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): - # https://github.com/simonw/sqlite-utils/issues/732 - fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") - rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] - - with pytest.raises(InvalidColumns) as ex: - fresh_db["t"].insert_all(rows, pk="not_a_column") - - assert ex.value.args == ( - "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", - ) - assert fresh_db["t"].count == 0 - - -@pytest.mark.parametrize("num_rows", (1, 2, 3, 10)) -def test_insert_all_pk_not_in_records_alter_raises(fresh_db, num_rows): - # With alter=True the check is deferred until the record keys are - # known - a pk column that is in neither the table nor the records - # still raises - fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") - rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] - - with pytest.raises(InvalidColumns) as ex: - fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) - - assert ex.value.args == ( - "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", - ) - assert fresh_db["t"].count == 0 - - -def test_insert_pk_in_records_with_alter_adds_column(fresh_db): - # 3.x allowed insert(pk=..., alter=True) to add the pk column from the - # records - the InvalidColumns check must not fire in that case - fresh_db["t"].insert({"a": 1}) - fresh_db["t"].insert({"id": 5, "a": 2}, pk="id", alter=True) - assert fresh_db["t"].columns_dict.keys() == {"a", "id"} - assert list(fresh_db.query("select * from t order by a")) == [ - {"a": 1, "id": None}, - {"a": 2, "id": 5}, - ] - - -def test_insert_all_invalid_pk_alter_empty_records_is_noop(fresh_db): - # With alter=True the pk check needs record keys, so an empty iterator - # returns without error - matching the 3.x no-op for empty inserts - fresh_db.conn.execute("CREATE TABLE t (a TEXT)") - fresh_db["t"].insert_all([], pk="not_a_column", alter=True) - assert fresh_db["t"].count == 0 - - def test_insert_ignore(fresh_db): fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again @@ -1018,114 +937,6 @@ def test_insert_ignore(fresh_db): assert rows == [{"id": 1, "bar": 2}] -def test_insert_ignore_reports_existing_row(fresh_db): - # An ignored insert (row already exists) should point last_rowid and - # last_pk at the existing conflicting row - see the Datasette insert API - fresh_db["docs"].insert({"id": 1, "title": "Exists"}, pk="id") - # Insert a conflicting row with ignore=True and no explicit pk= - table = fresh_db["docs"].insert({"id": 1, "title": "One"}, ignore=True) - assert table.last_rowid == 1 - assert table.last_pk == 1 - assert list(fresh_db["docs"].rows_where("rowid = ?", [table.last_rowid])) == [ - {"id": 1, "title": "Exists"} - ] - - -@pytest.mark.parametrize("rowid_alias", ("rowid", "_rowid_", "oid")) -@pytest.mark.parametrize("method", ("upsert", "insert_replace", "insert_ignore")) -def test_pk_rowid_alias_on_rowid_table(fresh_db, rowid_alias, method): - # rowid and its aliases are valid primary keys for a rowid table even - # though they are not listed among the table's columns - see the Datasette - # upsert API against tables without an explicit primary key - fresh_db["t"].insert({"title": "Hello"}) - assert fresh_db["t"].pks == ["rowid"] - record = {rowid_alias: 1, "title": "Updated"} - if method == "upsert": - table = fresh_db["t"].upsert(record, pk=rowid_alias) - elif method == "insert_replace": - table = fresh_db["t"].insert(record, pk=rowid_alias, replace=True) - else: - table = fresh_db["t"].insert(record, pk=rowid_alias, ignore=True) - assert table.last_pk == 1 - expected_title = "Hello" if method == "insert_ignore" else "Updated" - assert list(fresh_db["t"].rows) == [{"title": expected_title}] - - -def test_insert_ignore_reports_existing_row_compound_pk(fresh_db): - # Compound primary key variant of the ignored-insert lookup - fresh_db["t"].insert_all([{"a": 1, "b": 2, "note": "first"}], pk=("a", "b")) - table = fresh_db["t"].insert( - {"a": 1, "b": 2, "note": "second"}, pk=("a", "b"), ignore=True - ) - assert table.last_pk == (1, 2) - assert list(fresh_db["t"].rows_where("rowid = ?", [table.last_rowid])) == [ - {"a": 1, "b": 2, "note": "first"} - ] - - -def test_insert_ignore_reports_existing_row_list_mode(fresh_db): - # List-based iteration variant of the ignored-insert lookup - fresh_db["t"].insert_all([["id", "title"], [1, "first"]], pk="id") - table = fresh_db["t"].insert_all( - [["id", "title"], [1, "second"]], pk="id", ignore=True - ) - assert table.last_pk == 1 - assert table.last_rowid == 1 - assert list(fresh_db["t"].rows) == [{"id": 1, "title": "first"}] - - -def test_insert_ignore_hash_id_reports_pk(fresh_db): - # With hash_id the pk is the computed hash; the original record has no id - # column to look up so last_rowid is left unset - first = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id") - table = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id", ignore=True) - assert table.last_pk == first.last_pk - assert table.last_rowid is None - assert fresh_db["dogs"].count == 1 - - -def test_insert_ignore_unresolvable_conflict_leaves_pk_unset(fresh_db): - # When the conflict cannot be resolved to a primary key lookup, last_pk and - # last_rowid are left unset rather than reporting a misleading value - - # rowid table with a UNIQUE column and no primary key: no pk to look up - fresh_db["u"].db.execute("create table u (title text unique)") - fresh_db["u"].insert({"title": "x"}) - table = fresh_db["u"].insert({"title": "x"}, ignore=True) - assert table.last_pk is None - assert table.last_rowid is None - assert fresh_db["u"].count == 1 - - # Conflict on a UNIQUE column other than the primary key: the pk value from - # the record does not match the existing row, so the lookup finds nothing - fresh_db["docs"].db.execute( - "create table docs (id integer primary key, email text unique)" - ) - fresh_db["docs"].insert({"id": 1, "email": "a"}, pk="id") - table = fresh_db["docs"].insert({"id": 2, "email": "a"}, ignore=True) - assert table.last_pk is None - assert table.last_rowid is None - assert fresh_db["docs"].count == 1 - - -def test_insert_ignore_with_pk_after_other_table_insert(fresh_db): - # https://github.com/simonw/sqlite-utils/issues/554 - user = {"id": "abc", "name": "david"} - - fresh_db["users"].insert(user, pk="id") - fresh_db["comments"].insert_all( - [ - {"id": "def", "text": "ok"}, - {"id": "ghi", "text": "great"}, - ], - ) - - table = fresh_db["users"].insert(user, pk="id", ignore=True) - - assert table.last_pk == "abc" - assert list(fresh_db["users"].rows) == [user] - - def test_insert_hash_id(fresh_db): dogs = fresh_db["dogs"] id = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk diff --git a/tests/test_default_value.py b/tests/test_default_value.py index 3724d99..c5e4b17 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -21,11 +21,6 @@ EXAMPLES = [ # Strings ("TEXT DEFAULT 'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'"), ('TEXT DEFAULT "CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"'), - # Boolean and null keyword literals must stay unquoted - ("INTEGER DEFAULT TRUE", "TRUE", "TRUE"), - ("INTEGER DEFAULT FALSE", "FALSE", "FALSE"), - ("INTEGER DEFAULT true", "true", "true"), - ("TEXT DEFAULT NULL", "NULL", "NULL"), ] diff --git a/tests/test_extract.py b/tests/test_extract.py index c73ee7a..d24c597 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -189,116 +189,9 @@ def test_extract_works_with_null_values(fresh_db): ) assert list(fresh_db["listens"].rows) == [ {"id": 1, "track_title": "foo", "album_id": 1}, - {"id": 2, "track_title": "baz", "album_id": None}, + {"id": 2, "track_title": "baz", "album_id": 2}, ] assert list(fresh_db["albums"].rows) == [ {"id": 1, "album_title": "bar"}, + {"id": 2, "album_title": None}, ] - - -def test_extract_null_values_single_column(fresh_db): - # https://github.com/simonw/sqlite-utils/issues/186 - fresh_db["species"].insert({"id": 1, "species": "Wolf"}, pk="id") - fresh_db["individuals"].insert_all( - [ - {"id": 10, "name": "Terriana", "species": "Fox"}, - {"id": 11, "name": "Spenidorm", "species": None}, - {"id": 12, "name": "Grantheim", "species": "Wolf"}, - {"id": 13, "name": "Turnutopia", "species": None}, - {"id": 14, "name": "Wargal", "species": "Wolf"}, - ], - pk="id", - ) - fresh_db["individuals"].extract("species") - # No null row should have been added to species - assert list(fresh_db["species"].rows) == [ - {"id": 1, "species": "Wolf"}, - {"id": 2, "species": "Fox"}, - ] - assert list(fresh_db["individuals"].rows) == [ - {"id": 10, "name": "Terriana", "species_id": 2}, - {"id": 11, "name": "Spenidorm", "species_id": None}, - {"id": 12, "name": "Grantheim", "species_id": 1}, - {"id": 13, "name": "Turnutopia", "species_id": None}, - {"id": 14, "name": "Wargal", "species_id": 1}, - ] - - -def test_extract_null_values_multiple_columns(fresh_db): - # A row should be extracted if at least one column is not null - - # only rows where ALL extracted columns are null are left alone - fresh_db["circulation"].insert_all( - [ - {"id": 1, "title": "title one", "creator": "creator one", "year": 2018}, - {"id": 2, "title": "title two", "creator": None, "year": 2019}, - {"id": 3, "title": None, "creator": None, "year": 2020}, - {"id": 4, "title": None, "creator": None, "year": 2021}, - ], - pk="id", - ) - fresh_db["circulation"].extract( - ["title", "creator"], table="books", fk_column="book_id" - ) - assert list(fresh_db["books"].rows) == [ - {"id": 1, "title": "title one", "creator": "creator one"}, - {"id": 2, "title": "title two", "creator": None}, - ] - assert list(fresh_db["circulation"].rows) == [ - {"id": 1, "book_id": 1, "year": 2018}, - {"id": 2, "book_id": 2, "year": 2019}, - {"id": 3, "book_id": None, "year": 2020}, - {"id": 4, "book_id": None, "year": 2021}, - ] - - -def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db): - # Even if the lookup table already contains an all-null row, rows where - # every extracted column is null should keep a null foreign key - fresh_db["species"].insert({"id": 1, "species": None}, pk="id") - fresh_db["individuals"].insert_all( - [ - {"id": 10, "name": "Terriana", "species": "Fox"}, - {"id": 11, "name": "Spenidorm", "species": None}, - ], - pk="id", - ) - fresh_db["individuals"].extract("species") - assert list(fresh_db["species"].rows) == [ - {"id": 1, "species": None}, - {"id": 2, "species": "Fox"}, - ] - assert list(fresh_db["individuals"].rows) == [ - {"id": 10, "name": "Terriana", "species_id": 2}, - {"id": 11, "name": "Spenidorm", "species_id": None}, - ] - - -def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db): - # Unique indexes treat NULLs as distinct, so INSERT OR IGNORE alone - # cannot dedupe NULL-containing rows against the existing lookup - # table - extracting a second table into the same lookup previously - # inserted duplicate rows that nothing pointed to - fresh_db["t1"].insert_all( - [ - {"id": 1, "species": None, "common": "X"}, - {"id": 2, "species": "Oak", "common": "Oak"}, - ], - pk="id", - ) - fresh_db["t2"].insert_all([{"id": 1, "species": None, "common": "X"}], pk="id") - fresh_db["t1"].extract(["species", "common"], table="lk") - fresh_db["t2"].extract(["species", "common"], table="lk") - assert fresh_db["lk"].count == 2 - # Both tables point at the same lookup row - t1_fk = fresh_db.execute("select lk_id from t1 where id = 1").fetchone()[0] - t2_fk = fresh_db.execute("select lk_id from t2 where id = 1").fetchone()[0] - assert t1_fk == t2_fk - - -def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db): - # Non-NULL rows were already deduped by the unique index - keep it so - fresh_db["t1"].insert_all([{"id": 1, "species": "Oak"}], pk="id") - fresh_db["t2"].insert_all([{"id": 1, "species": "Oak"}], pk="id") - fresh_db["t1"].extract(["species"], table="lk") - fresh_db["t2"].extract(["species"], table="lk") - assert fresh_db["lk"].count == 1 diff --git a/tests/test_extracts.py b/tests/test_extracts.py index 7add79a..eb4f37e 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -67,51 +67,3 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): {"id": 2, "species_id": 1}, {"id": 3, "species_id": 2}, ] == list(fresh_db["Trees"].rows) - - -def test_extracts_null_values(fresh_db): - # https://github.com/simonw/sqlite-utils/issues/186 - # Null values should stay null, not be extracted into the lookup table - fresh_db["Trees"].insert_all( - [ - {"id": 1, "species_id": "Oak"}, - {"id": 2, "species_id": None}, - {"id": 3, "species_id": "Palm"}, - {"id": 4, "species_id": None}, - ], - extracts={"species_id": "Species"}, - ) - assert list(fresh_db["Species"].rows) == [ - {"id": 1, "value": "Oak"}, - {"id": 2, "value": "Palm"}, - ] - assert list(fresh_db["Trees"].rows) == [ - {"id": 1, "species_id": 1}, - {"id": 2, "species_id": None}, - {"id": 3, "species_id": 2}, - {"id": 4, "species_id": None}, - ] - - -def test_extracts_null_values_list_mode(fresh_db): - # Same as test_extracts_null_values but for list-based records - fresh_db["Trees"].insert_all( - [ - ["id", "species_id"], - [1, "Oak"], - [2, None], - [3, "Palm"], - [4, None], - ], - extracts={"species_id": "Species"}, - ) - assert list(fresh_db["Species"].rows) == [ - {"id": 1, "value": "Oak"}, - {"id": 2, "value": "Palm"}, - ] - assert list(fresh_db["Trees"].rows) == [ - {"id": 1, "species_id": 1}, - {"id": 2, "species_id": None}, - {"id": 3, "species_id": 2}, - {"id": 4, "species_id": None}, - ] diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py deleted file mode 100644 index b37d374..0000000 --- a/tests/test_foreign_keys.py +++ /dev/null @@ -1,691 +0,0 @@ -"""Tests for compound (multi-column) foreign keys - issue #594.""" - -import pytest -from sqlite_utils import Database -from sqlite_utils.db import AlterError, ForeignKey -from sqlite_utils.utils import sqlite3 - -COMPOUND_SCHEMA = """ -CREATE TABLE departments ( - campus_name TEXT NOT NULL, - dept_code TEXT NOT NULL, - dept_name TEXT, - PRIMARY KEY (campus_name, dept_code) -); -CREATE TABLE courses ( - course_code TEXT PRIMARY KEY, - course_name TEXT, - campus_name TEXT NOT NULL, - dept_code TEXT NOT NULL, - FOREIGN KEY (campus_name, dept_code) - REFERENCES departments(campus_name, dept_code) -); -""" - - -@pytest.fixture -def compound_db(): - db = Database(memory=True) - db.executescript(COMPOUND_SCHEMA) - return db - - -def test_compound_foreign_key(compound_db): - fks = compound_db["courses"].foreign_keys - assert len(fks) == 1 - fk = fks[0] - assert fk.is_compound is True - assert fk.table == "courses" - assert fk.other_table == "departments" - assert fk.columns == ("campus_name", "dept_code") - assert fk.other_columns == ("campus_name", "dept_code") - # Scalar column/other_column can't sensibly hold a compound key - assert fk.column is None - assert fk.other_column is None - - -def test_single_foreign_key_gets_columns_fields(fresh_db): - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fk = fresh_db["books"].foreign_keys[0] - assert fk.is_compound is False - assert fk.column == "author_id" - assert fk.other_column == "id" - assert fk.columns == ("author_id",) - assert fk.other_columns == ("id",) - - -def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): - # Clean break in 4.0: ForeignKey is a dataclass, not a namedtuple, so the - # old tuple unpacking and indexing patterns now fail hard. - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fk = fresh_db["books"].foreign_keys[0] - with pytest.raises(TypeError): - table, column, other_table, other_column = fk - with pytest.raises(TypeError): - fk[0] - - -def test_foreign_keys_are_sortable(fresh_db): - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["categories"].insert({"id": 1, "name": "Wildlife"}, pk="id") - fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1, "category_id": 1}) - fresh_db.add_foreign_keys( - [ - ("books", "author_id", "authors", "id"), - ("books", "category_id", "categories", "id"), - ] - ) - fks = sorted(fresh_db["books"].foreign_keys) - assert fks[0].column == "author_id" - assert fks[1].column == "category_id" - - -def test_mixed_compound_and_single_foreign_keys_are_sortable(): - # compound FKs have column=None, which must not break sorting - # against single-column FKs (None < str raises TypeError) - db = Database(memory=True) - db.executescript(""" - CREATE TABLE departments ( - campus_name TEXT NOT NULL, - dept_code TEXT NOT NULL, - PRIMARY KEY (campus_name, dept_code) - ); - CREATE TABLE accreditations (id INTEGER PRIMARY KEY); - CREATE TABLE courses ( - course_code TEXT PRIMARY KEY, - campus_name TEXT NOT NULL, - dept_code TEXT NOT NULL, - accreditation_id INTEGER REFERENCES accreditations(id), - FOREIGN KEY (campus_name, dept_code) - REFERENCES departments(campus_name, dept_code) - ); - """) - fks = db["courses"].foreign_keys - assert len(fks) == 2 - assert {fk.is_compound for fk in fks} == {True, False} - fks_sorted = sorted(fks) - assert fks_sorted[0].other_table == "accreditations" - assert fks_sorted[1].other_table == "departments" - - -@pytest.fixture -def departments_db(): - db = Database(memory=True) - db.create_table( - "departments", - {"campus_name": str, "dept_code": str, "dept_name": str}, - pk=("campus_name", "dept_code"), - ) - return db - - -EXPECTED_COURSES_SCHEMA = ( - 'CREATE TABLE "courses" (\n' - ' "course_code" TEXT PRIMARY KEY,\n' - ' "campus_name" TEXT,\n' - ' "dept_code" TEXT,\n' - ' FOREIGN KEY ("campus_name", "dept_code") ' - 'REFERENCES "departments"("campus_name", "dept_code")\n' - ")" -) - - -@pytest.mark.parametrize( - "foreign_keys", - ( - [ - ForeignKey( - table="courses", - column=None, - other_table="departments", - other_column=None, - columns=("campus_name", "dept_code"), - other_columns=("campus_name", "dept_code"), - is_compound=True, - ) - ], - [(("campus_name", "dept_code"), "departments", ("campus_name", "dept_code"))], - # Two-item form guesses the other table's primary key: - [(("campus_name", "dept_code"), "departments")], - # Lists work too, though tuples are the documented form: - [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])], - ), -) -def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): - departments_db.create_table( - "courses", - {"course_code": str, "campus_name": str, "dept_code": str}, - pk="course_code", - foreign_keys=foreign_keys, - ) - assert departments_db["courses"].schema == EXPECTED_COURSES_SCHEMA - fks = departments_db["courses"].foreign_keys - assert len(fks) == 1 - fk = fks[0] - assert fk.is_compound is True - assert fk.columns == ("campus_name", "dept_code") - assert fk.other_table == "departments" - assert fk.other_columns == ("campus_name", "dept_code") - - -def test_create_table_compound_foreign_key_enforced(departments_db): - departments_db.execute("PRAGMA foreign_keys = ON") - departments_db.create_table( - "courses", - {"course_code": str, "campus_name": str, "dept_code": str}, - pk="course_code", - foreign_keys=[(("campus_name", "dept_code"), "departments")], - ) - departments_db["departments"].insert( - {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"} - ) - departments_db["courses"].insert( - {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"} - ) - with pytest.raises(sqlite3.IntegrityError): - departments_db.execute( - "insert into courses (course_code, campus_name, dept_code) " - "values ('X1', 'Nowhere', 'NOPE')" - ) - - -def test_create_table_compound_foreign_key_missing_other_column(departments_db): - with pytest.raises(AlterError): - departments_db.create_table( - "courses", - {"course_code": str, "campus_name": str, "dept_code": str}, - pk="course_code", - foreign_keys=[ - (("campus_name", "dept_code"), "departments", ("campus_name", "nope")) - ], - ) - - -def test_transform_preserves_compound_foreign_key(compound_db): - compound_db["courses"].transform(rename={"course_name": "title"}) - fks = compound_db["courses"].foreign_keys - assert len(fks) == 1 - fk = fks[0] - assert fk.is_compound is True - assert fk.columns == ("campus_name", "dept_code") - assert fk.other_table == "departments" - assert fk.other_columns == ("campus_name", "dept_code") - - -def test_transform_rename_member_column_updates_compound_foreign_key(compound_db): - compound_db["courses"].transform(rename={"campus_name": "campus"}) - fks = compound_db["courses"].foreign_keys - assert len(fks) == 1 - fk = fks[0] - assert fk.is_compound is True - assert fk.columns == ("campus", "dept_code") - # Referenced columns in the other table are unchanged - assert fk.other_columns == ("campus_name", "dept_code") - - -def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): - # Matches single-column behavior: dropping the column silently - # drops the foreign key that used it - compound_db["courses"].transform(drop={"dept_code"}) - assert compound_db["courses"].foreign_keys == [] - assert "FOREIGN KEY" not in compound_db["courses"].schema - - -@pytest.mark.parametrize( - "drop_foreign_keys", - ( - # A bare column name matches any foreign key it participates in: - ["campus_name"], - # A tuple must match the full compound key: - [("campus_name", "dept_code")], - ), -) -def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys): - compound_db["courses"].transform(drop_foreign_keys=drop_foreign_keys) - assert compound_db["courses"].foreign_keys == [] - # The columns themselves survive - assert {"campus_name", "dept_code"} <= set( - compound_db["courses"].columns_dict.keys() - ) - - -@pytest.fixture -def courses_db(departments_db): - departments_db.create_table( - "courses", - {"course_code": str, "campus_name": str, "dept_code": str}, - pk="course_code", - ) - return departments_db - - -def test_add_compound_foreign_key(courses_db): - t = courses_db["courses"].add_foreign_key( - ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") - ) - # Returns self - assert t.name == "courses" - fks = courses_db["courses"].foreign_keys - assert len(fks) == 1 - fk = fks[0] - assert fk.is_compound is True - assert fk.columns == ("campus_name", "dept_code") - assert fk.other_table == "departments" - assert fk.other_columns == ("campus_name", "dept_code") - - -def test_add_compound_foreign_key_guesses_other_columns(courses_db): - # Lists work here too, though tuples are the documented form - courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") - fk = courses_db["courses"].foreign_keys[0] - assert fk.other_columns == ("campus_name", "dept_code") - - -def test_add_compound_foreign_key_error_if_already_exists(courses_db): - courses_db["courses"].add_foreign_key(("campus_name", "dept_code"), "departments") - with pytest.raises(AlterError) as ex: - courses_db["courses"].add_foreign_key( - ("campus_name", "dept_code"), "departments" - ) - assert "already exists" in ex.value.args[0] - # ignore=True should not raise - courses_db["courses"].add_foreign_key( - ("campus_name", "dept_code"), "departments", ignore=True - ) - - -def test_add_compound_foreign_key_error_if_column_missing(courses_db): - with pytest.raises(AlterError): - courses_db["courses"].add_foreign_key(("campus_name", "nope"), "departments") - - -def test_db_add_foreign_keys_compound(courses_db): - courses_db.add_foreign_keys( - [ - ( - "courses", - ("campus_name", "dept_code"), - "departments", - ("campus_name", "dept_code"), - ) - ] - ) - fk = courses_db["courses"].foreign_keys[0] - assert fk.is_compound is True - assert fk.columns == ("campus_name", "dept_code") - - -def test_index_foreign_keys_compound_creates_composite_index(compound_db): - compound_db.index_foreign_keys() - index_columns = [i.columns for i in compound_db["courses"].indexes] - assert ["campus_name", "dept_code"] in index_columns - # No separate single-column indexes for the members - assert ["campus_name"] not in index_columns - assert ["dept_code"] not in index_columns - - -def test_foreign_key_captures_on_delete_and_on_update(): - db = Database(memory=True) - db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - author_id INTEGER REFERENCES authors(id) - ON DELETE CASCADE ON UPDATE RESTRICT - ); - """) - fk = db["books"].foreign_keys[0] - assert fk.on_delete == "CASCADE" - assert fk.on_update == "RESTRICT" - - -def test_foreign_key_on_delete_defaults_to_no_action(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fk = fresh_db["books"].foreign_keys[0] - assert fk.on_delete == "NO ACTION" - assert fk.on_update == "NO ACTION" - - -def test_create_table_foreign_key_with_on_delete(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db.create_table( - "books", - {"id": int, "author_id": int}, - pk="id", - foreign_keys=[ - ForeignKey( - table="books", - column="author_id", - other_table="authors", - other_column="id", - on_delete="CASCADE", - ) - ], - ) - assert "ON DELETE CASCADE" in fresh_db["books"].schema - assert fresh_db["books"].foreign_keys[0].on_delete == "CASCADE" - - -def test_transform_preserves_on_delete_cascade(): - db = Database(memory=True) - db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE - ); - """) - db["books"].transform(rename={"title": "book_title"}) - fk = db["books"].foreign_keys[0] - assert fk.on_delete == "CASCADE" - assert fk.on_update == "NO ACTION" - assert "ON DELETE CASCADE" in db["books"].schema - - -def test_transform_preserves_compound_foreign_key_on_delete(): - db = Database(memory=True) - db.executescript(""" - CREATE TABLE departments ( - campus_name TEXT NOT NULL, - dept_code TEXT NOT NULL, - PRIMARY KEY (campus_name, dept_code) - ); - CREATE TABLE courses ( - course_code TEXT PRIMARY KEY, - campus_name TEXT NOT NULL, - dept_code TEXT NOT NULL, - FOREIGN KEY (campus_name, dept_code) - REFERENCES departments(campus_name, dept_code) ON DELETE CASCADE - ); - """) - db["courses"].transform(rename={"course_code": "code"}) - fk = db["courses"].foreign_keys[0] - assert fk.is_compound is True - assert fk.on_delete == "CASCADE" - assert "ON DELETE CASCADE" in db["courses"].schema - - -def test_implicit_primary_key_reference_is_resolved(): - # REFERENCES authors (no column) has "to" of None in the pragma - - # it should be resolved to the primary key of the other table - db = Database(memory=True) - db.executescript(""" - CREATE TABLE authors (author_id INTEGER PRIMARY KEY); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - author_id INTEGER REFERENCES authors - ); - """) - fk = db["books"].foreign_keys[0] - assert fk.is_compound is False - assert fk.other_column == "author_id" - assert fk.other_columns == ("author_id",) - - -def test_implicit_compound_primary_key_reference_is_resolved(): - db = Database(memory=True) - db.executescript(""" - CREATE TABLE departments ( - campus_name TEXT NOT NULL, - dept_code TEXT NOT NULL, - PRIMARY KEY (campus_name, dept_code) - ); - CREATE TABLE courses ( - course_code TEXT PRIMARY KEY, - campus_name TEXT NOT NULL, - dept_code TEXT NOT NULL, - FOREIGN KEY (campus_name, dept_code) REFERENCES departments - ); - """) - fk = db["courses"].foreign_keys[0] - assert fk.is_compound is True - assert fk.other_columns == ("campus_name", "dept_code") - - -def test_foreign_key_normalizes_list_columns_to_tuples(): - # Compound columns passed as lists are normalized to tuples, so they - # compare equal to introspected ForeignKeys - fk = ForeignKey( - table="courses", - column=None, - other_table="departments", - other_column=None, - columns=["campus_name", "dept_code"], - other_columns=["campus_name", "dept_code"], - is_compound=True, - ) - assert fk.columns == ("campus_name", "dept_code") - assert fk.other_columns == ("campus_name", "dept_code") - - -def test_add_foreign_keys_preserves_actions(fresh_db): - # https://github.com/simonw/sqlite-utils/issues/594 review finding: - # ForeignKey objects passed to db.add_foreign_keys() were flattened - # to plain tuples, losing on_delete/on_update - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db.add_foreign_keys( - [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] - ) - fk = fresh_db["books"].foreign_keys[0] - assert fk.on_delete == "CASCADE" - assert "ON DELETE CASCADE" in fresh_db["books"].schema - - -def test_add_foreign_keys_preserves_actions_compound(courses_db): - courses_db.add_foreign_keys( - [ - ForeignKey( - table="courses", - column=None, - other_table="departments", - other_column=None, - columns=("campus_name", "dept_code"), - other_columns=("campus_name", "dept_code"), - is_compound=True, - on_delete="CASCADE", - ) - ] - ) - fk = courses_db["courses"].foreign_keys[0] - assert fk.is_compound is True - assert fk.on_delete == "CASCADE" - assert "ON DELETE CASCADE" in courses_db["courses"].schema - - -def test_add_foreign_key_on_delete_on_update(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db["books"].add_foreign_key( - "author_id", "authors", "id", on_delete="CASCADE", on_update="RESTRICT" - ) - fk = fresh_db["books"].foreign_keys[0] - assert fk.on_delete == "CASCADE" - assert fk.on_update == "RESTRICT" - assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db["books"].schema - # The cascade should actually fire - fresh_db.execute("PRAGMA foreign_keys = ON") - fresh_db.execute("delete from authors where id = 1") - assert fresh_db["books"].count == 0 - - -def test_add_compound_foreign_key_on_delete(courses_db): - courses_db["courses"].add_foreign_key( - ("campus_name", "dept_code"), "departments", on_delete="SET NULL" - ) - fk = courses_db["courses"].foreign_keys[0] - assert fk.is_compound is True - assert fk.on_delete == "SET NULL" - assert "ON DELETE SET NULL" in courses_db["courses"].schema - - -def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db): - # The other table's PRIMARY KEY declares its columns in a different - # order to the table's column order. SQLite resolves the implicit - # "REFERENCES other" using PRIMARY KEY declaration order, so the - # introspected other_columns must too - fresh_db.execute("create table other (b text, a text, primary key (a, b))") - fresh_db.execute( - "create table child (x text, y text, foreign key (x, y) references other)" - ) - fk = fresh_db["child"].foreign_keys[0] - assert fk.other_columns == ("a", "b") - - -def test_transform_implicit_compound_foreign_key_stays_valid(fresh_db): - # transform() rewrites the implicit FK with explicit columns - they - # must be in PRIMARY KEY declaration order or valid data fails the - # foreign key check with an IntegrityError - fresh_db.execute("create table other (b text, a text, primary key (a, b))") - fresh_db.execute( - "create table child (x text, y text, foreign key (x, y) references other)" - ) - fresh_db.execute("PRAGMA foreign_keys = ON") - fresh_db["other"].insert({"a": "A", "b": "B"}) - fresh_db["child"].insert({"x": "A", "y": "B"}) - fresh_db["child"].transform(types={"x": str}) - assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") - # The constraint still points the right way around - fresh_db["child"].insert({"x": "A", "y": "B"}) - with pytest.raises(sqlite3.IntegrityError): - fresh_db["child"].insert({"x": "B", "y": "A"}) - - -def test_create_compound_foreign_key_guesses_pk_declaration_order(fresh_db): - fresh_db.execute("create table other (b text, a text, primary key (a, b))") - fresh_db["other"].insert({"a": "A", "b": "B"}) - fresh_db["child"].create( - {"id": int, "x": str, "y": str}, - pk="id", - foreign_keys=[(("x", "y"), "other")], - ) - assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") - fresh_db.execute("PRAGMA foreign_keys = ON") - fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}) - with pytest.raises(sqlite3.IntegrityError): - fresh_db["child"].insert({"id": 2, "x": "B", "y": "A"}) - - -def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db): - fresh_db.execute("create table other (b text, a text, primary key (a, b))") - fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id") - fresh_db["child"].add_foreign_key(("x", "y"), "other") - assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") - - -def test_foreign_keys_are_hashable(fresh_db): - # set() over foreign_keys worked with the 3.x namedtuple and must - # keep working with the dataclass - fresh_db["p"].insert({"id": 1}, pk="id") - fresh_db["c"].insert( - {"id": 1, "pid": 1}, pk="id", foreign_keys=[("pid", "p", "id")] - ) - fks = set(fresh_db["c"].foreign_keys) - assert len(fks) == 1 - assert ForeignKey("c", "pid", "p", "id") in fks - # Usable as dict keys too - assert {fk: True for fk in fks} - - -def test_foreign_key_is_immutable(): - import dataclasses - - fk = ForeignKey("c", "pid", "p", "id") - with pytest.raises(dataclasses.FrozenInstanceError): - fk.table = "other" - - -def test_foreign_key_equality_and_hash_include_actions(): - # Two foreign keys differing only in ON DELETE behavior are different - # constraints - they compare unequal and hash separately - plain = ForeignKey("c", "pid", "p", "id") - cascade = ForeignKey("c", "pid", "p", "id", on_delete="CASCADE") - assert plain != cascade - assert len({plain, cascade}) == 2 - assert plain == ForeignKey("c", "pid", "p", "id") - - -def test_create_table_mixed_foreign_keys_list(fresh_db): - # 3.x accepted a mix of ForeignKey objects, tuples and bare column - # strings in foreign_keys= (ForeignKey was a namedtuple, so it passed - # the tuple check) - keep accepting the mix - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["publishers"].insert({"id": 1}, pk="id") - fresh_db["books"].create( - {"id": int, "author_id": int, "publisher_id": int}, - pk="id", - foreign_keys=[ - ForeignKey("books", "author_id", "authors", "id"), - ("publisher_id", "publishers", "id"), - ], - ) - fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} - assert fks == {"author_id": "authors", "publisher_id": "publishers"} - - -def test_create_table_mixed_foreign_keys_with_string(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["publishers"].insert({"id": 1}, pk="id") - fresh_db["books"].create( - {"id": int, "author_id": int, "publisher_id": int}, - pk="id", - foreign_keys=[ - "author_id", # bare column, table and column guessed - ("publisher_id", "publishers", "id"), - ], - ) - fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} - assert fks == {"author_id": "authors", "publisher_id": "publishers"} - - -def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db): - # Requesting an existing foreign key with different ON DELETE/ON UPDATE - # actions was silently skipped, dropping the requested change - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert( - {"id": 1, "author_id": 1}, - pk="id", - foreign_keys=[("author_id", "authors", "id")], - ) - with pytest.raises(AlterError) as ex: - fresh_db.add_foreign_keys( - [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] - ) - assert "ON DELETE" in str(ex.value) - assert fresh_db["books"].foreign_keys[0].on_delete == "NO ACTION" - - -def test_add_foreign_keys_identical_existing_is_noop(fresh_db): - # An exact match, including actions, is silently skipped so repeated - # calls stay idempotent - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id", on_delete="CASCADE") - fresh_db.add_foreign_keys( - [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] - ) - fks = fresh_db["books"].foreign_keys - assert len(fks) == 1 - assert fks[0].on_delete == "CASCADE" - - -def test_add_foreign_keys_compound_column_count_mismatch_errors(fresh_db): - # Previously the extra other-column was silently discarded, creating - # a single-column foreign key to just ("id") - fresh_db["departments"].insert( - {"campus": "north", "code": "cs"}, pk=("campus", "code") - ) - fresh_db["courses"].insert({"id": 1, "campus": "north"}, pk="id") - with pytest.raises(ValueError) as ex: - fresh_db.add_foreign_keys( - [("courses", ("campus",), "departments", ("campus", "code"))] - ) - assert "same number of columns" in str(ex.value) - assert fresh_db["courses"].foreign_keys == [] diff --git a/tests/test_fts.py b/tests/test_fts.py index 64ec645..3f7c5a9 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -83,20 +83,6 @@ def test_enable_fts_escape_table_names(fresh_db): assert [] == list(table.search("bar")) -def test_search_duplicate_columns_are_deduped(fresh_db): - # https://github.com/simonw/sqlite-utils/issues/624 - table = fresh_db["t"] - table.insert_all(search_records) - table.enable_fts(["text", "country"], fts_version="FTS4") - rows = list(table.search("tanuki", columns=["text", "text"])) - assert rows == [ - { - "text": "tanuki are running tricksters", - "text_2": "tanuki are running tricksters", - } - ] - - def test_search_limit_offset(fresh_db): table = fresh_db["t"] table.insert_all(search_records) diff --git a/tests/test_gis.py b/tests/test_gis.py index f39554e..1b5ed70 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -6,6 +6,12 @@ from sqlite_utils.cli import cli from sqlite_utils.db import Database from sqlite_utils.utils import find_spatialite, sqlite3 +try: + import sqlean # type: ignore[import-not-found] +except ImportError: + sqlean = None + + pytestmark = [ pytest.mark.skipif( not find_spatialite(), reason="Could not find SpatiaLite extension" @@ -14,6 +20,9 @@ pytestmark = [ not hasattr(sqlite3.Connection, "enable_load_extension"), reason="sqlite3.Connection missing enable_load_extension", ), + pytest.mark.skipif( + sqlean is not None, reason="sqlean.py is not compatible with SpatiaLite" + ), ] diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 8b6765d..4ff3f77 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -321,18 +321,3 @@ def test_table_default_values(fresh_db, value): ) default_values = fresh_db["default_values"].default_values assert default_values == {"value": value} - - -def test_pks_use_primary_key_declaration_order(fresh_db): - # PRIMARY KEY (a, b) declared against columns stored in order (b, a) - - # pks must follow the declaration order, which is what SQLite uses to - # resolve implicit foreign key references and compound pk lookups - fresh_db.execute("create table t (b text, a text, primary key (a, b))") - assert fresh_db["t"].pks == ["a", "b"] - - -def test_transform_preserves_compound_pk_declaration_order(fresh_db): - fresh_db.execute("create table t (a text, b text, c text, primary key (b, a))") - fresh_db["t"].transform(drop={"c"}) - assert fresh_db["t"].pks == ["b", "a"] - assert 'PRIMARY KEY ("b", "a")' in fresh_db["t"].schema diff --git a/tests/test_lookup.py b/tests/test_lookup.py index da4f18b..a36b464 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -157,27 +157,3 @@ def test_lookup_with_extra_insert_parameters(fresh_db): def test_lookup_new_table_strict(fresh_db, strict): fresh_db["species"].lookup({"name": "Palm"}, strict=strict) assert fresh_db["species"].strict == strict or not fresh_db.supports_strict - - -def test_lookup_null_value_idempotent(fresh_db): - # https://github.com/simonw/sqlite-utils/issues/186 - # Repeated lookups of a null value should return the same row, - # not insert a duplicate row each time - species = fresh_db["species"] - first_id = species.lookup({"name": None}) - second_id = species.lookup({"name": None}) - assert first_id == second_id - assert list(species.rows) == [{"id": first_id, "name": None}] - - -def test_lookup_compound_key_with_null_idempotent(fresh_db): - species = fresh_db["species"] - palm_id = species.lookup({"name": "Palm", "type": None}) - oak_id = species.lookup({"name": "Oak", "type": "Tree"}) - assert palm_id == species.lookup({"name": "Palm", "type": None}) - assert oak_id == species.lookup({"name": "Oak", "type": "Tree"}) - assert palm_id != oak_id - assert list(species.rows) == [ - {"id": palm_id, "name": "Palm", "type": None}, - {"id": oak_id, "name": "Oak", "type": "Tree"}, - ] diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 04185fc..5634d79 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -214,33 +214,3 @@ def test_duplicate_migration_name_errors(): pass assert "m001" in str(ex.value) - - -def test_stop_before_applied_migration_errors(migrations): - # Stopping before a migration that has already been applied is - # impossible to honor - previously the stop name was only checked - # against pending migrations, so everything after it was applied - db = sqlite_utils.Database(memory=True) - migrations.apply(db, stop_before="m002") # applies m001 only - with pytest.raises(ValueError) as ex: - migrations.apply(db, stop_before="m001") - assert "m001" in str(ex.value) - assert "already been applied" in str(ex.value) - # Nothing else was applied - assert not db["cats"].exists() - - -def test_stop_before_applied_migration_errors_before_any_apply(migrations): - # The error fires before any pending migration runs, even those that - # come before the already-applied stop target in registration order - db = sqlite_utils.Database(memory=True) - only_second = Migrations("test") - - @only_second() - def m002(db): - db["cats"].create({"name": str}) - - only_second.apply(db) # m002 applied, m001 still pending - with pytest.raises(ValueError): - migrations.apply(db, stop_before="m002") - assert not db["dogs"].exists() diff --git a/tests/test_query.py b/tests/test_query.py index 06847da..cf8d9f5 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -48,22 +48,7 @@ def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): @pytest.mark.parametrize( - "sql", - [ - "begin", - "commit", - "rollback", - "vacuum", - "detach database foo", - "/* comment */ commit", - "-- comment\nbegin", - "/* multi\nline */ -- and another\n vacuum", - "\t /* a */ /* b */ savepoint s1", - "; commit", - ";;\n ; rollback", - "; /* comment */ vacuum", - "\ufeffbegin", - ], + "sql", ["begin", "commit", "rollback", "vacuum", "detach database foo"] ) def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): with pytest.raises(ValueError) as ex: @@ -72,38 +57,6 @@ def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): assert not fresh_db.conn.in_transaction -def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): - # A COMMIT hidden behind a leading comment must not slip past the - # keyword check - previously it committed the caller's open - # transaction before the ValueError was raised - fresh_db["dogs"].insert({"name": "Cleo"}) - fresh_db.begin() - fresh_db.execute("insert into dogs (name) values ('Pancakes')") - with pytest.raises(ValueError): - fresh_db.query("/* comment */ COMMIT") - # The explicit transaction is still open and can still be rolled back - assert fresh_db.conn.in_transaction - fresh_db.rollback() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] - - -@pytest.mark.parametrize("sql", ["; COMMIT", "\ufeffCOMMIT"]) -def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql): - # sqlite3 tolerates empty statements and a UTF-8 BOM before the first - # real token, so the keyword scanner must skip them too - previously - # '; COMMIT' slipped past the check and committed the caller's open - # transaction before raising OperationalError - fresh_db["dogs"].insert({"name": "Cleo"}) - fresh_db.begin() - fresh_db.execute("insert into dogs (name) values ('Pancakes')") - with pytest.raises(ValueError): - fresh_db.query(sql) - # The explicit transaction is still open and can still be rolled back - assert fresh_db.conn.in_transaction - fresh_db.rollback() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] - - def test_query_error_leaves_no_transaction_open(fresh_db): with pytest.raises(sqlite3.OperationalError): fresh_db.query("select * from missing_table") @@ -122,68 +75,6 @@ def test_query_pragma(tmpdir): db.close() -def test_query_rejected_pragma_still_takes_effect(fresh_db): - # Documented limitation: PRAGMAs run outside the savepoint guard, - # because some of them refuse to run inside a transaction - so a - # row-less PRAGMA takes effect even though it raises ValueError. - # If this test starts failing because the pragma was rolled back, - # the limitation has been fixed - update the docs in python-api.rst - # and the query() docstring to remove the carve-out - with pytest.raises(ValueError): - fresh_db.query("pragma user_version = 5") - assert fresh_db.execute("pragma user_version").fetchone()[0] == 5 - - -def test_query_comment_prefixed_pragma(tmpdir): - from sqlite_utils import Database - - db = Database(str(tmpdir / "test.db")) - # A leading comment must not stop a PRAGMA being recognized as one - - # previously it was executed inside the savepoint guard, where - # journal mode changes are refused - assert list(db.query("-- set WAL mode\npragma journal_mode = wal")) == [ - {"journal_mode": "wal"} - ] - db.close() - - -def test_query_comment_prefixed_pragma_inside_transaction(fresh_db): - fresh_db.begin() - assert list(fresh_db.query("-- check version\npragma user_version")) == [ - {"user_version": 0} - ] - assert fresh_db.conn.in_transaction - fresh_db.rollback() - - -@pytest.mark.parametrize( - "sql,expected", - [ - ("select 1", "SELECT"), - (" \t\n select 1", "SELECT"), - ("-- comment\nbegin", "BEGIN"), - ("/* one */ /* two */ pragma user_version", "PRAGMA"), - ("/* multi\nline */vacuum", "VACUUM"), - ("insert into t values (1)", "INSERT"), - ("-- only a comment", ""), - ("/* unterminated", ""), - ("", ""), - (" ", ""), - ("123", ""), - ("; commit", "COMMIT"), - (";;\n ; rollback", "ROLLBACK"), - ("; -- comment\n begin", "BEGIN"), - ("\ufeffcommit", "COMMIT"), - ("\ufeff ; select 1", "SELECT"), - (";", ""), - ], -) -def test_first_keyword(sql, expected): - from sqlite_utils.db import _first_keyword - - assert _first_keyword(sql) == expected - - @pytest.mark.skipif( sqlite3.sqlite_version_info < (3, 35, 0), reason="RETURNING requires SQLite 3.35.0 or higher", @@ -257,22 +148,6 @@ def test_query_insert_returning_respects_explicit_transaction(fresh_db): assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] -def test_query_duplicate_column_names_are_deduped(fresh_db): - # https://github.com/simonw/sqlite-utils/issues/624 - fresh_db["one"].insert({"id": 1, "value": "left"}) - fresh_db["two"].insert({"id": 2, "value": "right"}) - rows = list( - fresh_db.query("select one.id, two.id, one.value, two.value from one, two") - ) - assert rows == [{"id": 1, "id_2": 2, "value": "left", "value_2": "right"}] - - -def test_query_deduped_column_avoids_existing_names(fresh_db): - # The renamed duplicate must not overwrite a real column called id_2 - rows = list(fresh_db.query("select 1 as id, 2 as id, 3 as id_2")) - assert rows == [{"id": 1, "id_3": 2, "id_2": 3}] - - def test_execute_returning_dicts(fresh_db): # Like db.query() but returns a list, included for backwards compatibility # see https://github.com/simonw/sqlite-utils/issues/290 @@ -280,24 +155,3 @@ def test_execute_returning_dicts(fresh_db): assert fresh_db.execute_returning_dicts("select * from test") == [ {"id": 1, "bar": 2} ] - - -@pytest.mark.skipif( - sqlite3.sqlite_version_info < (3, 35, 0), - reason="RETURNING requires SQLite 3.35.0 or higher", -) -def test_query_preserves_error_from_transaction_destroying_trigger(fresh_db): - # RAISE(ROLLBACK) destroys the savepoint guard - the original - # IntegrityError must propagate, not "no such savepoint" - fresh_db.execute("create table t (id integer primary key, v text)") - fresh_db.execute(""" - create trigger no_bad before insert on t - when new.v = 'bad' - begin - select raise(rollback, 'trigger says no'); - end - """) - with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): - fresh_db.query("insert into t (id, v) values (1, 'bad') returning id") - assert not fresh_db.conn.in_transaction - assert fresh_db.execute("select count(*) from t").fetchone()[0] == 0 diff --git a/tests/test_rows.py b/tests/test_rows.py index 46d4f53..a8a4ca0 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -104,37 +104,3 @@ def test_pks_and_rows_where_compound_pk(fresh_db): (("number", 1), {"type": "number", "number": 1, "plusone": 2}), (("number", 2), {"type": "number", "number": 2, "plusone": 3}), ] - - -def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): - # https://github.com/simonw/sqlite-utils/issues/624 - fresh_db["t"].insert({"id": 1, "name": "Cleo"}) - rows = list(fresh_db["t"].rows_where(select="id, id, name")) - assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}] - - -def test_pks_and_rows_where_view(fresh_db): - # pks_and_rows_where() lives on Queryable so views expose it, but - # SQLite views have no rowid. Modern SQLite (3.36+) raises an - # OperationalError from the generated SQL; older versions returned - # NULL for a view's rowid. Either way it must not fail earlier with - # an AttributeError from View lacking Table-only properties - from sqlite_utils.utils import sqlite3 - - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") - fresh_db.create_view("dog_names", "select name from dogs") - try: - result = list(fresh_db["dog_names"].pks_and_rows_where()) - except sqlite3.OperationalError: - pass # SQLite 3.36+: no such column: rowid - else: - # Older SQLite returns NULL rowids for views - assert result == [(None, {"rowid": None, "name": "Cleo"})] - - -def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db): - # Compound pks are returned in PRIMARY KEY declaration order - fresh_db.execute("create table t (b text, a text, primary key (a, b))") - fresh_db["t"].insert({"a": "A", "b": "B"}) - pks_and_rows = list(fresh_db["t"].pks_and_rows_where()) - assert pks_and_rows == [(("A", "B"), {"b": "B", "a": "A"})] diff --git a/tests/test_transform.py b/tests/test_transform.py index 362f1ca..5eb501d 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,6 +1,4 @@ -import sqlite3 - -from sqlite_utils.db import ForeignKey, TransactionError, TransformError +from sqlite_utils.db import ForeignKey, TransformError from sqlite_utils.utils import OperationalError import pytest @@ -226,40 +224,6 @@ def test_transform_rename_pk(fresh_db): ) -def test_transform_preserves_keyword_literal_defaults(fresh_db): - # transform() used to requote keyword-literal defaults (DEFAULT TRUE became - # DEFAULT 'TRUE'), so a default insert stored the text 'TRUE' instead of the - # integer 1 -- silent value corruption on every rebuilt table. - fresh_db.execute( - "CREATE TABLE t (" - " id INTEGER PRIMARY KEY," - " is_active INTEGER DEFAULT TRUE," - " flag INTEGER DEFAULT FALSE," - " note TEXT DEFAULT NULL" - ")" - ) - table = fresh_db["t"] - table.insert({"id": 1}) - before = fresh_db.execute("SELECT is_active, flag, note FROM t").fetchone() - assert before == (1, 0, None) - - # Rebuild the table via an unrelated change. - table.transform(rename={"note": "note2"}) - - # The keyword literals stay unquoted in the schema ... - assert "DEFAULT TRUE" in table.schema - assert "DEFAULT FALSE" in table.schema - assert "DEFAULT NULL" in table.schema - assert "'TRUE'" not in table.schema - - # ... and a fresh default insert still yields 1 / 0 / NULL, not strings. - table.insert({"id": 2}) - after = fresh_db.execute( - "SELECT is_active, flag, note2 FROM t WHERE id = 2" - ).fetchone() - assert after == (1, 0, None) - - def test_transform_not_null(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") @@ -432,165 +396,6 @@ def test_transform_verify_foreign_keys(fresh_db): assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] -@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) -def test_transform_on_delete_cascade_does_not_delete_records( - fresh_db, use_pragma_foreign_keys -): - # Transforming a table drops and recreates it - if another table references - # it with ON DELETE CASCADE and PRAGMA foreign_keys is on, that drop must - # not cascade and delete the referencing records - if use_pragma_foreign_keys: - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE - ); - """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - # Transform the table on the other end of the cascading foreign key - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["authors"].rows) == [ - {"id": 1, "author_name": "Ursula K. Le Guin"} - ] - assert list(fresh_db["books"].rows) == [ - {"id": 1, "title": "The Dispossessed", "author_id": 1} - ] - # Transforming the table with the cascading foreign key should not - # delete its records either - fresh_db["books"].transform(rename={"title": "book_title"}) - assert list(fresh_db["books"].rows) == [ - {"id": 1, "book_title": "The Dispossessed", "author_id": 1} - ] - if use_pragma_foreign_keys: - assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] - - -@pytest.mark.parametrize("on_delete", ["CASCADE", "SET NULL", "SET DEFAULT", "cascade"]) -def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_delete): - # PRAGMA foreign_keys is a no-op inside a transaction, so transforming a - # table referenced by ON DELETE CASCADE / SET NULL / SET DEFAULT foreign - # keys inside an open transaction would fire those actions when the old - # table is dropped - transform() should refuse instead - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE {} - ); - """.format(on_delete)) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - previous_schema = fresh_db["authors"].schema - with fresh_db.atomic(): - with pytest.raises(TransactionError) as excinfo: - fresh_db["authors"].transform(rename={"name": "author_name"}) - message = str(excinfo.value) - assert "books" in message - assert "ON DELETE {}".format(on_delete.upper()) in message - # Nothing should have changed - assert fresh_db["authors"].schema == previous_schema - assert list(fresh_db["books"].rows) == [ - {"id": 1, "title": "The Dispossessed", "author_id": 1} - ] - assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] - - -def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db): - # The copied table carries a foreign key referencing the original table - # name, so a self-referential cascade would wipe the copy too - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" - CREATE TABLE categories ( - id INTEGER PRIMARY KEY, - name TEXT, - parent_id INTEGER REFERENCES categories(id) ON DELETE CASCADE - ); - """) - fresh_db["categories"].insert_all( - [ - {"id": 1, "name": "Fiction", "parent_id": None}, - {"id": 2, "name": "Science Fiction", "parent_id": 1}, - ] - ) - with fresh_db.atomic(): - with pytest.raises(TransactionError) as excinfo: - fresh_db["categories"].transform(rename={"name": "title"}) - assert "categories" in str(excinfo.value) - assert fresh_db["categories"].count == 2 - - -def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db): - # An inbound foreign key without a destructive ON DELETE action is safe - # inside a transaction thanks to PRAGMA defer_foreign_keys - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) - ); - """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["authors"].rows) == [ - {"id": 1, "author_name": "Ursula K. Le Guin"} - ] - assert list(fresh_db["books"].rows) == [ - {"id": 1, "title": "The Dispossessed", "author_id": 1} - ] - assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] - - -def test_transform_in_transaction_allowed_for_child_table(fresh_db): - # The table being transformed only has an outbound foreign key - dropping - # it fires no ON DELETE actions, so this is allowed inside a transaction - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE - ); - """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - with fresh_db.atomic(): - fresh_db["books"].transform(rename={"title": "book_title"}) - assert list(fresh_db["books"].rows) == [ - {"id": 1, "book_title": "The Dispossessed", "author_id": 1} - ] - - -def test_transform_in_transaction_allowed_with_foreign_keys_off(fresh_db): - # With PRAGMA foreign_keys off (the default) no cascades can fire, so - # transform inside a transaction is safe even with a CASCADE schema - fresh_db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE - ); - """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["books"].rows) == [ - {"id": 1, "title": "The Dispossessed", "author_id": 1} - ] - - def test_transform_add_foreign_keys_from_scratch(fresh_db): _add_country_city_continent(fresh_db) fresh_db["places"].insert(_CAVEAU) @@ -727,63 +532,13 @@ def test_transform_preserves_rowids(fresh_db, table_type): assert previous_rows == next_rows -@pytest.mark.parametrize( - "initial_strict,transform_strict,expected_strict", - ( - (False, None, False), - (True, None, True), - (False, True, True), - (True, False, False), - ), -) -def test_transform_strict(fresh_db, initial_strict, transform_strict, expected_strict): - if not fresh_db.supports_strict: - pytest.skip("SQLite version does not support strict tables") - dogs = fresh_db.table("dogs", strict=initial_strict) +@pytest.mark.parametrize("strict", (False, True)) +def test_transform_strict(fresh_db, strict): + dogs = fresh_db.table("dogs", strict=strict) dogs.insert({"id": 1, "name": "Cleo"}) - assert dogs.strict is initial_strict - dogs.transform(strict=transform_strict) - assert dogs.strict is expected_strict - - -def test_transform_to_strict_with_invalid_data(fresh_db): - if not fresh_db.supports_strict: - pytest.skip("SQLite version does not support strict tables") - dogs = fresh_db["dogs"] - dogs.create({"id": int}) - dogs.insert({"id": "not-an-integer"}) - - with pytest.raises(sqlite3.IntegrityError): - dogs.transform(strict=True) - - assert dogs.strict is False - assert list(dogs.rows) == [{"id": "not-an-integer"}] - assert fresh_db.table_names() == ["dogs"] - - -def test_transform_strict_updates_default(fresh_db): - if not fresh_db.supports_strict: - pytest.skip("SQLite version does not support strict tables") - table = fresh_db.table("items", strict=True) - table.create({"id": int}) - - table.transform(strict=False) - assert table.strict is False - - table.create({"id": int}, replace=True) - assert table.strict is False - - -@pytest.mark.parametrize("method_name", ("transform", "transform_sql")) -def test_transform_to_strict_not_supported(fresh_db, method_name): - table = fresh_db["items"] - table.create({"id": int}) - fresh_db._supports_strict = False - - with pytest.raises(TransformError, match="SQLite does not support STRICT tables"): - getattr(table, method_name)(strict=True) - - assert table.strict is False + assert dogs.strict == strict or not fresh_db.supports_strict + dogs.transform(not_null={"name"}) + assert dogs.strict == strict or not fresh_db.supports_strict @pytest.mark.parametrize( diff --git a/tests/test_utils.py b/tests/test_utils.py index 3de5e94..f728bcd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -83,20 +83,3 @@ def test_maximize_csv_field_size_limit(): ) def test_flatten(input, expected): assert utils.flatten(input) == expected - - -@pytest.mark.parametrize( - "input,expected", - ( - ([], []), - (["id", "name"], ["id", "name"]), - (["id", "id"], ["id", "id_2"]), - (["id", "id", "id"], ["id", "id_2", "id_3"]), - # A renamed duplicate must not clobber a real column called id_2 - (["id", "id", "id_2"], ["id", "id_3", "id_2"]), - (["id_2", "id", "id"], ["id_2", "id", "id_3"]), - (["id", "id", "id_2", "id_2"], ["id", "id_3", "id_2", "id_2_2"]), - ), -) -def test_dedupe_keys(input, expected): - assert utils.dedupe_keys(input) == expected diff --git a/tests/test_wal.py b/tests/test_wal.py index 2ddcf54..ee7ecf0 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -49,17 +49,6 @@ def test_disable_wal_inside_transaction_raises(db_path_tmpdir): assert [r["id"] for r in db["test"].rows] == [1] -def test_ensure_autocommit_on(db_path_tmpdir): - db, path, tmpdir = db_path_tmpdir - previous_isolation_level = db.conn.isolation_level - assert previous_isolation_level is not None - with db.ensure_autocommit_on(): - # isolation_level of None means driver-level autocommit mode - assert db.conn.isolation_level is None - # Restored afterwards - assert db.conn.isolation_level == previous_isolation_level - - def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): # Calling enable_wal() when WAL is already enabled is a no-op, # so it is fine inside a transaction @@ -69,20 +58,3 @@ def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): db["test"].insert({"id": 1}, pk="id") db.enable_wal() assert [r["id"] for r in db["test"].rows] == [1] - - -def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): - # Setting isolation_level commits any pending transaction as a side - # effect, silently breaking the caller's rollback guarantee - so - # entering autocommit mode with a transaction open is an error - db, path, tmpdir = db_path_tmpdir - db["test"].insert({"id": 1}, pk="id") - db.begin() - db.execute("insert into test (id) values (2)") - with pytest.raises(TransactionError): - with db.ensure_autocommit_on(): - pass - # The transaction is still open and can still be rolled back - assert db.conn.in_transaction - db.rollback() - assert [r["id"] for r in db["test"].rows] == [1]