diff --git a/.github/actions/setup-sqlite-version/action.yml b/.github/actions/setup-sqlite-version/action.yml new file mode 100644 index 0000000..fdbc71c --- /dev/null +++ b/.github/actions/setup-sqlite-version/action.yml @@ -0,0 +1,39 @@ +name: "Setup SQLite version" +description: "Build and activate a specific SQLite version from its amalgamation archive" +inputs: + version: + description: "The SQLite version to install" + required: true + cflags: + description: "CFLAGS to use when compiling SQLite" + required: false + default: "" + skip-activate: + description: "Set to true to skip modifying the library path" + required: false + default: "false" + fallback-urls: + description: "Whitespace-separated fallback download URLs to try after sqlite.org" + required: false + default: "" +outputs: + sqlite-location: + description: "Directory containing the compiled SQLite library" + value: ${{ steps.build.outputs.sqlite-location }} +runs: + using: "composite" + steps: + - shell: bash + run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/sqlite-versions/downloads + key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1 + - id: build + shell: bash + run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh" + env: + SQLITE_VERSION: ${{ inputs.version }} + SQLITE_CFLAGS: ${{ inputs.cflags }} + SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }} + SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }} diff --git a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh new file mode 100644 index 0000000..0df7290 --- /dev/null +++ b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}" +cflags="${SQLITE_CFLAGS:-}" +skip_activate="${SQLITE_SKIP_ACTIVATE:-false}" +extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}" + +case "$version_spec" in + 3.46 | 3.46.0) + sqlite_version="3.46.0" + sqlite_year="2024" + amalgamation_id="3460000" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip" + ;; + 3.23.1) + sqlite_version="3.23.1" + sqlite_year="2018" + amalgamation_id="3230100" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3230100.zip" + ;; + *) + echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh." + exit 1 + ;; +esac + +case "$(uname -s)" in + Linux) + library_name="libsqlite3.so.0" + library_path_var="LD_LIBRARY_PATH" + ;; + Darwin) + library_name="libsqlite3.dylib" + library_path_var="DYLD_LIBRARY_PATH" + ;; + *) + echo "::error::Unsupported platform $(uname -s)" + exit 1 + ;; +esac + +runner_temp="${RUNNER_TEMP:-}" +if [ -z "$runner_temp" ]; then + runner_temp="$(mktemp -d)" +fi + +filename="sqlite-amalgamation-${amalgamation_id}" +official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip" +download_dir="${runner_temp}/sqlite-versions/downloads" +source_root="${runner_temp}/sqlite-versions/source" +source_dir="${source_root}/${filename}" +build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}" +archive_path="${download_dir}/${filename}.zip" + +mkdir -p "$download_dir" "$source_root" "$build_dir" + +download_archive() { + local url + local candidate_path="${archive_path}.tmp" + local urls=("$official_url") + + for url in $builtin_fallback_urls $extra_fallback_urls; do + urls+=("$url") + done + + rm -f "$candidate_path" + for url in "${urls[@]}"; do + echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}" + if curl \ + --fail \ + --location \ + --show-error \ + --retry 5 \ + --retry-delay 2 \ + --retry-max-time 180 \ + --retry-all-errors \ + --connect-timeout 20 \ + --max-time 240 \ + --output "$candidate_path" \ + "$url"; then + mv "$candidate_path" "$archive_path" + return 0 + fi + + echo "::warning::Download failed from ${url}" + rm -f "$candidate_path" + done + + echo "::error::Could not download SQLite ${sqlite_version} amalgamation" + return 1 +} + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + if [ ! -f "$archive_path" ]; then + download_archive + fi + + rm -rf "$source_dir" + unzip -q "$archive_path" -d "$source_root" +fi + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}" + exit 1 +fi + +read -r -a cflag_args <<< "$cflags" + +echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}" +gcc \ + -fPIC \ + -shared \ + "${cflag_args[@]}" \ + "${source_dir}/sqlite3.c" \ + "-I${source_dir}" \ + -o "${build_dir}/${library_name}" + +if [ "$library_name" = "libsqlite3.so.0" ]; then + ln -sf "$library_name" "${build_dir}/libsqlite3.so" +fi + +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT" +else + echo "sqlite-location=${build_dir}" +fi + +case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in + true | 1 | yes) + echo "Skipping ${library_path_var} activation" + ;; + *) + existing_value="${!library_path_var:-}" + if [ -n "${GITHUB_ENV:-}" ]; then + if [ -n "$existing_value" ]; then + echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV" + else + echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV" + fi + fi + echo "Added ${build_dir} to ${library_path_var}" + ;; +esac diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 0000000..1aab04e --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,16 @@ +name: Read the Docs Pull Request Preview +on: + pull_request_target: + types: + - opened + +permissions: + pull-requests: write + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/actions/preview@v1 + with: + project-slug: "sqlite-utils" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c0bd779..23b23bd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,24 +9,19 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] os: [ubuntu-latest, windows-latest, macos-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev - name: Run tests run: | pytest @@ -34,25 +29,20 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: - python-version: '3.9' - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-publish-pip- + python-version: '3.14' + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install setuptools wheel twine + pip install build twine - name: Publish env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: | - python setup.py sdist bdist_wheel + python -m build twine upload dist/* diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 8a86cd2..2afa7b7 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -6,21 +6,16 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: 3.9 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install -e '.[docs]' + pip install . --group docs - name: Check spelling run: | codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 4b99cc5..7668f1b 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -12,22 +12,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v2 + uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: - python-version: 3.9 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: "3.11" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install SpatiaLite + run: sudo apt-get install libsqlite3-mod-spatialite - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install -e .[test] + python -m pip install . --group dev python -m pip install pytest-cov - name: Run tests run: |- diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml new file mode 100644 index 0000000..f195cba --- /dev/null +++ b/.github/workflows/test-sqlite-support.yml @@ -0,0 +1,41 @@ +name: Test SQLite versions + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ matrix.platform }} + continue-on-error: true + strategy: + matrix: + platform: [ubuntu-latest] + python-version: ["3.10"] + sqlite-version: [ + "3.46", + "3.23.1", # 2018-04-10, before UPSERT + ] + steps: + - uses: actions/checkout@v7 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + cache: pip + cache-dependency-path: pyproject.toml + - name: Set up SQLite ${{ matrix.sqlite-version }} + uses: ./.github/actions/setup-sqlite-version + with: + version: ${{ matrix.sqlite-version }} + cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" + - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)" + - name: Install dependencies + run: | + pip install . --group dev + pip freeze + - name: Run tests + run: | + python -m pytest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5dc3fe1..923de2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,40 +1,57 @@ name: Test -on: [push] +on: [push, pull_request] + +env: + FORCE_COLOR: 1 jobs: test: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15-dev"] numpy: [0, 1] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest, macos-14] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + allow-prereleases: true + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install -e '.[test,mypy,flake8]' + pip install . --group dev - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy + - name: Install SpatiaLite + if: matrix.os == 'ubuntu-latest' + run: sudo apt-get install libsqlite3-mod-spatialite + - name: Build extension for --load-extension test + if: matrix.os == 'ubuntu-latest' + run: |- + (cd tests && gcc ext.c -fPIC -shared -o ext.so && ls -lah) - name: Run tests run: | - pytest + pytest -v + - name: Run autocommit tests just on 3.14/Ubuntu + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' + run: pytest --sqlite-autocommit - name: run mypy - run: mypy sqlite_utils + run: mypy sqlite_utils tests - name: run flake8 run: flake8 + - name: run ty + if: matrix.os != 'windows-latest' && matrix.python-version == '3.14' + run: | + pip install uv + uv run ty check sqlite_utils - name: Check formatting run: black . --check + - name: Check if cog needs to be run + run: | + cog --check --diff README.md docs/*.rst diff --git a/.gitignore b/.gitignore index 75fa28e..6743708 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .venv +dist +build *.db __pycache__/ *.py[cod] @@ -12,3 +14,10 @@ venv .coverage .schema .vscode +.hypothesis +Pipfile +Pipfile.lock +uv.lock +tests/*.dylib +tests/*.so +tests/*.dll diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..08c2f81 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,17 @@ +version: 2 + +sphinx: + configuration: docs/conf.py + +build: + os: ubuntu-24.04 + tools: + python: "3.13" + jobs: + install: + - pip install --upgrade pip + - pip install . --group docs + +formats: +- pdf +- epub diff --git a/Justfile b/Justfile new file mode 100644 index 0000000..5caa120 --- /dev/null +++ b/Justfile @@ -0,0 +1,33 @@ +# Run tests and linters +@default: test lint + +# Run pytest with supplied options +@test *options: + uv run pytest {{options}} + +@run *options: + uv run -- {{options}} + +# Run linters: black, flake8, mypy, ty, cog +@lint: + just run black . --check + uv run flake8 + uv run mypy sqlite_utils tests + uv run ty check sqlite_utils + uv run cog --check README.md docs/*.rst + uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + +# Rebuild docs with cog +@cog: + uv run --group docs cog -r README.md docs/*.rst + +# Serve live docs on localhost:8000 +@docs: cog + #!/usr/bin/env bash + cd docs + uv run --group docs make livehtml + + +# Apply Black +@black: + uv run black . diff --git a/README.md b/README.md index 4c738ab..c444c64 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,29 @@ # sqlite-utils [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) -[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/latest/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/stable/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) -[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest) +[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable) [![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) +[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/Ass7bCAMDw) Python CLI utility and library for manipulating SQLite databases. ## Some feature highlights - [Pipe JSON](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema +- [Run in-memory SQL queries](https://sqlite-utils.datasette.io/en/stable/cli.html#querying-data-directly-using-an-in-memory-database), including joins, directly against data in CSV, TSV or JSON files and view the results - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance -- Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as dropping columns +- Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as changing the type of a column - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data +- [Manage database migrations](https://sqlite-utils.datasette.io/en/stable/migrations.html) using Python migration files and the `sqlite-utils migrate` command +- [Install plugins](https://sqlite-utils.datasette.io/en/stable/plugins.html) to add custom SQL functions and additional features -Read more on my blog: [ -sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). +Upgrading from sqlite-utils 3.x? See the [4.0 upgrade guide](https://sqlite-utils.datasette.io/en/stable/upgrading.html#upgrading-from-3-x-to-4-0). + +Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqlite-utils](https://simonwillison.net/tags/sqlite-utils/). ## Installation @@ -32,12 +37,19 @@ Or if you use [Homebrew](https://brew.sh/) for macOS: Now you can do things with the CLI utility like this: + $ sqlite-utils memory dogs.csv "select * from t" + [{"id": 1, "age": 4, "name": "Cleo"}, + {"id": 2, "age": 2, "name": "Pancakes"}] + + $ sqlite-utils insert dogs.db dogs dogs.csv --csv + [####################################] 100% + $ sqlite-utils tables dogs.db --counts [{"table": "dogs", "count": 2}] - $ sqlite-utils dogs.db "select * from dogs" - [{"id": 1, "age": 4, "name": "Cleo"}, - {"id": 2, "age": 2, "name": "Pancakes"}] + $ sqlite-utils dogs.db "select id, name from dogs" + [{"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Pancakes"}] $ sqlite-utils dogs.db "select * from dogs" --csv id,age,name @@ -61,7 +73,7 @@ Or for data in a CSV file: `sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command: - $ cat dogs.csv | sqlite-utils memory - "select name, age from dogs" + $ cat dogs.csv | sqlite-utils memory - "select name, age from stdin" See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands. diff --git a/docs/Makefile b/docs/Makefile index a279768..5578ae3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,4 +20,4 @@ help: @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) livehtml: - sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) + sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils diff --git a/docs/_static/js/custom.js b/docs/_static/js/custom.js new file mode 100644 index 0000000..8786865 --- /dev/null +++ b/docs/_static/js/custom.js @@ -0,0 +1,23 @@ +jQuery(function ($) { + // Show banner linking to /stable/ if this is a /latest/ page + if (!/\/latest\//.test(location.pathname)) { + return; + } + var stableUrl = location.pathname.replace("/latest/", "/stable/"); + // Check it's not a 404 + fetch(stableUrl, { method: "HEAD" }).then((response) => { + if (response.status == 200) { + var warning = $( + `
+

Note

+

+ This documentation covers the development version of sqlite-utils.

+

See this page for the current stable release. +

+
` + ); + warning.find("a").attr("href", stableUrl); + $("article[role=main]").prepend(warning); + } + }); +}); diff --git a/docs/_templates/base.html b/docs/_templates/base.html new file mode 100644 index 0000000..a253a46 --- /dev/null +++ b/docs/_templates/base.html @@ -0,0 +1,42 @@ +{%- extends "!base.html" %} + +{% block site_meta %} +{{ super() }} + +{% endblock %} + +{% block scripts %} +{{ super() }} + + +{% endblock %} diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a1669d..4c868f4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,7 +1,593 @@ +.. _changelog: + =========== Changelog =========== +.. _v4_1_1: + +4.1.1 (2026-07-12) +------------------ + +- ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) +- The :ref:`CLI ` and :ref:`Python API ` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) +.. _v4_1: + +4.1 (2026-07-11) +---------------- + +- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) +- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) +- New ``table.drop_index(name)`` method and ``sqlite-utils drop-index`` command for dropping an index by name. Both accept ``ignore=True``/``--ignore`` to ignore a missing index. (:issue:`626`) +- ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) +- ``sqlite-utils upsert`` can now infer the primary key of an existing table, so ``--pk`` can be omitted when upserting into a table that already has a primary key. +- ``table.transform()`` and ``table.transform_sql()`` now accept ``strict=True`` or ``strict=False`` to change a table's `SQLite strict mode `__. Omitting the option preserves the existing mode. (:issue:`787`) +- The ``sqlite-utils transform`` command now accepts ``--strict`` and ``--no-strict`` to change a table's strict mode. (:issue:`787`) + +.. _v4_0: + +4.0 (2026-07-07) +---------------- + +The 4.0 release includes some minor backwards-incompatible fixes (hence the major version number bump) and introduces three major new features: + +- :ref:`Database migrations `, providing a structured mechanism for evolving a project's schema over time. (:issue:`752`) +- :ref:`Nested transaction support ` via ``db.atomic()``, plus numerous improvements to how transactions work across the library. (:issue:`755`) +- Support for :ref:`compound foreign keys `, including creation, transformation and introspection through :ref:`table.foreign_keys `. (:issue:`594`) + +Other notable changes include: + +- Upserts now use SQLite's ``INSERT ... ON CONFLICT ... DO UPDATE SET`` syntax, detect existing table primary keys automatically and reject records that are missing required primary key values. (:issue:`652`) +- ``db.query()`` now executes immediately and rejects statements that do not return rows; use ``db.execute()`` for writes and DDL. +- CSV and TSV imports now detect column types by default, while inserts into existing tables preserve those tables' column types. (:issue:`679`) +- Foreign key handling now preserves ``ON DELETE``/``ON UPDATE`` actions during transforms and resolves referenced primary keys more accurately. (:issue:`530`) +- Column names passed to Python API methods are now matched case-insensitively, mirroring SQLite's own identifier behavior. (:issue:`760`) +- The command-line tool now emits UTF-8 JSON output by default, with ``--ascii`` available to restore escaped output. (:issue:`625`) +- ``table.extract()`` and ``extracts=`` no longer create lookup table records for all-``null`` values. (:issue:`186`) + +See :ref:`upgrading_3_to_4` for details on backwards-incompatible changes. + +The detailed release notes for the features and fixes shipped during the 4.0 pre-release cycle are available in :ref:`4.0a0 `, :ref:`4.0a1 `, :ref:`4.0rc1 `, :ref:`4.0rc2 `, :ref:`4.0rc3 ` and :ref:`4.0rc4 `. + +Bug fixes since 4.0rc4 +~~~~~~~~~~~~~~~~~~~~~~ + +- Fixed 4.0 regressions in ``insert``/``upsert`` against tables that use SQLite's implicit ``rowid`` primary key. Passing ``pk="rowid"``, ``pk="_rowid_"`` or ``pk="oid"`` now works again for rowid tables, and ``last_pk`` is set correctly. (:issue:`781`) +- Fixed ``insert(..., ignore=True)`` and ``insert_all(..., ignore=True)`` so an ignored insert that conflicts with an existing primary key row now reports that existing row in ``last_rowid`` and ``last_pk`` where possible. This also works for compound primary keys and list-mode inserts. (:issue:`783`) + +.. _v4_0rc4: + +4.0rc4 (2026-07-06) +------------------- + +- **Breaking change**: ``table.extract()`` - and the ``sqlite-utils extract`` command - no longer extract rows where every extracted column is ``null``. Those rows now keep a ``null`` value in the new foreign key column instead of pointing at an all-``null`` record in the lookup table. When extracting multiple columns, rows are still extracted if at least one of the columns has a value. (:issue:`186`) +- The ``extracts=`` option to ``table.insert()`` and friends no longer creates a lookup table record for ``None`` values - the column value stays ``null``. Previously every batch of inserted rows containing a ``None`` value would add a duplicate ``null`` record to the lookup table. +- Fixed a bug where ``table.lookup()`` inserted a duplicate row on every call if any of the lookup values were ``None``. Lookup values are now compared using ``IS`` so that ``None`` values match existing rows correctly. +- JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) +- ``--no-headers`` now omits the header row from ``--fmt`` and ``--table`` output, not just CSV and TSV output. (:issue:`566`) +- ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) +- Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) +- Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. +- Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. +- Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed exception masking when a statement destroys the enclosing transaction. An error such as a ``RAISE(ROLLBACK)`` trigger or ``INSERT OR ROLLBACK`` conflict rolls back the whole transaction, destroying every savepoint - the cleanup in ``db.atomic()`` and ``db.query()`` then failed with ``OperationalError: no such savepoint`` (or ``cannot rollback - no transaction is active``), hiding the original ``IntegrityError`` from code that tried to catch it. Cleanup now checks whether a transaction is still open first, so the original exception propagates. +- ``sqlite-utils migrate --list`` is now read-only even when the migrations file uses the legacy ``sqlite_migrate.Migrations`` class, whose listing methods create the ``_sqlite_migrations`` table as a side effect. The listing now runs inside a transaction that is rolled back. +- ``sqlite-utils insert ... --pk `` and ``sqlite-utils extract `` now show a clean ``Error:`` message instead of a raw Python traceback. The ``extract`` command also shows a clean error when pointed at a view. +- Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. +- ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. +- ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. +- ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. +- Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. +- Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. +- Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. +- The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. +- ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. +- Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. + +.. _v4_0rc3: + +4.0rc3 (2026-07-05) +------------------- + +Breaking changes +~~~~~~~~~~~~~~~~ + +- :ref:`table.foreign_keys ` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) +- Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. +- The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`) + +Compound foreign key support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Tables can now be created with :ref:`compound foreign keys `, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. +- ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. +- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table. +- ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. + +Other foreign key improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema. +- ``table.add_foreign_key()`` accepts new ``on_delete=`` and ``on_update=`` parameters for creating foreign keys with actions, e.g. ``table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")``. (:issue:`530`) +- Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. +- Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. + +Case-insensitive column matching +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Column names passed to Python API methods are now matched against the table schema case-insensitively, mirroring how SQLite itself treats identifiers. Previously many methods accepted mixed-case identifiers in the SQL they generated but then failed - or silently did nothing - when performing Python-side comparisons against the schema. (:issue:`760`) Fixes include: + +- ``table.insert()`` and ``table.upsert()`` now populate ``table.last_pk`` correctly when the ``pk=`` argument uses different casing to the table schema or the record keys - previously this raised a ``KeyError`` after the row had already been written. +- Upserts no longer raise or misbehave when the casing of ``pk=`` differs from the casing of the record keys. The primary key columns are correctly excluded from the generated ``DO UPDATE SET`` clause. +- ``table.transform()`` arguments ``types=``, ``rename=``, ``drop=``, ``pk=``, ``not_null=``, ``defaults=``, ``column_order=`` and ``drop_foreign_keys=`` all resolve column names case-insensitively. Previously options like ``rename={"name": "title"}`` against a column called ``Name`` were silently ignored. +- ``db.create_table(..., transform=True)`` now recognizes existing columns that differ only by case, instead of attempting to add them again and failing with ``duplicate column name``. The casing used in the existing schema is preserved. +- ``table.lookup()`` returns the primary key value even if ``pk=`` casing differs from the schema, and recognizes existing unique indexes case-insensitively instead of creating redundant ones. +- ``table.extract()`` and ``table.convert()`` - including ``multi=True`` and ``output=`` - accept column names in any casing. +- Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive. +- ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``. + +Everything else +~~~~~~~~~~~~~~~ + +- Fixed a bug where ``table.transform()`` could convert ``DEFAULT TRUE``, ``DEFAULT FALSE`` and ``DEFAULT NULL`` column defaults into quoted string defaults when rebuilding a table. Thanks, `Vincent Gao `__. (`#764 `__) + +.. _v4_0rc2: + +4.0rc2 (2026-07-04) +------------------- + +Breaking changes: + +- Write statements executed with ``db.execute()`` are now committed automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that stayed open until something committed it - writes appeared to work when read on the same connection but were silently rolled back when the connection closed. Code that relied on rolling back uncommitted ``db.execute()`` writes should use the new ``db.begin()`` method to open an explicit transaction first. The transaction model is documented in full at :ref:`python_api_transactions`. +- ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` are executed and committed immediately without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. A statement rejected this way is rolled back before the error is raised, so it has no effect on the database. +- Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead. +- ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place. +- ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions. +- The ``View`` class no longer has an ``enable_fts()`` method. It existed only to raise ``NotImplementedError``, since full-text search is not supported for views - calling it now raises ``AttributeError`` instead, and the method no longer appears in the API reference. The ``sqlite-utils enable-fts`` command shows a clean error when pointed at a view. +- The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection. +- ``Database()`` now raises a ``sqlite_utils.db.TransactionError`` if passed a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options. ``commit()`` and ``rollback()`` behave differently on those connections, which previously caused every write made by the library to be silently discarded when the connection closed. + +Everything else: + +- Fixed a bug where ``table.delete_where()``, ``table.optimize()`` and ``table.rebuild_fts()`` did not commit their changes, leaving the connection inside an open transaction. Their work - and any subsequent writes - could then be silently rolled back when the connection was closed. All three now use ``db.atomic()``, consistent with the other write methods. +- The ``sqlite-utils drop-table`` command now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. Both now exit with an error suggesting the correct command to use. +- Migrations applied by the new :ref:`migrations system ` now run inside a transaction, together with the record of the migration having been applied. If a migration raises an exception its changes are rolled back and it stays pending, so it can be safely re-applied after the error is fixed. Migrations that cannot run inside a transaction, such as those executing ``VACUUM``, can opt out using ``@migrations(transactional=False)`` - see :ref:`migrations_transactions`. +- ``table.upsert()`` and ``table.upsert_all()`` now detect the primary key or compound primary key of an existing table, so the ``pk=`` argument is no longer required when upserting into a table that already has a primary key. +- ``db.table(table_name).insert({})`` can now be used to insert a row consisting entirely of default values into an existing table, using ``INSERT INTO ... DEFAULT VALUES``. (:issue:`759`) +- Improvements to the ``sqlite-utils migrate`` command: ``--stop-before`` values that do not match any known migration are now an error instead of being silently ignored, ``--stop-before`` now works correctly with migration files that still use the older ``sqlite_migrate.Migrations`` class, and ``--list`` is now a read-only operation that no longer creates the database file or the migrations tracking table. ``migrations.applied()`` now returns migrations in the order they were applied. +- New ``db.begin()``, ``db.commit()`` and ``db.rollback()`` methods for taking manual control of transactions, as an alternative to the ``db.atomic()`` context manager. +- New documentation: :ref:`python_api_transactions` describes how transactions work and when changes are committed, and a new :ref:`upgrading` page details the changes needed to move between major versions. + +.. _v4_0rc1: + +4.0rc1 (2026-06-21) +------------------- + +- New :ref:`database migrations system `, incorporating functionality that was previously provided by the separate `sqlite-migrate `__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using the ``sqlite-utils migrate`` command or the :ref:`migrations Python API `. (:issue:`752`) +- New ``db.atomic()`` :ref:`context manager providing nested transaction support ` using SQLite transactions and savepoints. Internal multi-step operations such as ``table.transform()`` now use this mechanism to avoid unexpectedly committing an existing transaction. (:issue:`755`) +- ``Database`` objects can now be :ref:`used as context managers `, automatically closing the connection when the ``with`` block exits. The CLI also now closes database and file handles more reliably, resolving a number of ``ResourceWarning`` warnings. (:issue:`692`) +- The ``sqlite-utils convert`` command can now accept a direct callable reference such as ``r.parsedate`` or ``json.loads --import json`` as the conversion code, as an alternative to calling it explicitly with ``r.parsedate(value)``. (:issue:`686`) +- Fixed a bug where CSV or TSV files with only a header row could crash ``sqlite-utils insert`` and ``sqlite-utils memory`` when type detection was enabled. Thanks, `Rami Abdelrazzaq `__. (:issue:`702`, `#707 `__) +- Fixed a bug where installed plugins could be loaded while running the test suite, despite the test-mode safeguard that disables plugin loading. Thanks, `Rami Abdelrazzaq `__. (:issue:`713`, `#719 `__) +- ``table.detect_fts()`` now recognizes legacy FTS virtual tables that quote the ``content=`` table name using square brackets, allowing ``table.enable_fts(..., replace=True)`` to replace them correctly. (:issue:`694`) +- Now depends on Click 8.3.1 or later, removing compatibility workarounds for Click's ``Sentinel`` default values. (:issue:`666`) +- Improved type annotations throughout the package, with ``ty`` now run in CI. (:issue:`697`) +- Development tooling now uses ``uv`` dependency groups, with separate ``dev`` and ``docs`` groups. (:issue:`691`) +- The test suite now runs against Python 3.15-dev. (:issue:`738`) + +.. _v3_39: + +3.39 (2025-11-24) +----------------- + +- Fixed a bug with ``sqlite-utils install`` when the tool had been installed using ``uv``. (:issue:`687`) +- The ``--functions`` argument now optionally accepts a path to a Python file as an alternative to a string full of code, and can be specified multiple times - see :ref:`cli_query_functions`. (:issue:`659`) +- ``sqlite-utils`` now requires Python 3.10 or higher. + +.. _v4_0a1: + +4.0a1 (2025-11-23) +------------------ + +- **Breaking change**: The ``db.table(table_name)`` method now only works with tables. To access a SQL view use ``db.view(view_name)`` instead. (:issue:`657`) +- The ``table.insert_all()`` and ``table.upsert_all()`` methods can now accept an iterator of lists or tuples as an alternative to dictionaries. The first item should be a list/tuple of column names. See :ref:`python_api_insert_lists` for details. (:issue:`672`) +- **Breaking change**: The default floating point column type has been changed from ``FLOAT`` to ``REAL``, which is the correct SQLite type for floating point values. This affects auto-detected columns when inserting data. (:issue:`645`) +- Now uses ``pyproject.toml`` in place of ``setup.py`` for packaging. (:issue:`675`) +- Tables in the Python API now do a much better job of remembering the primary key and other schema details from when they were first created. (:issue:`655`) +- **Breaking change**: The ``table.convert()`` and ``sqlite-utils convert`` mechanisms no longer skip values that evaluate to ``False``. Previously the ``--skip-false`` option was needed, this has been removed. (:issue:`542`) +- **Breaking change**: Tables created by this library now wrap table and column names in ``"double-quotes"`` in the schema. Previously they would use ``[square-braces]``. (:issue:`677`) +- The ``--functions`` CLI argument now accepts a path to a Python file in addition to accepting a string full of Python code. It can also now be specified multiple times. (:issue:`659`) +- **Breaking change:** Type detection is now the default behavior for the ``insert`` and ``upsert`` CLI commands when importing CSV or TSV data. Previously all columns were treated as ``TEXT`` unless the ``--detect-types`` flag was passed. Use the new ``--no-detect-types`` flag to restore the old behavior. The ``SQLITE_UTILS_DETECT_TYPES`` environment variable has been removed. (:issue:`679`) + +.. _v4_0a0: + +4.0a0 (2025-05-08) +------------------ + +- Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax on all SQLite versions later than 3.23.1. This is a very slight breaking change for apps that depend on the previous ``INSERT OR IGNORE`` followed by ``UPDATE`` behavior. (:issue:`652`) +- Python library users can opt-in to the previous implementation by passing ``use_old_upsert=True`` to the ``Database()`` constructor, see :ref:`python_api_old_upsert`. +- Dropped support for Python 3.8, added support for Python 3.13. (:issue:`646`) +- ``sqlite-utils tui`` is now provided by the `sqlite-utils-tui `__ plugin. (:issue:`648`) +- Test suite now also runs against SQLite 3.23.1, the last version (from 2018-04-10) before the new ``INSERT ... ON CONFLICT SET`` syntax was added. (:issue:`654`) + +.. _v3_38: + +3.38 (2024-11-23) +----------------- + +- Plugins can now reuse the implementation of the ``sqlite-utils memory`` CLI command with the new ``return_db=True`` parameter. (:issue:`643`) +- ``table.transform()`` now recreates indexes after transforming a table. A new ``sqlite_utils.db.TransformError`` exception is raised if these indexes cannot be recreated due to conflicting changes to the table such as a column rename. Thanks, `Mat Miller `__. (:issue:`633`) +- ``table.search()`` now accepts a ``include_rank=True`` parameter, causing the resulting rows to have a ``rank`` column showing the calculated relevance score. Thanks, `liunux4odoo `__. (`#628 `__) +- Fixed an error that occurred when creating a strict table with at least one floating point column. These ``FLOAT`` columns are now correctly created as ``REAL`` as well, but only for strict tables. (:issue:`644`) + +.. _v3_37: + +3.37 (2024-07-18) +----------------- + +- The ``create-table`` and ``insert-files`` commands all now accept multiple ``--pk`` options for compound primary keys. (:issue:`620`) +- Now tested against Python 3.13 pre-release. (`#619 `__) +- Fixed a crash that can occur in environments with a broken ``numpy`` installation, producing a ``module 'numpy' has no attribute 'int8'``. (:issue:`632`) + +.. _v3_36: + +3.36 (2023-12-07) +----------------- + +- Support for creating tables in `SQLite STRICT mode `__. Thanks, `Taj Khattra `__. (:issue:`344`) + - CLI commands ``create-table``, ``insert`` and ``upsert`` all now accept a ``--strict`` option. + - Python methods that can create a table - ``table.create()`` and ``insert/upsert/insert_all/upsert_all`` all now accept an optional ``strict=True`` parameter. + - The ``transform`` command and ``table.transform()`` method preserve strict mode when transforming a table. +- The ``sqlite-utils create-table`` command now accepts ``str``, ``int`` and ``bytes`` as aliases for ``text``, ``integer`` and ``blob`` respectively. (:issue:`606`) + +.. _v3_35_2: + +3.35.2 (2023-11-03) +------------------- + +- The ``--load-extension=spatialite`` option and :ref:`find_spatialite() ` utility function now both work correctly on ``arm64`` Linux. Thanks, `Mike Coats `__. (:issue:`599`) +- Fix for bug where ``sqlite-utils insert`` could cause your terminal cursor to disappear. Thanks, `Luke Plant `__. (:issue:`433`) +- ``datetime.timedelta`` values are now stored as ``TEXT`` columns. Thanks, `Harald Nezbeda `__. (:issue:`522`) +- Test suite is now also run against Python 3.12. + +.. _v3_35_1: + +3.35.1 (2023-09-08) +------------------- + +- Fixed a bug where :ref:`table.transform() ` would sometimes re-assign the ``rowid`` values for a table rather than keeping them consistent across the operation. (:issue:`592`) + +.. _v3_35: + +3.35 (2023-08-17) +----------------- + +Adding foreign keys to a table no longer uses ``PRAGMA writable_schema = 1`` to directly manipulate the ``sqlite_master`` table. This was resulting in errors in some Python installations where the SQLite library was compiled in a way that prevented this from working, in particular on macOS. Foreign keys are now added using the :ref:`table transformation ` mechanism instead. (:issue:`577`) + +This new mechanism creates a full copy of the table, so it is likely to be significantly slower for large tables, but will no longer trigger ``table sqlite_master may not be modified`` errors on platforms that do not support ``PRAGMA writable_schema = 1``. + +A new plugin, `sqlite-utils-fast-fks `__, is now available for developers who still want to use that faster but riskier implementation. + +Other changes: + +- The :ref:`table.transform() method ` has two new parameters: ``foreign_keys=`` allows you to replace the foreign key constraints defined on a table, and ``add_foreign_keys=`` lets you specify new foreign keys to add. These complement the existing ``drop_foreign_keys=`` parameter. (:issue:`577`) +- The :ref:`sqlite-utils transform ` command has a new ``--add-foreign-key`` option which can be called multiple times to add foreign keys to a table that is being transformed. (:issue:`585`) +- :ref:`sqlite-utils convert ` now has a ``--pdb`` option for opening a debugger on the first encountered error in your conversion script. (:issue:`581`) +- Fixed a bug where ``sqlite-utils install -e '.[test]'`` option did not work correctly. + +.. _v3_34: + +3.34 (2023-07-22) +----------------- + +This release introduces a new :ref:`plugin system `. Read more about this in `sqlite-utils now supports plugins `__. (:issue:`567`) + +- Documentation describing :ref:`how to build a plugin `. +- Plugin hook: :ref:`plugins_hooks_register_commands`, for plugins to add extra commands to ``sqlite-utils``. (:issue:`569`) +- Plugin hook: :ref:`plugins_hooks_prepare_connection`. Plugins can use this to help prepare the SQLite connection to do things like registering custom SQL functions. Thanks, `Alex Garcia `__. (:issue:`574`) +- ``sqlite_utils.Database(..., execute_plugins=False)`` option for disabling plugin execution. (:issue:`575`) +- ``sqlite-utils install -e path-to-directory`` option for installing editable code. This option is useful during the development of a plugin. (:issue:`570`) +- ``table.create(...)`` method now accepts ``replace=True`` to drop and replace an existing table with the same name, or ``ignore=True`` to silently do nothing if a table already exists with the same name. (:issue:`568`) +- ``sqlite-utils insert ... --stop-after 10`` option for stopping the insert after a specified number of records. Works for the ``upsert`` command as well. (:issue:`561`) +- The ``--csv`` and ``--tsv`` modes for ``insert`` now accept a ``--empty-null`` option, which causes empty strings in the CSV file to be stored as ``null`` in the database. (:issue:`563`) +- New ``db.rename_table(table_name, new_name)`` method for renaming tables. (:issue:`565`) +- ``sqlite-utils rename-table my.db table_name new_name`` command for renaming tables. (:issue:`565`) +- The ``table.transform(...)`` method now takes an optional ``keep_table=new_table_name`` parameter, which will cause the original table to be renamed to ``new_table_name`` rather than being dropped at the end of the transformation. (:issue:`571`) +- Documentation now notes that calling ``table.transform()`` without any arguments will reformat the SQL schema stored by SQLite to be more aesthetically pleasing. (:issue:`564`) + +.. _v3_33: + +3.33 (2023-06-25) +----------------- + +- ``sqlite-utils`` will now use `sqlean.py `__ in place of ``sqlite3`` if it is installed in the same virtual environment. This is useful for Python environments with either an outdated version of SQLite or with restrictions on SQLite such as disabled extension loading or restrictions resulting in the ``sqlite3.OperationalError: table sqlite_master may not be modified`` error. (:issue:`559`) +- New ``with db.ensure_autocommit_off()`` context manager, which ensures that the database is in autocommit mode for the duration of a block of code. This is used by ``db.enable_wal()`` and ``db.disable_wal()`` to ensure they work correctly with ``pysqlite3`` and ``sqlean.py``. +- New ``db.iterdump()`` method, providing an iterator over SQL strings representing a dump of the database. This uses ``sqlite-dump`` if it is available, otherwise falling back on the ``conn.iterdump()`` method from ``sqlite3``. Both ``pysqlite3`` and ``sqlean.py`` omit support for ``iterdump()`` - this method helps paper over that difference. + +.. _v3_32_1: + +3.32.1 (2023-05-21) +------------------- + +- Examples in the :ref:`CLI documentation ` can now all be copied and pasted without needing to remove a leading ``$``. (:issue:`551`) +- Documentation now covers :ref:`installation_completion` for ``bash`` and ``zsh``. (:issue:`552`) + +.. _v3_32: + +3.32 (2023-05-21) +----------------- + +- New experimental ``sqlite-utils tui`` interface for interactively building command-line invocations, powered by `Trogon `__. This requires an optional dependency, installed using ``sqlite-utils install trogon``. (:issue:`545`) +- ``sqlite-utils analyze-tables`` command (:ref:`documentation `) now has a ``--common-limit 20`` option for changing the number of common/least-common values shown for each column. (:issue:`544`) +- ``sqlite-utils analyze-tables --no-most`` and ``--no-least`` options for disabling calculation of most-common and least-common values. +- If a column contains only ``null`` values, ``analyze-tables`` will no longer attempt to calculate the most common and least common values for that column. (:issue:`547`) +- Calling ``sqlite-utils analyze-tables`` with non-existent columns in the ``-c/--column`` option now results in an error message. (:issue:`548`) +- The ``table.analyze_column()`` method (:ref:`documented here `) now accepts ``most_common=False`` and ``least_common=False`` options for disabling calculation of those values. + +.. _v3_31: + +3.31 (2023-05-08) +----------------- + +- Dropped support for Python 3.6. Tests now ensure compatibility with Python 3.11. (:issue:`517`) +- Automatically locates the SpatiaLite extension on Apple Silicon. Thanks, Chris Amico. (`#536 `__) +- New ``--raw-lines`` option for the ``sqlite-utils query`` and ``sqlite-utils memory`` commands, which outputs just the raw value of the first column of every row. (:issue:`539`) +- Fixed a bug where ``table.upsert_all()`` failed if the ``not_null=`` option was passed. (:issue:`538`) +- Fixed a ``ResourceWarning`` when using ``sqlite-utils insert``. (:issue:`534`) +- Now shows a more detailed error message when ``sqlite-utils insert`` is called with invalid JSON. (:issue:`532`) +- ``table.convert(..., skip_false=False)`` and ``sqlite-utils convert --no-skip-false`` options, for avoiding a misfeature where the :ref:`convert() ` mechanism skips rows in the database with a falsey value for the specified column. Fixing this by default would be a backwards-incompatible change and is under consideration for a 4.0 release in the future. (:issue:`527`) +- Tables can now be created with self-referential foreign keys. Thanks, Scott Perry. (`#537 `__) +- ``sqlite-utils transform`` no longer breaks if a table defines default values for columns. Thanks, Kenny Song. (:issue:`509`) +- Fixed a bug where repeated calls to ``table.transform()`` did not work correctly. Thanks, Martin Carpenter. (:issue:`525`) +- Improved error message if ``rows_from_file()`` is passed a non-binary-mode file-like object. (:issue:`520`) + +.. _v3_30: + +3.30 (2022-10-25) +----------------- + +- Now tested against Python 3.11. (:issue:`502`) +- New ``table.search_sql(include_rank=True)`` option, which adds a ``rank`` column to the generated SQL. Thanks, Jacob Chapman. (`#480 `__) +- Progress bars now display for newline-delimited JSON files using the ``--nl`` option. Thanks, Mischa Untaga. (:issue:`485`) +- New ``db.close()`` method. (:issue:`504`) +- Conversion functions passed to :ref:`table.convert(...) ` can now return lists or dictionaries, which will be inserted into the database as JSON strings. (:issue:`495`) +- ``sqlite-utils install`` and ``sqlite-utils uninstall`` commands for installing packages into the same virtual environment as ``sqlite-utils``, :ref:`described here `. (:issue:`483`) +- New :ref:`sqlite_utils.utils.flatten() ` utility function. (:issue:`500`) +- Documentation on :ref:`using Just ` to run tests, linters and build documentation. +- Documentation now covers the :ref:`release_process` for this package. + +.. _v3_29: + +3.29 (2022-08-27) +----------------- + +- The ``sqlite-utils query``, ``memory`` and ``bulk`` commands now all accept a new ``--functions`` option. This can be passed a string of Python code, and any callable objects defined in that code will be made available to SQL queries as custom SQL functions. See :ref:`cli_query_functions` for details. (:issue:`471`) +- ``db[table].create(...)`` method now accepts a new ``transform=True`` parameter. If the table already exists it will be :ref:`transformed ` to match the schema configuration options passed to the function. This may result in columns being added or dropped, column types being changed, column order being updated or not null and default values for columns being set. (:issue:`467`) +- Related to the above, the ``sqlite-utils create-table`` command now accepts a ``--transform`` option. +- New introspection property: ``table.default_values`` returns a dictionary mapping each column name with a default value to the configured default value. (:issue:`475`) +- The ``--load-extension`` option can now be provided a path to a compiled SQLite extension module accompanied by the name of an entrypoint, separated by a colon - for example ``--load-extension ./lines0:sqlite3_lines0_noread_init``. This feature is modelled on code first `contributed to Datasette `__ by Alex Garcia. (:issue:`470`) +- Functions registered using the :ref:`db.register_function() ` method can now have a custom name specified using the new ``db.register_function(fn, name=...)`` parameter. (:issue:`458`) +- :ref:`sqlite-utils rows ` has a new ``--order`` option for specifying the sort order for the returned rows. (:issue:`469`) +- All of the CLI options that accept Python code blocks can now all be used to define functions that can access modules imported in that same block of code without needing to use the ``global`` keyword. (:issue:`472`) +- Fixed bug where ``table.extract()`` would not behave correctly for columns containing null values. Thanks, Forest Gregg. (:issue:`423`) +- New tutorial: `Cleaning data with sqlite-utils and Datasette `__ shows how to use ``sqlite-utils`` to import and clean an example CSV file. +- Datasette and ``sqlite-utils`` now have a Discord community. `Join the Discord here `__. + +.. _v3_28: + +3.28 (2022-07-15) +----------------- + +- New :ref:`table.duplicate(new_name) ` method for creating a copy of a table with a matching schema and row contents. Thanks, `David `__. (:issue:`449`) +- New ``sqlite-utils duplicate data.db table_name new_name`` CLI command for :ref:`cli_duplicate_table`. (:issue:`454`) +- ``sqlite_utils.utils.rows_from_file()`` is now a :ref:`documented API `. It can be used to read a sequence of dictionaries from a file-like object containing CSV, TSV, JSON or newline-delimited JSON. It can be passed an explicit format or can attempt to detect the format automatically. (:issue:`443`) +- ``sqlite_utils.utils.TypeTracker`` is now a documented API for detecting the likely column types for a sequence of string rows, see :ref:`python_api_typetracker`. (:issue:`445`) +- ``sqlite_utils.utils.chunks()`` is now a documented API for :ref:`splitting an iterator into chunks `. (:issue:`451`) +- ``sqlite-utils enable-fts`` now has a ``--replace`` option for replacing the existing FTS configuration for a table. (:issue:`450`) +- The ``create-index``, ``add-column`` and ``duplicate`` commands all now take a ``--ignore`` option for ignoring errors should the database not be in the right state for them to operate. (:issue:`450`) + +.. _v3_27: + +3.27 (2022-06-14) +----------------- + +See also `the annotated release notes `__ for this release. + +- Documentation now uses the `Furo `__ Sphinx theme. (:issue:`435`) +- Code examples in documentation now have a "copy to clipboard" button. (:issue:`436`) +- ``sqlite_utils.utils.utils.rows_from_file()`` is now a documented API, see :ref:`python_api_rows_from_file`. (:issue:`443`) +- ``rows_from_file()`` has two new parameters to help handle CSV files with rows that contain more values than are listed in that CSV file's headings: ``ignore_extras=True`` and ``extras_key="name-of-key"``. (:issue:`440`) +- ``sqlite_utils.utils.maximize_csv_field_size_limit()`` helper function for increasing the field size limit for reading CSV files to its maximum, see :ref:`python_api_maximize_csv_field_size_limit`. (:issue:`442`) +- ``table.search(where=, where_args=)`` parameters for adding additional ``WHERE`` clauses to a search query. The ``where=`` parameter is available on ``table.search_sql(...)`` as well. See :ref:`python_api_fts_search`. (:issue:`441`) +- Fixed bug where ``table.detect_fts()`` and other search-related functions could fail if two FTS-enabled tables had names that were prefixes of each other. (:issue:`434`) + +.. _v3_26_1: + +3.26.1 (2022-05-02) +------------------- + +- Now depends on `click-default-group-wheel `__, a pure Python wheel package. This means you can install and use this package with `Pyodide `__, which can run Python entirely in your browser using WebAssembly. (`#429 `__) + + Try that out using the `Pyodide REPL `__: + + .. code-block:: python + + >>> import micropip + >>> await micropip.install("sqlite-utils") + >>> import sqlite_utils + >>> db = sqlite_utils.Database(memory=True) + >>> list(db.query("select 3 * 5")) + [{'3 * 5': 15}] + +.. _v3_26: + +3.26 (2022-04-13) +----------------- + +- New ``errors=r.IGNORE/r.SET_NULL`` parameter for the ``r.parsedatetime()`` and ``r.parsedate()`` :ref:`convert recipes `. (:issue:`416`) +- Fixed a bug where ``--multi`` could not be used in combination with ``--dry-run`` for the :ref:`convert ` command. (:issue:`415`) +- New documentation: :ref:`cli_convert_complex`. (:issue:`420`) +- More robust detection for whether or not ``deterministic=True`` is supported. (:issue:`425`) + +.. _v3_25_1: + +3.25.1 (2022-03-11) +------------------- + +- Improved display of type information and parameters in the :ref:`API reference documentation `. (:issue:`413`) + +.. _v3_25: + +3.25 (2022-03-01) +----------------- + +- New ``hash_id_columns=`` parameter for creating a primary key that's a hash of the content of specific columns - see :ref:`python_api_hash` for details. (:issue:`343`) +- New :ref:`db.sqlite_version ` property, returning a tuple of integers representing the version of SQLite, for example ``(3, 38, 0)``. +- Fixed a bug where :ref:`register_function(deterministic=True) ` caused errors on versions of SQLite prior to 3.8.3. (:issue:`408`) +- New documented :ref:`hash_record(record, keys=...) ` function. + +.. _v3_24: + +3.24 (2022-02-15) +----------------- + +- SpatiaLite helpers for the ``sqlite-utils`` command-line tool - thanks, Chris Amico. (:issue:`398`) + + - :ref:`sqlite-utils create-database ` ``--init-spatialite`` option for initializing SpatiaLite on a newly created database. + - :ref:`sqlite-utils add-geometry-column ` command for adding geometry columns. + - :ref:`sqlite-utils create-spatial-index ` command for adding spatial indexes. + +- ``db[table].create(..., if_not_exists=True)`` option for :ref:`creating a table ` only if it does not already exist. (:issue:`397`) +- ``Database(memory_name="my_shared_database")`` parameter for creating a :ref:`named in-memory database ` that can be shared between multiple connections. (:issue:`405`) +- Documentation now describes :ref:`how to add a primary key to a rowid table ` using ``sqlite-utils transform``. (:issue:`403`) + +.. _v3_23: + +3.23 (2022-02-03) +----------------- + +This release introduces four new utility methods for working with `SpatiaLite `__. Thanks, Chris Amico. (`#385 `__) + +- ``sqlite_utils.utils.find_spatialite()`` :ref:`finds the location of the SpatiaLite module ` on disk. +- ``db.init_spatialite()`` :ref:`initializes SpatiaLite ` for the given database. +- ``table.add_geometry_column(...)`` :ref:`adds a geometry column ` to an existing table. +- ``table.create_spatial_index(...)`` :ref:`creates a spatial index ` for a column. +- ``sqlite-utils batch`` now accepts a ``--batch-size`` option. (:issue:`392`) + +.. _v3_22_1: + +3.22.1 (2022-01-25) +------------------- + +- All commands now include example usage in their ``--help`` - see :ref:`cli_reference`. (:issue:`384`) +- Python library documentation has a new :ref:`python_api_getting_started` section. (:issue:`387`) +- Documentation now uses `Plausible analytics `__. (:issue:`389`) + +.. _v3_22: + +3.22 (2022-01-11) +----------------- + +- New :ref:`cli_reference` documentation page, listing the output of ``--help`` for every one of the CLI commands. (:issue:`383`) +- ``sqlite-utils rows`` now has ``--limit`` and ``--offset`` options for paginating through data. (:issue:`381`) +- ``sqlite-utils rows`` now has ``--where`` and ``-p`` options for filtering the table using a ``WHERE`` query, see :ref:`cli_rows`. (:issue:`382`) + +.. _v3_21: + +3.21 (2022-01-10) +----------------- + +CLI and Python library improvements to help run `ANALYZE `__ after creating indexes or inserting rows, to gain better performance from the SQLite query planner when it runs against indexes. + +Three new CLI commands: ``create-database``, ``analyze`` and ``bulk``. + +More details and examples can be found in `the annotated release notes `__. + +- New ``sqlite-utils create-database`` command for creating new empty database files. (:issue:`348`) +- New Python methods for running ``ANALYZE`` against a database, table or index: ``db.analyze()`` and ``table.analyze()``, see :ref:`python_api_analyze`. (:issue:`366`) +- New :ref:`sqlite-utils analyze command ` for running ``ANALYZE`` using the CLI. (:issue:`379`) +- The ``create-index``, ``insert`` and ``upsert`` commands now have a new ``--analyze`` option for running ``ANALYZE`` after the command has completed. (:issue:`379`) +- New :ref:`sqlite-utils bulk command ` which can import records in the same way as ``sqlite-utils insert`` (from JSON, CSV or TSV) and use them to bulk execute a parametrized SQL query. (:issue:`375`) +- The CLI tool can now also be run using ``python -m sqlite_utils``. (:issue:`368`) +- Using ``--fmt`` now implies ``--table``, so you don't need to pass both options. (:issue:`374`) +- The ``--convert`` function applied to rows can now modify the row in place. (:issue:`371`) +- The :ref:`insert-files command ` supports two new columns: ``stem`` and ``suffix``. (:issue:`372`) +- The ``--nl`` import option now ignores blank lines in the input. (:issue:`376`) +- Fixed bug where streaming input to the ``insert`` command with ``--batch-size 1`` would appear to only commit after several rows had been ingested, due to unnecessary input buffering. (:issue:`364`) + +.. _v3_20: + +3.20 (2022-01-05) +----------------- + +- ``sqlite-utils insert ... --lines`` to insert the lines from a file into a table with a single ``line`` column, see :ref:`cli_insert_unstructured`. +- ``sqlite-utils insert ... --text`` to insert the contents of the file into a table with a single ``text`` column and a single row. +- ``sqlite-utils insert ... --convert`` allows a Python function to be provided that will be used to convert each row that is being inserted into the database. See :ref:`cli_insert_convert`, including details on special behavior when combined with ``--lines`` and ``--text``. (:issue:`356`) +- ``sqlite-utils convert`` now accepts a code value of ``-`` to read code from standard input. (:issue:`353`) +- ``sqlite-utils convert`` also now accepts code that defines a named ``convert(value)`` function, see :ref:`cli_convert`. +- ``db.supports_strict`` property showing if the database connection supports `SQLite strict tables `__. +- ``table.strict`` property (see :ref:`python_api_introspection_strict`) indicating if the table uses strict mode. (:issue:`344`) +- Fixed bug where ``sqlite-utils upsert ... --detect-types`` ignored the ``--detect-types`` option. (:issue:`362`) + +.. _v3_19: + +3.19 (2021-11-20) +----------------- + +- The :ref:`table.lookup() method ` now accepts keyword arguments that match those on the underlying ``table.insert()`` method: ``foreign_keys=``, ``column_order=``, ``not_null=``, ``defaults=``, ``extracts=``, ``conversions=`` and ``columns=``. You can also now pass ``pk=`` to specify a different column name to use for the primary key. (:issue:`342`) + +.. _v3_18: + +3.18 (2021-11-14) +----------------- + +- The ``table.lookup()`` method now has an optional second argument which can be used to populate columns only the first time the record is created, see :ref:`python_api_lookup_tables`. (:issue:`339`) +- ``sqlite-utils memory`` now has a ``--flatten`` option for :ref:`flattening nested JSON objects ` into separate columns, consistent with ``sqlite-utils insert``. (:issue:`332`) +- ``table.create_index(..., find_unique_name=True)`` parameter, which finds an available name for the created index even if the default name has already been taken. This means that ``index-foreign-keys`` will work even if one of the indexes it tries to create clashes with an existing index name. (:issue:`335`) +- Added ``py.typed`` to the module, so `mypy `__ should now correctly pick up the type annotations. Thanks, Andreas Longo. (:issue:`331`) +- Now depends on ``python-dateutil`` instead of depending on ``dateutils``. Thanks, Denys Pavlov. (:issue:`324`) +- ``table.create()`` (see :ref:`python_api_explicit_create`) now handles ``dict``, ``list`` and ``tuple`` types, mapping them to ``TEXT`` columns in SQLite so that they can be stored encoded as JSON. (:issue:`338`) +- Inserted data with square braces in the column names (for example a CSV file containing a ``item[price]``) column now have the braces converted to underscores: ``item_price_``. Previously such columns would be rejected with an error. (:issue:`329`) +- Now also tested against Python 3.10. (`#330 `__) + +.. _v3_17.1: + +3.17.1 (2021-09-22) +------------------- + +- :ref:`sqlite-utils memory ` now works if files passed to it share the same file name. (:issue:`325`) +- :ref:`sqlite-utils query ` now returns ``[]`` in JSON mode if no rows are returned. (:issue:`328`) + +.. _v3_17: + +3.17 (2021-08-24) +----------------- + +- The :ref:`sqlite-utils memory ` command has a new ``--analyze`` option, which runs the equivalent of the :ref:`analyze-tables ` command directly against the in-memory database created from the incoming CSV or JSON data. (:issue:`320`) +- :ref:`sqlite-utils insert-files ` now has the ability to insert file contents in to ``TEXT`` columns in addition to the default ``BLOB``. Pass the ``--text`` option or use ``content_text`` as a column specifier. (:issue:`319`) + +.. _v3_16: + +3.16 (2021-08-18) +----------------- + +- Type signatures added to more methods, including ``table.resolve_foreign_keys()``, ``db.create_table_sql()``, ``db.create_table()`` and ``table.create()``. (:issue:`314`) +- New ``db.quote_fts(value)`` method, see :ref:`python_api_quote_fts` - thanks, Mark Neumann. (:issue:`246`) +- ``table.search()`` now accepts an optional ``quote=True`` parameter. (:issue:`296`) +- CLI command ``sqlite-utils search`` now accepts a ``--quote`` option. (:issue:`296`) +- Fixed bug where ``--no-headers`` and ``--tsv`` options to :ref:`sqlite-utils insert ` could not be used together. (:issue:`295`) +- Various small improvements to :ref:`reference` documentation. + +.. _v3_15.1: + +3.15.1 (2021-08-10) +------------------- + +- Python library now includes type annotations on almost all of the methods, plus detailed docstrings describing each one. (:issue:`311`) +- New :ref:`reference` documentation page, powered by those docstrings. +- Fixed bug where ``.add_foreign_keys()`` failed to raise an error if called against a ``View``. (:issue:`313`) +- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existent table. (:issue:`315`) + .. _v3_15: 3.15 (2021-08-09) @@ -953,3 +1539,19 @@ A few other changes: ---------------- - ``enable_fts()``, ``populate_fts()`` and ``search()`` table methods + +0.3.1 (2018-07-31) +------------------ + +- Documented related projects +- Added badges to the documentation + +0.3 (2018-07-31) +---------------- + +- New ``Table`` class representing a table in the SQLite database + +0.2 (2018-07-28) +---------------- + +- Initial release to PyPI diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst new file mode 100644 index 0000000..9fafe28 --- /dev/null +++ b/docs/cli-reference.rst @@ -0,0 +1,1647 @@ +.. _cli_reference: + +=============== + CLI reference +=============== + +This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. + +.. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here + +.. [[[cog + from sqlite_utils import cli + import sys + sys._called_from_test = True + from click.testing import CliRunner + import textwrap + commands = list(cli.cli.commands.keys()) + go_first = [ + "query", "memory", "insert", "upsert", "bulk", "search", "transform", "extract", + "schema", "insert-files", "analyze-tables", "convert", "tables", "views", "rows", + "triggers", "indexes", "create-database", "create-table", "create-index", "drop-index", + "migrate", "enable-fts", "populate-fts", "rebuild-fts", "disable-fts" + ] + refs = { + "query": "cli_query", + "memory": "cli_memory", + "insert": [ + "cli_inserting_data", "cli_insert_csv_tsv", "cli_insert_unstructured", "cli_insert_convert" + ], + "upsert": "cli_upsert", + "tables": "cli_tables", + "views": "cli_views", + "optimize": "cli_optimize", + "rows": "cli_rows", + "triggers": "cli_triggers", + "indexes": "cli_indexes", + "enable-fts": "cli_fts", + "analyze": "cli_analyze", + "vacuum": "cli_vacuum", + "dump": "cli_dump", + "add-column": "cli_add_column", + "rename-table": "cli_renaming_tables", + "duplicate": "cli_duplicate_table", + "add-foreign-key": "cli_add_foreign_key", + "add-foreign-keys": "cli_add_foreign_keys", + "index-foreign-keys": "cli_index_foreign_keys", + "create-index": "cli_create_index", + "drop-index": "cli_drop_index", + "enable-wal": "cli_wal", + "enable-counts": "cli_enable_counts", + "bulk": "cli_bulk", + "migrate": "cli_migrate", + "create-database": "cli_create_database", + "create-table": "cli_create_table", + "drop-table": "cli_drop_table", + "create-view": "cli_create_view", + "drop-view": "cli_drop_view", + "search": "cli_search", + "transform": "cli_transform_table", + "extract": "cli_extract", + "schema": "cli_schema", + "insert-files": "cli_insert_files", + "analyze-tables": "cli_analyze_tables", + "convert": "cli_convert", + "add-geometry-column": "cli_spatialite", + "create-spatial-index": "cli_spatialite_indexes", + "install": "cli_install", + "uninstall": "cli_uninstall", + } + commands.sort(key = lambda command: go_first.index(command) if command in go_first else 999) + cog.out("\n") + for command in commands: + cog.out(".. _cli_ref_" + command.replace("-", "_") + ":\n\n") + cog.out(command + "\n") + cog.out(("=" * len(command)) + "\n\n") + if command in refs: + command_refs = refs[command] + if isinstance(command_refs, str): + command_refs = [command_refs] + cog.out( + "See {}.\n\n".format( + ", ".join(":ref:`{}`".format(c) for c in command_refs) + ) + ) + cog.out("::\n\n") + result = CliRunner().invoke(cli.cli, [command, "--help"]) + output = result.output.replace("Usage: cli ", "Usage: sqlite-utils ") + output = output.replace('\b', '') + cog.out(textwrap.indent(output, ' ')) + cog.out("\n\n") +.. ]]] + +.. _cli_ref_query: + +query +===== + +See :ref:`cli_query`. + +:: + + Usage: sqlite-utils query [OPTIONS] PATH SQL + + Execute SQL query and return the results as JSON + + Example: + + sqlite-utils data.db \ + "select * from chickens where age > :age" \ + -p age 1 + + Pass "-" as the SQL to read the query from standard input: + + echo "select * from chickens" | sqlite-utils data.db - + + Options: + --attach ... Additional databases to attach - specify alias and + filepath + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit headers from CSV/TSV and table/--fmt output + -t, --table Output as a formatted table + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, + fancy_outline, github, grid, heavy_grid, + heavy_outline, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, mixed_grid, + mixed_outline, moinmoin, orgtbl, outline, pipe, + plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not + escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX + -r, --raw Raw output, first column of first row + --raw-lines Raw output, first column of each row + -p, --param ... Named :parameters for SQL query + --functions TEXT Python code or a file path defining custom SQL + functions; can be used multiple times + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_memory: + +memory +====== + +See :ref:`cli_memory`. + +:: + + Usage: sqlite-utils memory [OPTIONS] [PATHS]... SQL + + Execute SQL query against an in-memory database, optionally populated by + imported data + + To import data from CSV, TSV or JSON files pass them on the command-line: + + sqlite-utils memory one.csv two.json \ + "select * from one join two on one.two_id = two.id" + + For data piped into the tool from standard input, use "-" or "stdin": + + cat animals.csv | sqlite-utils memory - \ + "select * from stdin where species = 'dog'" + + The format of the data will be automatically detected. You can specify the + format explicitly using :json, :csv, :tsv or :nl (for newline-delimited JSON) + - for example: + + cat animals.csv | sqlite-utils memory stdin:csv places.dat:nl \ + "select * from stdin where place_id in (select id from places)" + + Use --schema to view the SQL schema of any imported files: + + sqlite-utils memory animals.csv --schema + + Options: + --functions TEXT Python code or a file path defining custom SQL + functions; can be used multiple times + --attach ... Additional databases to attach - specify alias and + filepath + --flatten Flatten nested JSON objects, so {"foo": {"bar": + 1}} becomes {"foo_bar": 1} + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit headers from CSV/TSV and table/--fmt output + -t, --table Output as a formatted table + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, + fancy_outline, github, grid, heavy_grid, + heavy_outline, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, mixed_grid, + mixed_outline, moinmoin, orgtbl, outline, pipe, + plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not + escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX + -r, --raw Raw output, first column of first row + --raw-lines Raw output, first column of each row + -p, --param ... Named :parameters for SQL query + --encoding TEXT Character encoding for CSV input, defaults to + utf-8 + -n, --no-detect-types Treat all CSV/TSV columns as TEXT + --schema Show SQL schema for in-memory database + --dump Dump SQL for in-memory database + --save FILE Save in-memory database to this file + --analyze Analyze resulting tables and output results + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_insert: + +insert +====== + +See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstructured`, :ref:`cli_insert_convert`. + +:: + + Usage: sqlite-utils insert [OPTIONS] PATH TABLE [FILE] + + Insert records from FILE into a table, creating the table if it does not + already exist. + + Example: + + echo '{"name": "Lila"}' | sqlite-utils insert data.db chickens - + + By default the input is expected to be a JSON object or array of objects. + + - Use --nl for newline-delimited JSON objects + - Use --csv or --tsv for comma-separated or tab-separated input + - Use --lines to write each incoming line to a column called "line" + - Use --text to write the entire input to a column called "text" + + Use --type column-name type to override the type automatically chosen when the + table is created. + + You can also use --convert to pass a fragment of Python code that will be used + to convert each input. + + Your Python code will be passed a "row" variable representing the imported + row, and can return a modified row. + + This example uses just the name, latitude and longitude columns from a CSV + file, converting name to upper case and latitude and longitude to floating + point numbers: + + sqlite-utils insert plants.db plants plants.csv --csv --convert ' + return { + "name": row["name"].upper(), + "latitude": float(row["latitude"]), + "longitude": float(row["longitude"]), + }' + + If you are using --lines your code will be passed a "line" variable, and for + --text a "text" variable. + + When using --text your function can return an iterator of rows to insert. This + example inserts one record per word in the input: + + echo 'A bunch of words' | sqlite-utils insert words.db words - \ + --text --convert '({"word": w} for w in text.split())' + + Instead of a FILE you can use --code to provide a block of Python code that + defines the rows to insert, as either a rows() function that yields + dictionaries or a "rows" iterable. --code can also be a path to a .py file: + + sqlite-utils insert data.db creatures --code ' + def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} + ' --pk id + + Options: + --pk TEXT Columns to use as the primary key, e.g. id + --code TEXT Python code defining a rows() function or iterable + of rows to insert + --flatten Flatten nested JSON objects, so {"a": {"b": 1}} + becomes {"a_b": 1} + --nl Expect newline-delimited JSON + -c, --csv Expect CSV input + --tsv Expect TSV input + --empty-null Treat empty strings as NULL + --lines Treat each line as a single value called 'line' + --text Treat input as a single value called 'text' + --convert TEXT Python code to convert each item + --import TEXT Python modules to import + --delimiter TEXT Delimiter to use for CSV files + --quotechar TEXT Quote character to use for CSV/TSV + --sniff Detect delimiter and quote character + --no-headers CSV file has no header row + --encoding TEXT Character encoding for input, defaults to utf-8 + --batch-size INTEGER Commit every X records + --stop-after INTEGER Stop after X records + --alter Alter existing table to add any missing columns + --not-null TEXT Columns that should be created as NOT NULL + --default ... Default value that should be set for a column + --type ... Column types to use when creating the table + --no-detect-types Treat all CSV/TSV columns as TEXT + --analyze Run ANALYZE at the end of this operation + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + --silent Do not show progress bar + --strict Apply STRICT mode to created table + --ignore Ignore records if pk already exists + --replace Replace records if pk already exists + --truncate Truncate table before inserting records, if table + already exists + -h, --help Show this message and exit. + + +.. _cli_ref_upsert: + +upsert +====== + +See :ref:`cli_upsert`. + +:: + + Usage: sqlite-utils upsert [OPTIONS] PATH TABLE [FILE] + + Upsert records based on their primary key. Works like 'insert' but if an + incoming record has a primary key that matches an existing record the existing + record will be updated. + + If the table already exists and has a primary key, --pk can be omitted. + + Use --type column-name type to override the type automatically chosen when the + table is created. + + Example: + + echo '[ + {"id": 1, "name": "Lila"}, + {"id": 2, "name": "Suna"} + ]' | sqlite-utils upsert data.db chickens - --pk id + + Options: + --pk TEXT Columns to use as the primary key, e.g. id + --code TEXT Python code defining a rows() function or iterable + of rows to insert + --flatten Flatten nested JSON objects, so {"a": {"b": 1}} + becomes {"a_b": 1} + --nl Expect newline-delimited JSON + -c, --csv Expect CSV input + --tsv Expect TSV input + --empty-null Treat empty strings as NULL + --lines Treat each line as a single value called 'line' + --text Treat input as a single value called 'text' + --convert TEXT Python code to convert each item + --import TEXT Python modules to import + --delimiter TEXT Delimiter to use for CSV files + --quotechar TEXT Quote character to use for CSV/TSV + --sniff Detect delimiter and quote character + --no-headers CSV file has no header row + --encoding TEXT Character encoding for input, defaults to utf-8 + --batch-size INTEGER Commit every X records + --stop-after INTEGER Stop after X records + --alter Alter existing table to add any missing columns + --not-null TEXT Columns that should be created as NOT NULL + --default ... Default value that should be set for a column + --type ... Column types to use when creating the table + --no-detect-types Treat all CSV/TSV columns as TEXT + --analyze Run ANALYZE at the end of this operation + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + --silent Do not show progress bar + --strict Apply STRICT mode to created table + -h, --help Show this message and exit. + + +.. _cli_ref_bulk: + +bulk +==== + +See :ref:`cli_bulk`. + +:: + + Usage: sqlite-utils bulk [OPTIONS] PATH SQL FILE + + Execute parameterized SQL against the provided list of documents. + + Example: + + echo '[ + {"id": 1, "name": "Lila2"}, + {"id": 2, "name": "Suna2"} + ]' | sqlite-utils bulk data.db ' + update chickens set name = :name where id = :id + ' - + + Options: + --batch-size INTEGER Commit every X records + --functions TEXT Python code or a file path defining custom SQL + functions; can be used multiple times + --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes + {"a_b": 1} + --nl Expect newline-delimited JSON + -c, --csv Expect CSV input + --tsv Expect TSV input + --empty-null Treat empty strings as NULL + --lines Treat each line as a single value called 'line' + --text Treat input as a single value called 'text' + --convert TEXT Python code to convert each item + --import TEXT Python modules to import + --delimiter TEXT Delimiter to use for CSV files + --quotechar TEXT Quote character to use for CSV/TSV + --sniff Detect delimiter and quote character + --no-headers CSV file has no header row + --encoding TEXT Character encoding for input, defaults to utf-8 + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_search: + +search +====== + +See :ref:`cli_search`. + +:: + + Usage: sqlite-utils search [OPTIONS] PATH DBTABLE Q + + Execute a full-text search against this table + + Example: + + sqlite-utils search data.db chickens lila + + Options: + -o, --order TEXT Order by ('column' or 'column desc') + -c, --column TEXT Columns to return + --limit INTEGER Number of rows to return - defaults to everything + --sql Show SQL query that would be run + --quote Apply FTS quoting rules to search term + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit headers from CSV/TSV and table/--fmt output + -t, --table Output as a formatted table + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_transform: + +transform +========= + +See :ref:`cli_transform_table`. + +:: + + Usage: sqlite-utils transform [OPTIONS] PATH TABLE + + Transform a table beyond the capabilities of ALTER TABLE + + Example: + + sqlite-utils transform mydb.db mytable \ + --drop column1 \ + --rename column2 column_renamed + + Options: + --type ... Change column type to INTEGER, TEXT, FLOAT, + REAL or BLOB + --drop TEXT Drop this column + --rename ... Rename this column to X + -o, --column-order TEXT Reorder columns + --not-null TEXT Set this column to NOT NULL + --not-null-false TEXT Remove NOT NULL from this column + --pk TEXT Make this column the primary key + --pk-none Remove primary key (convert to rowid table) + --default ... Set default value for this column + --default-none TEXT Remove default from this column + --add-foreign-key ... + Add a foreign key constraint from a column to + another table with another column + --drop-foreign-key TEXT Drop foreign key constraint for this column + --strict / --no-strict Enable or disable STRICT mode (default: + preserve current mode) + --sql Output SQL without executing it + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_extract: + +extract +======= + +See :ref:`cli_extract`. + +:: + + Usage: sqlite-utils extract [OPTIONS] PATH TABLE COLUMNS... + + Extract one or more columns into a separate table + + Example: + + sqlite-utils extract trees.db Street_Trees species + + Options: + --table TEXT Name of the other table to extract columns to + --fk-column TEXT Name of the foreign key column to add to the table + --rename ... Rename this column in extracted table + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_schema: + +schema +====== + +See :ref:`cli_schema`. + +:: + + Usage: sqlite-utils schema [OPTIONS] PATH [TABLES]... + + Show full schema for this database or for specified tables + + Example: + + sqlite-utils schema trees.db + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_insert_files: + +insert-files +============ + +See :ref:`cli_insert_files`. + +:: + + Usage: sqlite-utils insert-files [OPTIONS] PATH TABLE FILE_OR_DIR... + + Insert one or more files using BLOB columns in the specified table + + Example: + + sqlite-utils insert-files pics.db images *.gif \ + -c name:name \ + -c content:content \ + -c content_hash:sha256 \ + -c created:ctime_iso \ + -c modified:mtime_iso \ + -c size:size \ + --pk name + + Options: + -c, --column TEXT Column definitions for the table + --pk TEXT Column to use as primary key + --alter Alter table to add missing columns + --replace Replace files with matching primary key + --upsert Upsert files with matching primary key + --name TEXT File name to use + --text Store file content as TEXT, not BLOB + --encoding TEXT Character encoding for input, defaults to utf-8 + -s, --silent Don't show a progress bar + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_analyze_tables: + +analyze-tables +============== + +See :ref:`cli_analyze_tables`. + +:: + + Usage: sqlite-utils analyze-tables [OPTIONS] PATH [TABLES]... + + Analyze the columns in one or more tables + + Example: + + sqlite-utils analyze-tables data.db trees + + Options: + -c, --column TEXT Specific columns to analyze + --save Save results to _analyze_tables table + --common-limit INTEGER How many common values + --no-most Skip most common values + --no-least Skip least common values + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_convert: + +convert +======= + +See :ref:`cli_convert`. + +:: + + Usage: sqlite-utils convert [OPTIONS] DB_PATH TABLE COLUMNS... CODE + + Convert columns using Python code you supply. For example: + + sqlite-utils convert my.db mytable mycolumn \ + '"\n".join(textwrap.wrap(value, 10))' \ + --import=textwrap + + "value" is a variable with the column value to be converted. + + CODE can also be a reference to a callable that takes the value, for example: + + sqlite-utils convert my.db mytable date r.parsedate + sqlite-utils convert my.db mytable data json.loads --import json + + Use "-" for CODE to read Python code from standard input. + + The following common operations are available as recipe functions: + + r.jsonsplit(value: 'str', delimiter: 'str' = ',', type: 'Callable[[str], + object]' = ) -> 'str' + + Convert a string like a,b,c into a JSON array ["a", "b", "c"] + + r.parsedate(value: 'str', dayfirst: 'bool' = False, yearfirst: 'bool' = False, + errors: 'Optional[object]' = None) -> 'Optional[str]' + + Parse a date and convert it to ISO date format: yyyy-mm-dd + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null + + r.parsedatetime(value: 'str', dayfirst: 'bool' = False, yearfirst: 'bool' = + False, errors: 'Optional[object]' = None) -> 'Optional[str]' + + Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null + + You can use these recipes like so: + + sqlite-utils convert my.db mytable mycolumn \ + 'r.jsonsplit(value, delimiter=":")' + + Options: + --import TEXT Python modules to import + --dry-run Show results of running this against first 10 + rows + --multi Populate columns for keys in returned + dictionary + --where TEXT Optional where clause + -p, --param ... Named :parameters for where clause + --output TEXT Optional separate column to populate with the + output + --output-type [integer|float|blob|text] + Column type to use for the output column + --drop Drop original column afterwards + -s, --silent Don't show a progress bar + --pdb Open pdb debugger on first error + -h, --help Show this message and exit. + + +.. _cli_ref_tables: + +tables +====== + +See :ref:`cli_tables`. + +:: + + Usage: sqlite-utils tables [OPTIONS] PATH + + List the tables in the database + + Example: + + sqlite-utils tables trees.db + + Options: + --fts4 Just show FTS4 enabled tables + --fts5 Just show FTS5 enabled tables + --counts Include row counts per table + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit headers from CSV/TSV and table/--fmt output + -t, --table Output as a formatted table + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX + --columns Include list of columns for each table + --schema Include schema for each table + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_views: + +views +===== + +See :ref:`cli_views`. + +:: + + Usage: sqlite-utils views [OPTIONS] PATH + + List the views in the database + + Example: + + sqlite-utils views trees.db + + Options: + --counts Include row counts per view + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit headers from CSV/TSV and table/--fmt output + -t, --table Output as a formatted table + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX + --columns Include list of columns for each view + --schema Include schema for each view + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_rows: + +rows +==== + +See :ref:`cli_rows`. + +:: + + Usage: sqlite-utils rows [OPTIONS] PATH DBTABLE + + Output all rows in the specified table + + Example: + + sqlite-utils rows trees.db Trees + + Options: + -c, --column TEXT Columns to return + --where TEXT Optional where clause + -o, --order TEXT Order by ('column' or 'column desc') + -p, --param ... Named :parameters for where clause + --limit INTEGER Number of rows to return - defaults to everything + --offset INTEGER SQL offset to use + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit headers from CSV/TSV and table/--fmt output + -t, --table Output as a formatted table + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, + fancy_outline, github, grid, heavy_grid, + heavy_outline, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, mixed_grid, + mixed_outline, moinmoin, orgtbl, outline, pipe, + plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not + escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_triggers: + +triggers +======== + +See :ref:`cli_triggers`. + +:: + + Usage: sqlite-utils triggers [OPTIONS] PATH [TABLES]... + + Show triggers configured in this database + + Example: + + sqlite-utils triggers trees.db + + Options: + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit headers from CSV/TSV and table/--fmt output + -t, --table Output as a formatted table + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_indexes: + +indexes +======= + +See :ref:`cli_indexes`. + +:: + + Usage: sqlite-utils indexes [OPTIONS] PATH [TABLES]... + + Show indexes for the whole database or specific tables + + Example: + + sqlite-utils indexes trees.db Trees + + Options: + --aux Include auxiliary columns + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit headers from CSV/TSV and table/--fmt output + -t, --table Output as a formatted table + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_create_database: + +create-database +=============== + +See :ref:`cli_create_database`. + +:: + + Usage: sqlite-utils create-database [OPTIONS] PATH + + Create a new empty database file + + Example: + + sqlite-utils create-database trees.db + + Options: + --enable-wal Enable WAL mode on the created database + --init-spatialite Enable SpatiaLite on the created database + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_create_table: + +create-table +============ + +See :ref:`cli_create_table`. + +:: + + Usage: sqlite-utils create-table [OPTIONS] PATH TABLE COLUMNS... + + Add a table with the specified columns. Columns should be specified using + name, type pairs, for example: + + sqlite-utils create-table my.db people \ + id integer \ + name text \ + height real \ + photo blob --pk id + + Valid column types are text, integer, real, float and blob. + + Options: + --pk TEXT Column to use as primary key + --not-null TEXT Columns that should be created as NOT NULL + --default ... Default value that should be set for a column + --fk ... Column, other table, other column to set as a + foreign key + --ignore If table already exists, do nothing + --replace If table already exists, replace it + --transform If table already exists, try to transform the schema + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + --strict Apply STRICT mode to created table + -h, --help Show this message and exit. + + +.. _cli_ref_create_index: + +create-index +============ + +See :ref:`cli_create_index`. + +:: + + Usage: sqlite-utils create-index [OPTIONS] PATH TABLE COLUMN... + + Add an index to the specified table for the specified columns + + Example: + + sqlite-utils create-index chickens.db chickens name + + To create an index in descending order: + + sqlite-utils create-index chickens.db chickens -- -name + + Options: + --name TEXT Explicit name for the new index + --unique Make this a unique index + --if-not-exists, --ignore Ignore if index already exists + --analyze Run ANALYZE after creating the index + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_drop_index: + +drop-index +========== + +See :ref:`cli_drop_index`. + +:: + + Usage: sqlite-utils drop-index [OPTIONS] PATH TABLE INDEX + + Drop an index by index name from the specified table + + Example: + + sqlite-utils drop-index chickens.db chickens idx_chickens_name + + Options: + --ignore Ignore if index does not exist + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_migrate: + +migrate +======= + +See :ref:`cli_migrate`. + +:: + + Usage: sqlite-utils migrate [OPTIONS] DB_PATH [MIGRATIONS]... + + Apply pending database migrations. + + Usage: + + sqlite-utils migrate database.db + + This will find the migrations.py file in the current directory or + subdirectories and apply any pending migrations. + + Or pass paths to one or more migrations.py files directly: + + sqlite-utils migrate database.db path/to/migrations.py + + Pass --list to see a list of applied and pending migrations without applying + them. + + Use --stop-before migration_set:name to stop before a migration. This option + can be used multiple times. + + Options: + --stop-before TEXT Stop before applying this migration. Use set:name to + target a migration set. + --list List migrations without running them + -v, --verbose Show verbose output + -h, --help Show this message and exit. + + +.. _cli_ref_enable_fts: + +enable-fts +========== + +See :ref:`cli_fts`. + +:: + + Usage: sqlite-utils enable-fts [OPTIONS] PATH TABLE COLUMN... + + Enable full-text search for specific table and columns + + Example: + + sqlite-utils enable-fts chickens.db chickens name + + Options: + --fts4 Use FTS4 + --fts5 Use FTS5 + --tokenize TEXT Tokenizer to use, e.g. porter + --create-triggers Create triggers to update the FTS tables when the + parent table changes. + --replace Replace existing FTS configuration if it exists + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_populate_fts: + +populate-fts +============ + +:: + + Usage: sqlite-utils populate-fts [OPTIONS] PATH TABLE COLUMN... + + Re-populate full-text search for specific table and columns + + Example: + + sqlite-utils populate-fts chickens.db chickens name + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_rebuild_fts: + +rebuild-fts +=========== + +:: + + Usage: sqlite-utils rebuild-fts [OPTIONS] PATH [TABLES]... + + Rebuild all or specific full-text search tables + + Example: + + sqlite-utils rebuild-fts chickens.db chickens + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_disable_fts: + +disable-fts +=========== + +:: + + Usage: sqlite-utils disable-fts [OPTIONS] PATH TABLE + + Disable full-text search for specific table + + Example: + + sqlite-utils disable-fts chickens.db chickens + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_optimize: + +optimize +======== + +See :ref:`cli_optimize`. + +:: + + Usage: sqlite-utils optimize [OPTIONS] PATH [TABLES]... + + Optimize all full-text search tables and then run VACUUM - should shrink the + database file + + Example: + + sqlite-utils optimize chickens.db + + Options: + --no-vacuum Don't run VACUUM + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_analyze: + +analyze +======= + +See :ref:`cli_analyze`. + +:: + + Usage: sqlite-utils analyze [OPTIONS] PATH [NAMES]... + + Run ANALYZE against the whole database, or against specific named indexes and + tables + + Example: + + sqlite-utils analyze chickens.db + + Options: + -h, --help Show this message and exit. + + +.. _cli_ref_vacuum: + +vacuum +====== + +See :ref:`cli_vacuum`. + +:: + + Usage: sqlite-utils vacuum [OPTIONS] PATH + + Run VACUUM against the database + + Example: + + sqlite-utils vacuum chickens.db + + Options: + -h, --help Show this message and exit. + + +.. _cli_ref_dump: + +dump +==== + +See :ref:`cli_dump`. + +:: + + Usage: sqlite-utils dump [OPTIONS] PATH + + Output a SQL dump of the schema and full contents of the database + + Example: + + sqlite-utils dump chickens.db + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_add_column: + +add-column +========== + +See :ref:`cli_add_column`. + +:: + + Usage: sqlite-utils add-column [OPTIONS] PATH TABLE COL_NAME + [integer|int|float|real|text|str|blob|bytes] + + Add a column to the specified table + + Example: + + sqlite-utils add-column chickens.db chickens weight float + + Options: + --fk TEXT Table to reference as a foreign key + --fk-col TEXT Referenced column on that foreign key table - if + omitted will automatically use the primary key + --not-null-default TEXT Add NOT NULL DEFAULT 'TEXT' constraint + --ignore If column already exists, do nothing + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_add_foreign_key: + +add-foreign-key +=============== + +See :ref:`cli_add_foreign_key`. + +:: + + Usage: sqlite-utils add-foreign-key [OPTIONS] PATH TABLE COLUMN [OTHER_TABLE] + [OTHER_COLUMN] + + Add a new foreign key constraint to an existing table + + Example: + + sqlite-utils add-foreign-key my.db books author_id authors id + + Options: + --ignore If foreign key already exists, do nothing + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_add_foreign_keys: + +add-foreign-keys +================ + +See :ref:`cli_add_foreign_keys`. + +:: + + Usage: sqlite-utils add-foreign-keys [OPTIONS] PATH [FOREIGN_KEY]... + + Add multiple new foreign key constraints to a database + + Example: + + sqlite-utils add-foreign-keys my.db \ + books author_id authors id \ + authors country_id countries id + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_index_foreign_keys: + +index-foreign-keys +================== + +See :ref:`cli_index_foreign_keys`. + +:: + + Usage: sqlite-utils index-foreign-keys [OPTIONS] PATH + + Ensure every foreign key column has an index on it + + Example: + + sqlite-utils index-foreign-keys chickens.db + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_enable_wal: + +enable-wal +========== + +See :ref:`cli_wal`. + +:: + + Usage: sqlite-utils enable-wal [OPTIONS] PATH... + + Enable WAL for database files + + Example: + + sqlite-utils enable-wal chickens.db + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_disable_wal: + +disable-wal +=========== + +:: + + Usage: sqlite-utils disable-wal [OPTIONS] PATH... + + Disable WAL for database files + + Example: + + sqlite-utils disable-wal chickens.db + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_enable_counts: + +enable-counts +============= + +See :ref:`cli_enable_counts`. + +:: + + Usage: sqlite-utils enable-counts [OPTIONS] PATH [TABLES]... + + Configure triggers to update a _counts table with row counts + + Example: + + sqlite-utils enable-counts chickens.db + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_reset_counts: + +reset-counts +============ + +:: + + Usage: sqlite-utils reset-counts [OPTIONS] PATH + + Reset calculated counts in the _counts table + + Example: + + sqlite-utils reset-counts chickens.db + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_duplicate: + +duplicate +========= + +See :ref:`cli_duplicate_table`. + +:: + + Usage: sqlite-utils duplicate [OPTIONS] PATH TABLE NEW_TABLE + + Create a duplicate of this table, copying across the schema and all row data. + + Options: + --ignore If table does not exist, do nothing + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_rename_table: + +rename-table +============ + +See :ref:`cli_renaming_tables`. + +:: + + Usage: sqlite-utils rename-table [OPTIONS] PATH TABLE NEW_NAME + + Rename this table. + + Options: + --ignore If table does not exist, do nothing + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_drop_table: + +drop-table +========== + +See :ref:`cli_drop_table`. + +:: + + Usage: sqlite-utils drop-table [OPTIONS] PATH TABLE + + Drop the specified table + + Example: + + sqlite-utils drop-table chickens.db chickens + + Options: + --ignore If table does not exist, do nothing + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_create_view: + +create-view +=========== + +See :ref:`cli_create_view`. + +:: + + Usage: sqlite-utils create-view [OPTIONS] PATH VIEW SELECT + + Create a view for the provided SELECT query + + Example: + + sqlite-utils create-view chickens.db heavy_chickens \ + 'select * from chickens where weight > 3' + + Options: + --ignore If view already exists, do nothing + --replace If view already exists, replace it + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_drop_view: + +drop-view +========= + +See :ref:`cli_drop_view`. + +:: + + Usage: sqlite-utils drop-view [OPTIONS] PATH VIEW + + Drop the specified view + + Example: + + sqlite-utils drop-view chickens.db heavy_chickens + + Options: + --ignore If view does not exist, do nothing + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_install: + +install +======= + +See :ref:`cli_install`. + +:: + + Usage: sqlite-utils install [OPTIONS] [PACKAGES]... + + Install packages from PyPI into the same environment as sqlite-utils + + Options: + -U, --upgrade Upgrade packages to latest version + -e, --editable TEXT Install a project in editable mode from this path + -h, --help Show this message and exit. + + +.. _cli_ref_uninstall: + +uninstall +========= + +See :ref:`cli_uninstall`. + +:: + + Usage: sqlite-utils uninstall [OPTIONS] PACKAGES... + + Uninstall Python packages from the sqlite-utils environment + + Options: + -y, --yes Don't ask for confirmation + -h, --help Show this message and exit. + + +.. _cli_ref_add_geometry_column: + +add-geometry-column +=================== + +See :ref:`cli_spatialite`. + +:: + + Usage: sqlite-utils add-geometry-column [OPTIONS] DB_PATH TABLE COLUMN_NAME + + Add a SpatiaLite geometry column to an existing table. Requires SpatiaLite + extension. + + By default, this command will try to load the SpatiaLite extension from usual + paths. To load it from a specific path, use --load-extension. + + Options: + -t, --type [point|linestring|polygon|multipoint|multilinestring|multipolygon|geometrycollection|geometry] + Specify a geometry type for this column. + [default: GEOMETRY] + --srid INTEGER Spatial Reference ID. See + https://spatialreference.org for details on + specific projections. [default: 4326] + --dimensions TEXT Coordinate dimensions. Use XYZ for three- + dimensional geometries. + --not-null Add a NOT NULL constraint. + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_create_spatial_index: + +create-spatial-index +==================== + +See :ref:`cli_spatialite_indexes`. + +:: + + Usage: sqlite-utils create-spatial-index [OPTIONS] DB_PATH TABLE COLUMN_NAME + + Create a spatial index on a SpatiaLite geometry column. The table and geometry + column must already exist before trying to add a spatial index. + + By default, this command will try to load the SpatiaLite extension from usual + paths. To load it from a specific path, use --load-extension. + + Options: + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + +.. _cli_ref_plugins: + +plugins +======= + +:: + + Usage: sqlite-utils plugins [OPTIONS] + + List installed plugins + + Options: + -h, --help Show this message and exit. + + +.. [[[end]]] diff --git a/docs/cli.rst b/docs/cli.rst index 063b9e9..2e506dd 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -6,37 +6,70 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite databases in a number of different ways. +Once :ref:`installed ` the tool should be available as ``sqlite-utils``. It can also be run using ``python -m sqlite_utils``. + .. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here .. _cli_query: Running SQL queries =================== -The ``sqlite-utils query`` command lets you run queries directly against a SQLite database file. This is the default subcommand, so the following two examples work the same way:: +The ``sqlite-utils query`` command lets you run queries directly against a SQLite database file. This is the default subcommand, so the following two examples work the same way: - $ sqlite-utils query dogs.db "select * from dogs" - $ sqlite-utils dogs.db "select * from dogs" +.. code-block:: bash + + sqlite-utils query dogs.db "select * from dogs" + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" + +.. note:: + In Python: :ref:`db.query() ` CLI reference: :ref:`sqlite-utils query ` + +Pass ``-`` as the SQL query to read the query from standard input. This is useful for longer queries that would otherwise require careful shell escaping, or for piping in SQL generated by another tool: + +.. code-block:: bash + + echo "select * from dogs" | sqlite-utils query dogs.db - + +.. code-block:: bash + + sqlite-utils query dogs.db - < query.sql .. _cli_query_json: Returning JSON -------------- -The default format returned for queries is JSON:: +The default format returned for queries is JSON: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] +If the query returns more than one column with the same name, later occurrences are renamed with a numeric suffix - ``select 1 as id, 2 as id`` returns ``[{"id": 1, "id_2": 2}]``. This only applies to JSON output: :ref:`CSV and TSV ` and :ref:`table ` output keep the duplicate column headers unchanged. + .. _cli_query_nl: Newline-delimited JSON ~~~~~~~~~~~~~~~~~~~~~~ -Use ``--nl`` to get back newline-delimited JSON objects:: +Use ``--nl`` to get back newline-delimited JSON objects: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" --nl + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" --nl {"id": 1, "age": 4, "name": "Cleo"} {"id": 2, "age": 2, "name": "Pancakes"} @@ -45,21 +78,36 @@ Use ``--nl`` to get back newline-delimited JSON objects:: JSON arrays ~~~~~~~~~~~ -You can use ``--arrays`` to request arrays instead of objects:: +You can use ``--arrays`` to request arrays instead of objects: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" --arrays + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" --arrays [[1, 4, "Cleo"], [2, 2, "Pancakes"]] -You can also combine ``--arrays`` and ``--nl``:: +You can also combine ``--arrays`` and ``--nl``: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" --arrays --nl + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" --arrays --nl [1, 4, "Cleo"] [2, 2, "Pancakes"] -If you want to pretty-print the output further, you can pipe it through ``python -mjson.tool``:: +If you want to pretty-print the output further, you can pipe it through ``python -mjson.tool``: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" | python -mjson.tool + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" | python -mjson.tool [ { "id": 1, @@ -73,14 +121,46 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +.. _cli_query_json_ascii: + +Unicode characters in JSON +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +JSON output includes unicode characters directly, without escaping them: + +.. code-block:: bash + + sqlite-utils dogs.db "select '日本語' as text" + +.. code-block:: output + + [{"text": "日本語"}] + +Use ``--ascii`` to escape non-ASCII characters as ``\uXXXX`` sequences instead: + +.. code-block:: bash + + sqlite-utils dogs.db "select '日本語' as text" --ascii + +.. code-block:: output + + [{"text": "\u65e5\u672c\u8a9e"}] + +The ``--ascii`` option can help on systems that cannot display or process UTF-8, such as Windows consoles using a legacy code page. On Windows, setting the ``PYTHONUTF8=1`` environment variable is an alternative fix for ``UnicodeEncodeError`` crashes when redirecting output to a file. + .. _cli_query_binary_json: Binary data in JSON ~~~~~~~~~~~~~~~~~~~ -Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this:: +Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this: + +.. code-block:: bash + + sqlite-utils dogs.db "select name, content from images" | python -mjson.tool + +.. code-block:: output - $ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool [ { "name": "transparent.gif", @@ -91,15 +171,19 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b } ] - .. _cli_json_values: Nested JSON values ~~~~~~~~~~~~~~~~~~ -If one of your columns contains JSON, by default it will be returned as an escaped string:: +If one of your columns contains JSON, by default it will be returned as an escaped string: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" | python -mjson.tool + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" | python -mjson.tool [ { "id": 1, @@ -108,9 +192,14 @@ If one of your columns contains JSON, by default it will be returned as an escap } ] -You can use the ``--json-cols`` option to automatically detect these JSON columns and output them as nested JSON data:: +You can use the ``--json-cols`` option to automatically detect these JSON columns and output them as nested JSON data: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" --json-cols | python -mjson.tool + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" --json-cols | python -mjson.tool [ { "id": 1, @@ -131,22 +220,37 @@ You can use the ``--json-cols`` option to automatically detect these JSON column Returning CSV or TSV -------------------- -You can use the ``--csv`` option to return results as CSV:: +You can use the ``--csv`` option to return results as CSV: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" --csv + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" --csv id,age,name 1,4,Cleo 2,2,Pancakes -This will default to including the column names as a header row. To exclude the headers, use ``--no-headers``:: +This will default to including the column names as a header row. To exclude the headers, use ``--no-headers``: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" --csv --no-headers + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" --csv --no-headers 1,4,Cleo 2,2,Pancakes -Use ``--tsv`` instead of ``--csv`` to get back tab-separated values:: +Use ``--tsv`` instead of ``--csv`` to get back tab-separated values: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" --tsv + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" --tsv id age name 1 4 Cleo 2 2 Pancakes @@ -156,17 +260,27 @@ Use ``--tsv`` instead of ``--csv`` to get back tab-separated values:: Table-formatted output ---------------------- -You can use the ``--table`` option (or ``-t`` shortcut) to output query results as a table:: +You can use the ``--table`` option (or ``-t`` shortcut) to output query results as a table: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" --table + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" --table id age name ---- ----- -------- 1 4 Cleo 2 2 Pancakes -You can use the ``--fmt`` option to specify different table formats, for example ``rst`` for reStructuredText:: +You can use the ``--fmt`` option to specify different table formats, for example ``rst`` for reStructuredText: + +.. code-block:: bash + + sqlite-utils dogs.db "select * from dogs" --fmt rst + +.. code-block:: output - $ sqlite-utils dogs.db "select * from dogs" --table --fmt rst ==== ===== ======== id age name ==== ===== ======== @@ -174,7 +288,54 @@ You can use the ``--fmt`` option to specify different table formats, for example 2 2 Pancakes ==== ===== ======== -For a full list of table format options, run ``sqlite-utils query --help``. +Available ``--fmt`` options are: + +.. [[[cog + import tabulate + cog.out("\n" + "\n".join('- ``{}``'.format(t) for t in tabulate.tabulate_formats) + "\n\n") +.. ]]] + +- ``asciidoc`` +- ``colon_grid`` +- ``double_grid`` +- ``double_outline`` +- ``fancy_grid`` +- ``fancy_outline`` +- ``github`` +- ``grid`` +- ``heavy_grid`` +- ``heavy_outline`` +- ``html`` +- ``jira`` +- ``latex`` +- ``latex_booktabs`` +- ``latex_longtable`` +- ``latex_raw`` +- ``mediawiki`` +- ``mixed_grid`` +- ``mixed_outline`` +- ``moinmoin`` +- ``orgtbl`` +- ``outline`` +- ``pipe`` +- ``plain`` +- ``presto`` +- ``pretty`` +- ``psql`` +- ``rounded_grid`` +- ``rounded_outline`` +- ``rst`` +- ``simple`` +- ``simple_grid`` +- ``simple_outline`` +- ``textile`` +- ``tsv`` +- ``unsafehtml`` +- ``youtrack`` + +.. [[[end]]] + +This list can also be found by running ``sqlite-utils query --help``. .. _cli_query_raw: @@ -183,19 +344,31 @@ Returning raw data, such as binary content If your table contains binary data in a ``BLOB`` you can use the ``--raw`` option to output specific columns directly to standard out. -For example, to retrieve a binary image from a ``BLOB`` column and store it in a file you can use the following:: +For example, to retrieve a binary image from a ``BLOB`` column and store it in a file you can use the following: - $ sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg +.. code-block:: bash + sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg + +To return the first column of each result as raw data, separated by newlines, use ``--raw-lines``: + +.. code-block:: bash + + sqlite-utils photos.db "select caption from photos" --raw-lines > captions.txt .. _cli_query_parameters: Using named parameters ---------------------- -You can pass named parameters to the query using ``-p``:: +You can pass named parameters to the query using ``-p name value``: + +.. code-block:: bash + + sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 + +.. code-block:: output - $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 [{":num * :num2": 30}] These will be correctly quoted and escaped in the SQL query, providing a safe way to combine other values with SQL. @@ -205,19 +378,68 @@ These will be correctly quoted and escaped in the SQL query, providing a safe wa UPDATE, INSERT and DELETE ------------------------- -If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: +If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows: + +.. code-block:: bash + + sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" + +.. code-block:: output - $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" [{"rows_affected": 1}] +.. _cli_query_functions: + +Defining custom SQL functions +----------------------------- + +You can use the ``--functions`` option to pass a block of Python code that defines additional functions which can then be called by your SQL query. + +This example defines a function which extracts the domain from a URL: + +.. code-block:: bash + + sqlite-utils query sites.db "select url, domain(url) from urls" --functions ' + from urllib.parse import urlparse + + def domain(url): + return urlparse(url).netloc + ' + +Every callable object defined in the block will be registered as a SQL function with the same name, with the exception of functions with names that begin with an underscore. + +You can also pass the path to a Python file containing function definitions: + +.. code-block:: bash + + sqlite-utils query sites.db "select url, domain(url) from urls" --functions functions.py + +The ``--functions`` option can be used multiple times to load functions from multiple sources: + +.. code-block:: bash + + sqlite-utils query sites.db "select url, domain(url), extract_path(url) from urls" \ + --functions domain_funcs.py \ + --functions 'def extract_path(url): + from urllib.parse import urlparse + return urlparse(url).path' + +.. note:: + In Python: :ref:`db.register_function() ` + +.. _cli_query_extensions: + SQLite extensions ----------------- You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. -:: +.. code-block:: bash + + sqlite-utils dogs.db "select spatialite_version()" --load-extension=spatialite + +.. code-block:: output - $ sqlite-utils dogs.db "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] .. _cli_query_attach: @@ -229,11 +451,16 @@ SQLite supports cross-database SQL queries, which can join data from tables in m You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. -This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: +This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database: + +.. code-block:: bash sqlite-utils dogs.db --attach books books.db \ 'select * from sqlite_master union all select * from books.sqlite_master' +.. note:: + In Python: :ref:`db.attach() ` + .. _cli_memory: Querying data directly using an in-memory database @@ -243,17 +470,33 @@ The ``sqlite-utils memory`` command works similar to ``sqlite-utils query``, but You can also pass this command CSV or JSON files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite. -Without any extra arguments, this command executes SQL against the in-memory database directly:: +Without any extra arguments, this command executes SQL against the in-memory database directly: + +.. code-block:: bash + + sqlite-utils memory 'select sqlite_version()' + +.. code-block:: output - $ sqlite-utils memory 'select sqlite_version()' [{"sqlite_version()": "3.35.5"}] -It takes all of the same output formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``:: +It takes all of the same output formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``: + +.. code-block:: bash + + sqlite-utils memory 'select sqlite_version()' --csv + +.. code-block:: output - $ sqlite-utils memory 'select sqlite_version()' --csv sqlite_version() 3.35.5 - $ sqlite-utils memory 'select sqlite_version()' --table --fmt grid + +.. code-block:: bash + + sqlite-utils memory 'select sqlite_version()' --fmt grid + +.. code-block:: output + +--------------------+ | sqlite_version() | +====================+ @@ -265,33 +508,50 @@ It takes all of the same output formatting options as :ref:`sqlite-utils query < Running queries directly against CSV or JSON -------------------------------------------- -If you have data in CSV or JSON format you can load it into an in-memory SQLite database and run queries against it directly in a single command using ``sqlite-utils memory`` like this:: +If you have data in CSV or JSON format you can load it into an in-memory SQLite database and run queries against it directly in a single command using ``sqlite-utils memory`` like this: - $ sqlite-utils memory data.csv "select * from data" +.. code-block:: bash -You can pass multiple files to the command if you want to run joins between data from different files:: + sqlite-utils memory data.csv "select * from data" - $ sqlite-utils memory one.csv two.json "select * from one join two on one.id = two.other_id" +You can pass multiple files to the command if you want to run joins between data from different files: + +.. code-block:: bash + + sqlite-utils memory one.csv two.json \ + "select * from one join two on one.id = two.other_id" If your data is JSON it should be the same format supported by the :ref:`sqlite-utils insert command ` - so either a single JSON object (treated as a single row) or a list of JSON objects. CSV data can be comma- or tab- delimited. -The in-memory tables will be named after the files without their extensions. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table:: +The in-memory tables will be named after the files without their extensions. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table: - $ sqlite-utils memory example.csv "select * from t" +.. code-block:: bash -To read from standard input, use either ``-`` or ``stdin`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: + sqlite-utils memory example.csv "select * from t" - $ cat example.csv | sqlite-utils memory - "select * from stdin" +If two files have the same name they will be assigned a numeric suffix: -Incoming CSV data will be assumed to use ``utf-8``. If your data uses a different character encoding you can specify that with ``--encoding``:: +.. code-block:: bash - $ cat example.csv | sqlite-utils memory - "select * from stdin" --encoding=latin-1 + sqlite-utils memory foo/data.csv bar/data.csv "select * from data_2" + +To read from standard input, use either ``-`` or ``stdin`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name: + +.. code-block:: bash + + cat example.csv | sqlite-utils memory - "select * from stdin" + +Incoming CSV data will be assumed to use ``utf-8``. If your data uses a different character encoding you can specify that with ``--encoding``: + +.. code-block:: bash + + cat example.csv | sqlite-utils memory - "select * from stdin" --encoding=latin-1 If you are joining across multiple CSV files they must all use the same encoding. -Column types will be automatically detected in CSV or TSV data, using the same mechanism as ``--detect-types`` described in :ref:`cli_insert_csv_tsv`. You can pass the ``--no-detect-types`` option to disable this automatic type detection and treat all CSV and TSV columns as ``TEXT``. +Column types will be automatically detected in CSV or TSV data, as described in :ref:`cli_insert_csv_tsv`. You can pass the ``--no-detect-types`` option to disable this automatic type detection and treat all CSV and TSV columns as ``TEXT``. .. _cli_memory_explicit: @@ -300,24 +560,31 @@ Explicitly specifying the format By default, ``sqlite-utils memory`` will attempt to detect the incoming data format (JSON, TSV or CSV) automatically. -You can instead specify an explicit format by adding a ``:csv``, ``:tsv``, ``:json`` or ``:nl`` (for newline-delimited JSON) suffix to the filename. For example:: +You can instead specify an explicit format by adding a ``:csv``, ``:tsv``, ``:json`` or ``:nl`` (for newline-delimited JSON) suffix to the filename. For example: - $ sqlite-utils memory one.dat:csv two.dat:nl "select * from one union select * from two" +.. code-block:: bash + + sqlite-utils memory one.dat:csv two.dat:nl \ + "select * from one union select * from two" Here the contents of ``one.dat`` will be treated as CSV and the contents of ``two.dat`` will be treated as newline-delimited JSON. -To explicitly specify the format for data piped into the tool on standard input, use ``stdin:format`` - for example:: +To explicitly specify the format for data piped into the tool on standard input, use ``stdin:format`` - for example: - $ cat one.dat | sqlite-utils memory stdin:csv "select * from stdin" +.. code-block:: bash + + cat one.dat | sqlite-utils memory stdin:csv "select * from stdin" .. _cli_memory_attach: Joining in-memory data against existing databases using \-\-attach ------------------------------------------------------------------ -The :ref:`attach option ` can be used to attach database files to the in-memory connection, enabling joins between in-memory data loaded from a file and tables in existing SQLite database files. An example:: +The :ref:`attach option ` can be used to attach database files to the in-memory connection, enabling joins between in-memory data loaded from a file and tables in existing SQLite database files. An example: - $ echo "id\n1\n3\n5" | sqlite-utils memory - --attach trees trees.db \ +.. code-block:: bash + + echo "id\n1\n3\n5" | sqlite-utils memory - --attach trees trees.db \ "select * from trees.trees where rowid in (select id from stdin)" Here the ``--attach trees trees.db`` option makes the ``trees.db`` database available with an alias of ``trees``. @@ -328,38 +595,82 @@ The CSV data that was piped into the script is available in the ``stdin`` table, .. _cli_memory_schema_dump_save: -\-\-schema, \-\-dump and \-\-save ---------------------------------- +\-\-schema, \-\-analyze, \-\-dump and \-\-save +---------------------------------------------- -To see the schema that will be created for a file or multiple files, use ``--schema``:: +To see the in-memory database schema that would be used for a file or for multiple files, use ``--schema``: - % sqlite-utils memory dogs.csv --schema - CREATE TABLE [dogs] ( - [id] INTEGER, - [age] INTEGER, - [name] TEXT +.. code-block:: bash + + sqlite-utils memory dogs.csv --schema + +.. code-block:: output + + CREATE TABLE "dogs" ( + "id" INTEGER, + "age" INTEGER, + "name" TEXT ); - CREATE VIEW t1 AS select * from [dogs]; - CREATE VIEW t AS select * from [dogs]; + CREATE VIEW "t1" AS select * from "dogs"; + CREATE VIEW "t" AS select * from "dogs"; -You can output SQL that will both create the tables and insert the full data used to populate the in-memory database using ``--dump``:: +You can run the equivalent of the :ref:`analyze-tables ` command using ``--analyze``: + +.. code-block:: bash + + sqlite-utils memory dogs.csv --analyze + +.. code-block:: output + + dogs.id: (1/3) + + Total rows: 2 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + + dogs.name: (2/3) + + Total rows: 2 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + + dogs.age: (3/3) + + Total rows: 2 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + +You can output SQL that will both create the tables and insert the full data used to populate the in-memory database using ``--dump``: + +.. code-block:: bash + + sqlite-utils memory dogs.csv --dump + +.. code-block:: output - % sqlite-utils memory dogs.csv --dump BEGIN TRANSACTION; - CREATE TABLE [dogs] ( - [id] INTEGER, - [age] INTEGER, - [name] TEXT + CREATE TABLE "dogs" ( + "id" INTEGER, + "age" INTEGER, + "name" TEXT ); INSERT INTO "dogs" VALUES('1','4','Cleo'); INSERT INTO "dogs" VALUES('2','2','Pancakes'); - CREATE VIEW t1 AS select * from [dogs]; - CREATE VIEW t AS select * from [dogs]; + CREATE VIEW "t1" AS select * from "dogs"; + CREATE VIEW "t" AS select * from "dogs"; COMMIT; -Passing ``--save other.db`` will instead use that SQL to populate a new database file:: +Passing ``--save other.db`` will instead use that SQL to populate a new database file: - % sqlite-utils memory dogs.csv --save dogs.db +.. code-block:: bash + + sqlite-utils memory dogs.csv --save dogs.db These features are mainly intended as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. @@ -368,80 +679,155 @@ These features are mainly intended as debugging tools - for much more finely gra Returning all rows in a table ============================= -You can return every row in a specified table using the ``rows`` command:: +You can return every row in a specified table using the ``rows`` command: + +.. code-block:: bash + + sqlite-utils rows dogs.db dogs + +.. code-block:: output - $ sqlite-utils rows dogs.db dogs [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] This command accepts the same output options as ``query`` - so you can pass ``--nl``, ``--csv``, ``--tsv``, ``--no-headers``, ``--table`` and ``--fmt``. -You can use the ``-c`` option to specify a subset of columns to return:: +You can use the ``-c`` option to specify a subset of columns to return: + +.. code-block:: bash + + sqlite-utils rows dogs.db dogs -c age -c name + +.. code-block:: output - $ sqlite-utils rows dogs.db dogs -c age -c name [{"age": 4, "name": "Cleo"}, {"age": 2, "name": "Pancakes"}] +You can filter rows using a where clause with the ``--where`` option: + +.. code-block:: bash + + sqlite-utils rows dogs.db dogs -c name --where 'name = "Cleo"' + +.. code-block:: output + + [{"name": "Cleo"}] + +Or pass named parameters using ``--where`` in combination with ``-p``: + +.. code-block:: bash + + sqlite-utils rows dogs.db dogs -c name --where 'name = :name' -p name Cleo + +.. code-block:: output + + [{"name": "Cleo"}] + +You can define a sort order using ``--order column`` or ``--order 'column desc'``. + +Use ``--limit N`` to only return the first ``N`` rows. Use ``--offset N`` to return rows starting from the specified offset. + +.. note:: + In Python: :ref:`table.rows ` CLI reference: :ref:`sqlite-utils rows ` + .. _cli_tables: Listing tables ============== -You can list the names of tables in a database using the ``tables`` command:: +You can list the names of tables in a database using the ``tables`` command: + +.. code-block:: bash + + sqlite-utils tables mydb.db + +.. code-block:: output - $ sqlite-utils tables mydb.db [{"table": "dogs"}, {"table": "cats"}, {"table": "chickens"}] -You can output this list in CSV using the ``--csv`` or ``--tsv`` options:: +You can output this list in CSV using the ``--csv`` or ``--tsv`` options: + +.. code-block:: bash + + sqlite-utils tables mydb.db --csv --no-headers + +.. code-block:: output - $ sqlite-utils tables mydb.db --csv --no-headers dogs cats chickens -If you just want to see the FTS4 tables, you can use ``--fts4`` (or ``--fts5`` for FTS5 tables):: +If you just want to see the FTS4 tables, you can use ``--fts4`` (or ``--fts5`` for FTS5 tables): + +.. code-block:: bash + + sqlite-utils tables docs.db --fts4 + +.. code-block:: output - $ sqlite-utils tables docs.db --fts4 [{"table": "docs_fts"}] -Use ``--counts`` to include a count of the number of rows in each table:: +Use ``--counts`` to include a count of the number of rows in each table: + +.. code-block:: bash + + sqlite-utils tables mydb.db --counts + +.. code-block:: output - $ sqlite-utils tables mydb.db --counts [{"table": "dogs", "count": 12}, {"table": "cats", "count": 332}, {"table": "chickens", "count": 9}] -Use ``--columns`` to include a list of columns in each table:: +Use ``--columns`` to include a list of columns in each table: + +.. code-block:: bash + + sqlite-utils tables dogs.db --counts --columns + +.. code-block:: output - $ sqlite-utils tables dogs.db --counts --columns [{"table": "Gosh", "count": 0, "columns": ["c1", "c2", "c3"]}, {"table": "Gosh2", "count": 0, "columns": ["c1", "c2", "c3"]}, {"table": "dogs", "count": 2, "columns": ["id", "age", "name"]}] -Use ``--schema`` to include the schema of each table:: +Use ``--schema`` to include the schema of each table: + +.. code-block:: bash + + sqlite-utils tables dogs.db --schema --table + +.. code-block:: output - $ sqlite-utils tables dogs.db --schema --table table schema ------- ----------------------------------------------- Gosh CREATE TABLE Gosh (c1 text, c2 text, c3 text) Gosh2 CREATE TABLE Gosh2 (c1 text, c2 text, c3 text) - dogs CREATE TABLE [dogs] ( - [id] INTEGER, - [age] INTEGER, - [name] TEXT) + dogs CREATE TABLE "dogs" ( + "id" INTEGER, + "age" INTEGER, + "name" TEXT) The ``--nl``, ``--csv``, ``--tsv``, ``--table`` and ``--fmt`` options are also available. +.. note:: + In Python: :ref:`db.tables or db.table_names() ` CLI reference: :ref:`sqlite-utils tables ` + .. _cli_views: Listing views ============= -The ``views`` command shows any views defined in the database:: +The ``views`` command shows any views defined in the database: + +.. code-block:: bash + + sqlite-utils views sf-trees.db --table --counts --columns --schema + +.. code-block:: output - $ sqlite-utils views sf-trees.db --table --counts --columns --schema view count columns schema --------- ------- -------------------- -------------------------------------------------------------- demo_view 189144 ['qSpecies'] CREATE VIEW demo_view AS select qSpecies from Street_Tree_List @@ -457,14 +843,22 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. note:: + In Python: :ref:`db.views or db.view_names() ` CLI reference: :ref:`sqlite-utils views ` + .. _cli_indexes: Listing indexes =============== -The ``indexes`` command lists any indexes configured for the database:: +The ``indexes`` command lists any indexes configured for the database: + +.. code-block:: bash + + sqlite-utils indexes covid.db --table + +.. code-block:: output - $ sqlite-utils indexes covid.db --table table index_name seqno cid name desc coll key -------------------------------- ------------------------------------------------------ ------- ----- ----------------- ------ ------ ----- johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_combined_key 0 12 combined_key 0 BINARY 1 @@ -476,59 +870,83 @@ The ``indexes`` command lists any indexes configured for the database:: ny_times_us_counties idx_ny_times_us_counties_county 0 1 county 0 BINARY 1 ny_times_us_counties idx_ny_times_us_counties_state 0 2 state 0 BINARY 1 -It shows indexes across all tables. To see indexes for specific tables, list those after the database:: +It shows indexes across all tables. To see indexes for specific tables, list those after the database: - $ sqlite-utils indexes covid.db johns_hopkins_csse_daily_reports --table +.. code-block:: bash + + sqlite-utils indexes covid.db johns_hopkins_csse_daily_reports --table The command defaults to only showing the columns that are explicitly part of the index. To also include auxiliary columns use the ``--aux`` option - these columns will be listed with a ``key`` of ``0``. The command takes the same format options as the ``tables`` and ``views`` commands. +.. note:: + In Python: :ref:`table.indexes ` CLI reference: :ref:`sqlite-utils indexes ` + .. _cli_triggers: Listing triggers ================ -The ``triggers`` command shows any triggers configured for the database:: +The ``triggers`` command shows any triggers configured for the database: + +.. code-block:: bash + + sqlite-utils triggers global-power-plants.db --table + +.. code-block:: output - $ sqlite-utils triggers global-power-plants.db --table name table sql --------------- --------- ----------------------------------------------------------------- - plants_insert plants CREATE TRIGGER [plants_insert] AFTER INSERT ON [plants] + plants_insert plants CREATE TRIGGER "plants_insert" AFTER INSERT ON "plants" BEGIN - INSERT OR REPLACE INTO [_counts] + INSERT OR REPLACE INTO "_counts" VALUES ( 'plants', COALESCE( - (SELECT count FROM [_counts] WHERE [table] = 'plants'), + (SELECT count FROM "_counts" WHERE "table" = 'plants'), 0 ) + 1 ); END -It defaults to showing triggers for all tables. To see triggers for one or more specific tables pass their names as arguments:: +It defaults to showing triggers for all tables. To see triggers for one or more specific tables pass their names as arguments: - $ sqlite-utils triggers global-power-plants.db plants +.. code-block:: bash + + sqlite-utils triggers global-power-plants.db plants The command takes the same format options as the ``tables`` and ``views`` commands. +.. note:: + In Python: :ref:`table.triggers or db.triggers ` CLI reference: :ref:`sqlite-utils triggers ` + .. _cli_schema: Showing the schema ================== -The ``sqlite-utils schema`` command shows the full SQL schema for the database:: +The ``sqlite-utils schema`` command shows the full SQL schema for the database: + +.. code-block:: bash + + sqlite-utils schema dogs.db + +.. code-block:: output - $ sqlite-utils schema dogs.db CREATE TABLE "dogs" ( - [id] INTEGER PRIMARY KEY, - [name] TEXT + "id" INTEGER PRIMARY KEY, + "name" TEXT ); -This will show the schema for every table and index in the database. To view the schema just for a specified subset of tables pass those as additional arguments:: +This will show the schema for every table and index in the database. To view the schema just for a specified subset of tables pass those as additional arguments: - $ sqlite-utils schema dogs.db dogs chickens - ... +.. code-block:: bash + + sqlite-utils schema dogs.db dogs chickens + +.. note:: + In Python: :ref:`table.schema ` or :ref:`db.schema ` CLI reference: :ref:`sqlite-utils schema ` .. _cli_analyze_tables: @@ -537,9 +955,14 @@ Analyzing tables When working with a new database it can be useful to get an idea of the shape of the data. The ``sqlite-utils analyze-tables`` command inspects specified tables (or all tables) and calculates some useful details about each of the columns in those tables. -To inspect the ``tags`` table in the ``github.db`` database, run the following:: +To inspect the ``tags`` table in the ``github.db`` database, run the following: + +.. code-block:: bash + + sqlite-utils analyze-tables github.db tags + +.. code-block:: output - $ sqlite-utils analyze-tables github.db tags tags.repo: (1/3) Total rows: 261 @@ -586,38 +1009,57 @@ To inspect the ``tags`` table in the ``github.db`` database, run the following:: For each column this tool displays the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values. -If you do not specify any tables every table in the database will be analyzed:: +If you do not specify any tables every table in the database will be analyzed: - $ sqlite-utils analyze-tables github.db +.. code-block:: bash -If you wish to analyze one or more specific columns, use the ``-c`` option:: + sqlite-utils analyze-tables github.db - $ sqlite-utils analyze-tables github.db tags -c sha +If you wish to analyze one or more specific columns, use the ``-c`` option: + +.. code-block:: bash + + sqlite-utils analyze-tables github.db tags -c sha + +To show more than 10 common values, use ``--common-limit 20``. To skip the most common or least common value analysis, use ``--no-most`` or ``--no-least``: + +.. code-block:: bash + + sqlite-utils analyze-tables github.db tags --common-limit 20 --no-least + +.. note:: + In Python: :ref:`table.analyze_column() ` CLI reference: :ref:`sqlite-utils analyze-tables ` .. _cli_analyze_tables_save: Saving the analyzed table details --------------------------------- -``analyze-tables`` can take quite a while to run for large database files. You can save the results of the analysis to a database table called ``_analyze_tables_`` using the ``--save`` option:: +``analyze-tables`` can take quite a while to run for large database files. You can save the results of the analysis to a database table called ``_analyze_tables_`` using the ``--save`` option: - $ sqlite-utils analyze-tables github.db --save +.. code-block:: bash -The ``_analyze_tables_`` table has the following schema:: + sqlite-utils analyze-tables github.db --save - CREATE TABLE [_analyze_tables_] ( - [table] TEXT, - [column] TEXT, - [total_rows] INTEGER, - [num_null] INTEGER, - [num_blank] INTEGER, - [num_distinct] INTEGER, - [most_common] TEXT, - [least_common] TEXT, - PRIMARY KEY ([table], [column]) +The ``_analyze_tables_`` table has the following schema: + +.. code-block:: sql + + CREATE TABLE "_analyze_tables_" ( + "table" TEXT, + "column" TEXT, + "total_rows" INTEGER, + "num_null" INTEGER, + "num_blank" INTEGER, + "num_distinct" INTEGER, + "most_common" TEXT, + "least_common" TEXT, + PRIMARY KEY ("table", "column") ); -The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most common and least common values that look like this:: +The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most common and least common values that look like this: + +.. code-block:: json [ ["Del Libertador, Av", 5068], @@ -632,6 +1074,66 @@ The ``most_common`` and ``least_common`` columns will contain nested JSON arrays ["Condarco", 4288] ] +.. _cli_create_database: + +Creating an empty database +========================== + +You can create a new empty database file using the ``create-database`` command: + +.. code-block:: bash + + sqlite-utils create-database empty.db + +To enable :ref:`cli_wal` on the newly created database add the ``--enable-wal`` option: + +.. code-block:: bash + + sqlite-utils create-database empty.db --enable-wal + +To enable SpatiaLite metadata on a newly created database, add the ``--init-spatialite`` flag: + +.. code-block:: bash + + sqlite-utils create-database empty.db --init-spatialite + +That will look for SpatiaLite in a set of predictable locations. To load it from somewhere else, use the ``--load-extension`` option: + +.. code-block:: bash + + sqlite-utils create-database empty.db --init-spatialite --load-extension /path/to/spatialite.so + +.. _cli_migrate: + +Running migrations +================== + +The ``migrate`` command applies pending Python migrations to a database. For the full migration file format and Python API, see :ref:`migrations`. + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py + +If you omit the migration path it will search the current directory and subdirectories for files called ``migrations.py``: + +.. code-block:: bash + + sqlite-utils migrate creatures.db + +Use ``--list`` to list applied and pending migrations without running them: + +.. code-block:: bash + + sqlite-utils migrate creatures.db --list + +Use ``--stop-before`` to stop before a named migration. The option can be passed more than once, and can target a specific migration set using ``migration_set:migration_name``: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py \ + --stop-before creatures:add_weight \ + --stop-before sales:drop_index + .. _cli_inserting_data: Inserting JSON data @@ -641,15 +1143,19 @@ If you have data as JSON, you can use ``sqlite-utils insert tablename`` to inser You can pass in a single JSON object or a list of JSON objects, either as a filename or piped directly to standard-in (by using ``-`` as the filename). -Here's the simplest possible example:: +Here's the simplest possible example: - $ echo '{"name": "Cleo", "age": 4}' | sqlite-utils insert dogs.db dogs - +.. code-block:: bash + + echo '{"name": "Cleo", "age": 4}' | sqlite-utils insert dogs.db dogs - To specify a column as the primary key, use ``--pk=column_name``. To create a compound primary key across more than one column, use ``--pk`` multiple times. -If you feed it a JSON list it will insert multiple records. For example, if ``dogs.json`` looks like this:: +If you feed it a JSON list it will insert multiple records. For example, if ``dogs.json`` looks like this: + +.. code-block:: json [ { @@ -669,24 +1175,39 @@ If you feed it a JSON list it will insert multiple records. For example, if ``do } ] -You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so:: +You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so: - $ sqlite-utils insert dogs.db dogs dogs.json --pk=id +.. code-block:: bash -You can skip inserting any records that have a primary key that already exists using ``--ignore``:: + sqlite-utils insert dogs.db dogs dogs.json --pk=id - $ sqlite-utils insert dogs.db dogs dogs.json --ignore +Pass ``--pk`` multiple times to define a compound primary key. -You can delete all the existing rows in the table before inserting the new records using ``--truncate``:: +You can skip inserting any records that have a primary key that already exists using ``--ignore``: - $ sqlite-utils insert dogs.db dogs dogs.json --truncate +.. code-block:: bash + + sqlite-utils insert dogs.db dogs dogs.json --pk=id --ignore + +You can delete all the existing rows in the table before inserting the new records using ``--truncate``: + +.. code-block:: bash + + sqlite-utils insert dogs.db dogs dogs.json --truncate + +You can add the ``--analyze`` option to run ``ANALYZE`` against the table after the rows have been inserted. + +.. note:: + In Python: :ref:`table.insert_all() ` CLI reference: :ref:`sqlite-utils insert ` .. _cli_inserting_data_binary: Inserting binary data --------------------- -You can insert binary data into a BLOB column by first encoding it using base64 and then structuring it like this:: +You can insert binary data into a BLOB column by first encoding it using base64 and then structuring it like this: + +.. code-block:: json [ { @@ -703,18 +1224,38 @@ You can insert binary data into a BLOB column by first encoding it using base64 Inserting newline-delimited JSON -------------------------------- -You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette `__ can export newline-delimited JSON, you can combine the two tools like so:: +You can also import newline-delimited JSON (see `JSON Lines `__) using the ``--nl`` option: - $ curl -L "https://latest.datasette.io/fixtures/facetable.json?_shape=array&_nl=on" \ +.. code-block:: bash + + echo '{"id": 1, "name": "Cleo"} + {"id": 2, "name": "Suna"}' | sqlite-utils insert creatures.db creatures - --nl + +Newline-delimited JSON consists of full JSON objects separated by newlines. + +If you are processing data using ``jq`` you can use the ``jq -c`` option to output valid newline-delimited JSON. + +Since `Datasette `__ can export newline-delimited JSON, you can combine the Datasette and ``sqlite-utils`` like so: + +.. code-block:: bash + + curl -L "https://latest.datasette.io/fixtures/facetable.json?_shape=array&_nl=on" \ | sqlite-utils insert nl-demo.db facetable - --pk=id --nl -This also means you pipe ``sqlite-utils`` together to easily create a new SQLite database file containing the results of a SQL query against another database:: +You can also pipe ``sqlite-utils`` together to create a new SQLite database file containing the results of a SQL query against another database: - $ sqlite-utils sf-trees.db \ +.. code-block:: bash + + sqlite-utils sf-trees.db \ "select TreeID, qAddress, Latitude, Longitude from Street_Tree_List" --nl \ | sqlite-utils insert saved.db trees - --nl - # This creates saved.db with a single table called trees: - $ sqlite-utils saved.db "select * from trees limit 5" --csv + +.. code-block:: bash + + sqlite-utils saved.db "select * from trees limit 5" --csv + +.. code-block:: output + TreeID,qAddress,Latitude,Longitude 141565,501X Baker St,37.7759676911831,-122.441396661871 232565,940 Elizabeth St,37.7517102172731,-122.441498017841 @@ -727,11 +1268,13 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite Flattening nested JSON objects ------------------------------ -``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. +``sqlite-utils insert`` and ``sqlite-utils memory`` both expect incoming JSON data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. If your data is nested you can use the ``--flatten`` option to create columns that are derived from the nested data. -Consider this example document, in a file called ``log.json``:: +Consider this example document, in a file called ``log.json``: + +.. code-block:: json { "httpRequest": { @@ -746,23 +1289,27 @@ Consider this example document, in a file called ``log.json``:: } } -Inserting this into a table using ``sqlite-utils insert logs.db log log.json`` will create a table with the following schema:: +Inserting this into a table using ``sqlite-utils insert logs.db logs log.json`` will create a table with the following schema: - CREATE TABLE [logs] ( - [httpRequest] TEXT, - [insertId] TEXT, - [labels] TEXT +.. code-block:: sql + + CREATE TABLE "logs" ( + "httpRequest" TEXT, + "insertId" TEXT, + "labels" TEXT ); -With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db log log.json --flatten`` will create the following schema instead:: +With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db logs log.json --flatten`` will create the following schema instead: - CREATE TABLE [logs] ( - [httpRequest_latency] TEXT, - [httpRequest_requestMethod] TEXT, - [httpRequest_requestSize] TEXT, - [httpRequest_status] INTEGER, - [insertId] TEXT, - [labels_service] TEXT +.. code-block:: sql + + CREATE TABLE "logs" ( + "httpRequest_latency" TEXT, + "httpRequest_requestMethod" TEXT, + "httpRequest_requestSize" TEXT, + "httpRequest_status" INTEGER, + "insertId" TEXT, + "labels_service" TEXT ); .. _cli_insert_csv_tsv: @@ -770,44 +1317,104 @@ With the ``--flatten`` option columns will be created using ``topkey_nextkey`` c Inserting CSV or TSV data ========================= -If your data is in CSV format, you can insert it using the ``--csv`` option:: +If your data is in CSV format, you can insert it using the ``--csv`` option: - $ sqlite-utils insert dogs.db dogs docs.csv --csv +.. code-block:: bash -For tab-delimited data, use ``--tsv``:: + sqlite-utils insert dogs.db dogs dogs.csv --csv - $ sqlite-utils insert dogs.db dogs dogs.tsv --tsv +For tab-delimited data, use ``--tsv``: -Data is expected to be encoded as Unicode UTF-8. If your data is an another character encoding you can specify it using the ``--encoding`` option:: +.. code-block:: bash - $ sqlite-utils insert dogs.db dogs dogs.tsv --tsv --encoding=latin-1 + sqlite-utils insert dogs.db dogs dogs.tsv --tsv + +Data is expected to be encoded as Unicode UTF-8. If your data is an another character encoding you can specify it using the ``--encoding`` option: + +.. code-block:: bash + + sqlite-utils insert dogs.db dogs dogs.tsv --tsv --encoding=latin-1 + +To stop inserting after a specified number of records - useful for getting a faster preview of a large file - use the ``--stop-after`` option: + +.. code-block:: bash + + sqlite-utils insert dogs.db dogs dogs.csv --csv --stop-after=10 A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option. -By default every column inserted from a CSV or TSV file will be of type ``TEXT``. To automatically detect column types - resulting in a mix of ``TEXT``, ``INTEGER`` and ``FLOAT`` columns, use the ``--detect-types`` option (or its shortcut ``-d``). +By default, column types are automatically detected for CSV or TSV files - resulting in a mix of ``TEXT``, ``INTEGER`` and ``REAL`` columns. To disable type detection and treat all columns as ``TEXT``, use the ``--no-detect-types`` option. -For example, given a ``creatures.csv`` file containing this:: +Detected types are only applied when the table is created by the command. Inserting CSV or TSV data into a table that already exists leaves the existing column types unchanged - values are inserted using the table's existing schema. + +For example, given a ``creatures.csv`` file containing this: + +.. code-block:: name,age,weight Cleo,6,45.5 Dori,1,3.5 -The following command:: +The following command: - $ sqlite-utils insert creatures.db creatures creatures.csv --csv --detect-types +.. code-block:: bash -Will produce this schema:: + sqlite-utils insert creatures.db creatures creatures.csv --csv + +Will produce this schema with automatically detected types: + +.. code-block:: bash + + sqlite-utils schema creatures.db + +.. code-block:: output - $ sqlite-utils schema creatures.db CREATE TABLE "creatures" ( - [name] TEXT, - [age] INTEGER, - [weight] FLOAT + "name" TEXT, + "age" INTEGER, + "weight" REAL ); -You can set the ``SQLITE_UTILS_DETECT_TYPES`` environment variable if you want ``--detect-types`` to be the default behavior:: +.. _cli_insert_csv_tsv_column_types: - $ export SQLITE_UTILS_DETECT_TYPES=1 +Overriding column types +----------------------- + +Use ``--type column-name type`` to override the type automatically chosen when the table is created. This option can be used more than once, and works with both ``insert`` and ``upsert``: + +.. code-block:: bash + + sqlite-utils insert places.db places places.csv --csv \ + --type zipcode text \ + --type score real + +This is useful for values such as ZIP codes, which may look like integers but should be stored as ``TEXT`` to preserve leading zeros. + +The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or ``BLOB``. Column types are matched case-insensitively. + +As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged. + +To disable type detection and treat all columns as TEXT, use ``--no-detect-types``: + +.. code-block:: bash + + sqlite-utils insert creatures.db creatures creatures.csv --csv --no-detect-types + +If a CSV or TSV file includes empty cells, like this one: + +:: + + name,age,weight + Cleo,6, + Dori,,3.5 + +They will be imported into SQLite as empty string values, ``""``. + +To import them as ``NULL`` values instead, use the ``--empty-null`` option: + +.. code-block:: bash + + sqlite-utils insert creatures.db creatures creatures.csv --csv --empty-null .. _cli_insert_csv_tsv_delimiter: @@ -816,7 +1423,9 @@ Alternative delimiters and quote characters If your file uses a delimiter other than ``,`` or a quote character other than ``"`` you can attempt to detect delimiters or you can specify them explicitly. -The ``--sniff`` option can be used to attempt to detect the delimiters:: +The ``--sniff`` option can be used to attempt to detect the delimiters: + +.. code-block:: bash sqlite-utils insert dogs.db dogs dogs.csv --sniff @@ -828,9 +1437,11 @@ Here's a CSV file that uses ``;`` for delimiters and the ``|`` symbol for quote Cleo;|Very fine; a friendly dog| Pancakes;A local corgi -You can import that using:: +You can import that using: - $ sqlite-utils insert dogs.db dogs dogs.csv --delimiter=";" --quotechar="|" +.. code-block:: bash + + sqlite-utils insert dogs.db dogs dogs.csv --delimiter=";" --quotechar="|" Passing ``--delimiter``, ``--quotechar`` or ``--sniff`` implies ``--csv``, so you can omit the ``--csv`` option. @@ -845,19 +1456,176 @@ If your file does not include this row, you can use the ``--no-headers`` option If you do this, the table will be created with column names called ``untitled_1`` and ``untitled_2`` and so on. You can then rename them using the ``sqlite-utils transform ... --rename`` command, see :ref:`cli_transform_table`. +.. _cli_insert_unstructured: + +Inserting unstructured data with \-\-lines and \-\-text +======================================================= + +If you have an unstructured file you can insert its contents into a table with a single ``line`` column containing each line from the file using ``--lines``. This can be useful if you intend to further analyze those lines using SQL string functions or :ref:`sqlite-utils convert `: + +.. code-block:: bash + + sqlite-utils insert logs.db loglines logfile.log --lines + +This will produce the following schema: + +.. code-block:: sql + + CREATE TABLE "loglines" ( + "line" TEXT + ); + +You can also insert the entire contents of the file into a single column called ``text`` using ``--text``: + +.. code-block:: bash + + sqlite-utils insert content.db content file.txt --text + +The schema here will be: + +.. code-block:: sql + + CREATE TABLE "content" ( + "text" TEXT + ); + +.. _cli_insert_convert: + +Applying conversions while inserting data +========================================= + +The ``--convert`` option can be used to apply a Python conversion function to imported data before it is inserted into the database. It works in a similar way to :ref:`sqlite-utils convert `. + +Your Python function will be passed a dictionary called ``row`` for each item that is being imported. You can modify that dictionary and return it - or return a fresh dictionary - to change the data that will be inserted. + +Given a JSON file called ``dogs.json`` containing this: + +.. code-block:: json + + [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Pancakes"} + ] + +The following command will insert that data and add an ``is_good`` column set to ``1`` for each dog: + +.. code-block:: bash + + sqlite-utils insert dogs.db dogs dogs.json --convert 'row["is_good"] = 1' + +The ``--convert`` option also works with the ``--csv``, ``--tsv`` and ``--nl`` insert options. + +As with ``sqlite-utils convert`` you can use ``--import`` to import additional Python modules, see :ref:`cli_convert_import` for details. + +You can also pass code that runs some initialization steps and defines a ``convert(value)`` function, see :ref:`cli_convert_complex`. + +.. _cli_insert_convert_lines: + +\-\-convert with \-\-lines +-------------------------- + +Things work slightly differently when combined with the ``--lines`` or ``--text`` options. + +With ``--lines``, instead of being passed a ``row`` dictionary your function will be passed a ``line`` string representing each line of the input. Given a file called ``access.log`` containing the following:: + + INFO: 127.0.0.1:60581 - GET / HTTP/1.1 200 OK + INFO: 127.0.0.1:60581 - GET /foo/-/static/app.css?cead5a HTTP/1.1 200 OK + +You could convert it into structured data like so: + +.. code-block:: bash + + sqlite-utils insert logs.db loglines access.log --convert ' + type, source, _, verb, path, _, status, _ = line.split() + return { + "type": type, + "source": source, + "verb": verb, + "path": path, + "status": status, + }' --lines + +The resulting table would look like this: + +====== =============== ====== ============================ ======== +type source verb path status +====== =============== ====== ============================ ======== +INFO: 127.0.0.1:60581 GET / 200 +INFO: 127.0.0.1:60581 GET /foo/-/static/app.css?cead5a 200 +====== =============== ====== ============================ ======== + +.. _cli_insert_convert_text: + +\-\-convert with \-\-text +------------------------- + +With ``--text`` the entire input to the command will be made available to the function as a variable called ``text``. + +The function can return a single dictionary which will be inserted as a single row, or it can return a list or iterator of dictionaries, each of which will be inserted. + +Here's how to use ``--convert`` and ``--text`` to insert one record per word in the input: + +.. code-block:: bash + + echo 'A bunch of words' | sqlite-utils insert words.db words - \ + --text --convert '({"word": w} for w in text.split())' + +The result looks like this: + +.. code-block:: bash + + sqlite-utils dump words.db + +.. code-block:: output + + BEGIN TRANSACTION; + CREATE TABLE "words" ( + "word" TEXT + ); + INSERT INTO "words" VALUES('A'); + INSERT INTO "words" VALUES('bunch'); + INSERT INTO "words" VALUES('of'); + INSERT INTO "words" VALUES('words'); + COMMIT; + + +.. _cli_insert_code: + +Inserting rows generated by Python code +======================================= + +Instead of providing a ``FILE`` to import, you can use the ``--code`` option to pass a block of Python code that generates the rows to insert. This is the command-line equivalent of calling ``db["creatures"].insert_all(rows())`` from the :ref:`Python API `. + +Your code should define either a ``rows()`` function that returns or yields dictionaries, or a ``rows`` iterable such as a list of dictionaries: + +.. code-block:: bash + + sqlite-utils insert data.db creatures --code ' + def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} + ' --pk id + +``--code`` can also be given a path to a Python ``.py`` file. + +The ``--code`` option works with both ``sqlite-utils insert`` and ``sqlite-utils upsert``, and composes with table options such as ``--pk``, ``--replace``, ``--alter``, ``--not-null`` and ``--default``. It cannot be combined with a ``FILE`` argument or with input format options such as ``--csv`` or ``--convert``. + .. _cli_insert_replace: Insert-replacing data ===================== -Insert-replacing works exactly like inserting, with the exception that if your data has a primary key that matches an already existing record that record will be replaced with the new data. +The ``--replace`` option to ``insert`` causes any existing records with the same primary key to be replaced entirely by the new records. -After running the above ``dogs.json`` example, try running this:: +To replace a dog with in ID of 2 with a new record, run the following: - $ echo '{"id": 2, "name": "Pancakes", "age": 3}' | \ +.. code-block:: bash + + echo '{"id": 2, "name": "Pancakes", "age": 3}' | \ sqlite-utils insert dogs.db dogs - --pk=id --replace -This will replace the record for id=2 (Pancakes) with a new record with an updated age. +.. note:: + In Python: :ref:`table.insert(..., replace=True) ` CLI reference: :ref:`sqlite-utils insert ` .. _cli_upsert: @@ -868,58 +1636,115 @@ Upserting is update-or-insert. If a row exists with the specified primary key th Unlike ``insert --replace``, an upsert will ignore any column values that exist but are not present in the upsert document. -For example:: +For example: - $ echo '{"id": 2, "age": 4}' | \ +.. code-block:: bash + + echo '{"id": 2, "age": 4}' | \ sqlite-utils upsert dogs.db dogs - --pk=id -This will update the dog with id=2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is. +This will update the dog with an ID of 2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is. + +If the table already exists and has a primary key, you can omit the ``--pk`` option and ``sqlite-utils`` will use that existing primary key. The command will fail if you reference columns that do not exist on the table. To automatically create missing columns, use the ``--alter`` option. .. note:: ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. + +.. note:: + In Python: :ref:`table.upsert() ` CLI reference: :ref:`sqlite-utils upsert ` + +.. _cli_bulk: + +Executing SQL in bulk +===================== + +If you have a JSON, newline-delimited JSON, CSV or TSV file you can execute a bulk SQL query using each of the records in that file using the ``sqlite-utils bulk`` command. + +The command takes the database file, the SQL to be executed and the file containing records to be used when evaluating the SQL query. + +The SQL query should include ``:named`` parameters that match the keys in the records. + +For example, given a ``chickens.csv`` CSV file containing the following: + +.. code-block:: + + id,name + 1,Blue + 2,Snowy + 3,Azi + 4,Lila + 5,Suna + 6,Cardi + +You could insert those rows into a pre-created ``chickens`` table like so: + +.. code-block:: bash + + sqlite-utils bulk chickens.db \ + 'insert into chickens (id, name) values (:id, :name)' \ + chickens.csv --csv + +This command takes the same options as the ``sqlite-utils insert`` command - so it defaults to expecting JSON but can accept other formats using ``--csv`` or ``--tsv`` or ``--nl`` or other options described above. + +By default all of the SQL queries will be executed in a single transaction. To commit every 20 records, use ``--batch-size 20``. + .. _cli_insert_files: -Inserting binary data from files -================================ +Inserting data from files +========================= -SQLite ``BLOB`` columns can be used to store binary content. It can be useful to insert the contents of files into a SQLite table. +The ``insert-files`` command can be used to insert the content of files, along with their metadata, into a SQLite table. -The ``insert-files`` command can be used to insert the content of files, along with their metadata. +Here's an example that inserts all of the GIF files in the current directory into a ``gifs.db`` database, placing the file contents in an ``images`` table: -Here's an example that inserts all of the GIF files in the current directory into a ``gifs.db`` database, placing the file contents in an ``images`` table:: +.. code-block:: bash - $ sqlite-utils insert-files gifs.db images *.gif + sqlite-utils insert-files gifs.db images *.gif -You can also pass one or more directories, in which case every file in those directories will be added recursively:: +You can also pass one or more directories, in which case every file in those directories will be added recursively: - $ sqlite-utils insert-files gifs.db images path/to/my-gifs +.. code-block:: bash -By default this command will create a table with the following schema:: + sqlite-utils insert-files gifs.db images path/to/my-gifs - CREATE TABLE [images] ( - [path] TEXT PRIMARY KEY, - [content] BLOB, - [size] INTEGER +By default this command will create a table with the following schema: + +.. code-block:: sql + + CREATE TABLE "images" ( + "path" TEXT PRIMARY KEY, + "content" BLOB, + "size" INTEGER ); -You can customize the schema using one or more ``-c`` options. For a table schema that includes just the path, MD5 hash and last modification time of the file, you would use this:: +Content will be treated as binary by default and stored in a ``BLOB`` column. You can use the ``--text`` option to store that content in a ``TEXT`` column instead. - $ sqlite-utils insert-files gifs.db images *.gif -c path -c md5 -c mtime --pk=path +You can customize the schema using one or more ``-c`` options. For a table schema that includes just the path, MD5 hash and last modification time of the file, you would use this: -This will result in the following schema:: +.. code-block:: bash - CREATE TABLE [images] ( - [path] TEXT PRIMARY KEY, - [md5] TEXT, - [mtime] FLOAT + sqlite-utils insert-files gifs.db images *.gif -c path -c md5 -c mtime --pk=path + +This will result in the following schema: + +.. code-block:: sql + + CREATE TABLE "images" ( + "path" TEXT PRIMARY KEY, + "md5" TEXT, + "mtime" REAL ); -You can change the name of one of these columns using a ``-c colname:coldef`` parameter. To rename the ``mtime`` column to ``last_modified`` you would use this:: +Note that there's no ``content`` column here at all - if you specify custom columns using ``-c`` you need to include ``-c content`` to create that column. - $ sqlite-utils insert-files gifs.db images *.gif \ +You can change the name of one of these columns using a ``-c colname:coldef`` parameter. To rename the ``mtime`` column to ``last_modified`` you would use this: + +.. code-block:: bash + + sqlite-utils insert-files gifs.db images *.gif \ -c path -c md5 -c last_modified:mtime --pk=path You can pass ``--replace`` or ``--upsert`` to indicate what should happen if you try to insert a file with an existing primary key. Pass ``--alter`` to cause any missing columns to be added to the table. @@ -940,6 +1765,8 @@ The full list of column definitions you can use is as follows: The permission bits of the file, as an integer - you may want to convert this to octal ``content`` The binary file contents, which will be stored as a BLOB +``content_text`` + The text file contents, which will be stored as TEXT ``mtime`` The modification time of the file, as floating point seconds since the Unix epoch ``ctime`` @@ -954,14 +1781,20 @@ The full list of column definitions you can use is as follows: The creation time is an ISO timestamp ``size`` The integer size of the file in bytes +``stem`` + The filename without the extension - for ``file.txt.gz`` this would be ``file.txt`` +``suffix`` + The file extension - for ``file.txt.gz`` this would be ``.gz`` -You can insert data piped from standard input like this:: +You can insert data piped from standard input like this: + +.. code-block:: bash cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg The ``-`` argument indicates data should be read from standard input. The string passed using the ``--name`` option will be used for the file name and path values. -When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``. +When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``content_text``, ``sha256``, ``md5`` and ``size``. .. _cli_convert: @@ -970,36 +1803,143 @@ Converting data in columns The ``convert`` command can be used to transform the data in a specified column - for example to parse a date string into an ISO timestamp, or to split a string of tags into a JSON array. -The command accepts a database, table, one or more columns and a string of Python code to be executed against the values from those columns. The following example would replace the values in the ``headline`` column in the ``articles`` table with an upper-case version:: +The command accepts a database, table, one or more columns and a string of Python code to be executed against the values from those columns. The following example would replace the values in the ``headline`` column in the ``articles`` table with an upper-case version: - $ sqlite-utils convert content.db articles headline 'value.upper()' +.. code-block:: bash + + sqlite-utils convert content.db articles headline 'value.upper()' The Python code is passed as a string. Within that Python code the ``value`` variable will be the value of the current column. -The code you provide will be compiled into a function that takes ``value`` as a single argument. If you break your function body into multiple lines the last line should be a ``return`` statement:: +The code you provide will be compiled into a function that takes ``value`` as a single argument. If you break your function body into multiple lines the last line should be a ``return`` statement: - $ sqlite-utils convert content.db articles headline ' +.. code-block:: bash + + sqlite-utils convert content.db articles headline ' value = str(value) return value.upper()' -You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options. This example uses the ``textwrap`` module to wrap the ``content`` column at 100 characters:: +Your code will be automatically wrapped in a function, but you can also define a function called ``convert(value)`` which will be called, if available: - $ sqlite-utils convert content.db articles content \ +.. code-block:: bash + + sqlite-utils convert content.db articles headline ' + def convert(value): + return value.upper()' + +Use a ``CODE`` value of ``-`` to read from standard input: + +.. code-block:: bash + + cat mycode.py | sqlite-utils convert content.db articles headline - + +Where ``mycode.py`` contains a fragment of Python code that looks like this: + +.. code-block:: python + + def convert(value): + return value.upper() + +The conversion will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``: + +.. code-block:: bash + + sqlite-utils convert content.db articles headline 'value.upper()' \ + --where "headline like '%cat%'" + +You can include named parameters in your where clause and populate them using one or more ``--param`` options: + +.. code-block:: bash + + sqlite-utils convert content.db articles headline 'value.upper()' \ + --where "headline like :query" \ + --param query '%cat%' + +The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. + +.. note:: + In Python: :ref:`table.convert() ` CLI reference: :ref:`sqlite-utils convert ` + +.. _cli_convert_import: + +Importing additional modules +---------------------------- + +You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options. This example uses the ``textwrap`` module to wrap the ``content`` column at 100 characters: + +.. code-block:: bash + + sqlite-utils convert content.db articles content \ '"\n".join(textwrap.wrap(value, 100))' \ --import=textwrap -The transformation will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``:: +This supports nested imports as well, for example to use `ElementTree `__: - $ sqlite-utils convert content.db articles headline 'value.upper()' \ - --where "headline like '%cat%'" +.. code-block:: bash -You can include named parameters in your where clause and populate them using one or more ``--param`` options:: + sqlite-utils convert content.db articles content \ + 'xml.etree.ElementTree.fromstring(value).attrib["title"]' \ + --import=xml.etree.ElementTree - $ sqlite-utils convert content.db articles headline 'value.upper()' \ - --where "headline like :like" \ - --param like '%cat%' +.. _cli_convert_debugger: -The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. +Using the debugger +------------------ + +If an error occurs while running your conversion operation you may see a message like this:: + + user-defined function raised exception + +Add the ``--pdb`` option to catch the error and open the Python debugger at that point. The conversion operation will exit after you type ``q`` in the debugger. + +Here's an example debugging session. First, create a ``articles`` table with invalid XML in the ``content`` column: + +.. code-block:: bash + + echo '{"content": "This is not XML"}' | sqlite-utils insert content.db articles - + +Now run the conversion with the ``--pdb`` option: + +.. code-block:: bash + + sqlite-utils convert content.db articles content \ + 'xml.etree.ElementTree.fromstring(value).attrib["title"]' \ + --import=xml.etree.ElementTree \ + --pdb + +When the error occurs the debugger will open:: + + Exception raised, dropping into pdb...: syntax error: line 1, column 0 + > .../python3.11/xml/etree/ElementTree.py(1338)XML() + -> parser.feed(text) + (Pdb) args + text = 'This is not XML' + parser = + (Pdb) q + +``args`` here shows the arguments to the current function in the stack. The Python `pdb documentation `__ has full details on the other available commands. + +.. _cli_convert_complex: + +Defining a convert() function +----------------------------- + +Instead of providing a single line of code to be executed against each value, you can define a function called ``convert(value)``. + +This mechanism can be used to execute one-off initialization code that runs once at the start of the conversion run. + +The following example adds a new ``score`` column, then updates it to list a random number - after first seeding the random number generator to ensure that multiple runs produce the same results: + +.. code-block:: bash + + sqlite-utils add-column content.db articles score float --not-null-default 1.0 + sqlite-utils convert content.db articles score ' + import random + random.seed(10) + + def convert(value): + return random.random() + ' .. _cli_convert_recipes: @@ -1019,22 +1959,46 @@ Various built-in recipe functions are available for common operations. These are Would produce an array like this: ``[1.2, 3.0, 4.5]`` -``r.parsedate(value, dayfirst=False, yearfirst=False)`` +``r.parsedate(value, dayfirst=False, yearfirst=False, errors=None)`` Parse a date and convert it to ISO date format: ``yyyy-mm-dd`` In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted. -``r.parsedatetime(value, dayfirst=False, yearfirst=False)`` + Use the ``errors=`` parameter to specify what should happen if a value cannot be parsed. + + By default, if any value cannot be parsed an error will be occurred and all values will be left as they were. + + Set ``errors=r.IGNORE`` to ignore any values that cannot be parsed, leaving them unchanged. + + Set ``errors=r.SET_NULL`` to set any values that cannot be parsed to ``null``. + +``r.parsedatetime(value, dayfirst=False, yearfirst=False, errors=None)`` Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS`` -These recipes can be used in the code passed to ``sqlite-utils convert`` like this:: +These recipes can be used in the code passed to ``sqlite-utils convert`` like this: - $ sqlite-utils convert my.db mytable mycolumn \ +.. code-block:: bash + + sqlite-utils convert my.db mytable mycolumn \ 'r.jsonsplit(value)' -To use any of the documented parameters, do this:: +You can also pass the recipe function directly without the ``(value)`` part - sqlite-utils will detect that it is a callable and use it automatically: - $ sqlite-utils convert my.db mytable mycolumn \ +.. code-block:: bash + + sqlite-utils convert my.db mytable mycolumn r.parsedate + +This shorter syntax works for any callable, including functions from imported modules: + +.. code-block:: bash + + sqlite-utils convert my.db mytable mycolumn json.loads --import json + +To use any of the documented parameters, use the full function call syntax: + +.. code-block:: bash + + sqlite-utils convert my.db mytable mycolumn \ 'r.jsonsplit(value, delimiter=":")' .. _cli_convert_output: @@ -1042,14 +2006,18 @@ To use any of the documented parameters, do this:: Saving the result to a different column --------------------------------------- -The ``--output`` and ``--output-type`` options can be used to save the result of the conversion to a separate column, which will be created if that column does not already exist:: +The ``--output`` and ``--output-type`` options can be used to save the result of the conversion to a separate column, which will be created if that column does not already exist: - $ sqlite-utils convert content.db articles headline 'value.upper()' \ +.. code-block:: bash + + sqlite-utils convert content.db articles headline 'value.upper()' \ --output headline_upper -The type of the created column defaults to ``text``, but a different column type can be specified using ``--output-type``. This example will create a new floating point column called ``id_as_a_float`` with a copy of each item's ID increased by 0.5:: +The type of the created column defaults to ``text``, but a different column type can be specified using ``--output-type``. This example will create a new floating point column called ``id_as_a_float`` with a copy of each item's ID increased by 0.5: - $ sqlite-utils convert content.db articles id 'float(value) + 0.5' \ +.. code-block:: bash + + sqlite-utils convert content.db articles id 'float(value) + 0.5' \ --output id_as_a_float \ --output-type float @@ -1064,9 +2032,11 @@ Sometimes you may wish to convert a single column into multiple derived columns. You can achieve this using the ``--multi`` option to ``sqlite-utils convert``. This option expects your Python code to return a Python dictionary: new columns well be created and populated for each of the keys in that dictionary. -For the ``latitude,longitude`` example you would use the following:: +For the ``latitude,longitude`` example you would use the following: - $ sqlite-utils convert demo.db places location \ +.. code-block:: bash + + sqlite-utils convert demo.db places location \ 'bits = value.split(",") return { "latitude": float(bits[0]), @@ -1077,10 +2047,10 @@ The type of the returned values will be taken into account when creating the new .. code-block:: sql - CREATE TABLE [places] ( - [location] TEXT, - [latitude] FLOAT, - [longitude] FLOAT + CREATE TABLE "places" ( + "location" TEXT, + "latitude" REAL, + "longitude" REAL ); The code function can also return ``None``, in which case its output will be ignored. You can drop the original column at the end of the operation by adding ``--drop``. @@ -1090,19 +2060,23 @@ The code function can also return ``None``, in which case its output will be ign Creating tables =============== -Most of the time creating tables by inserting example data is the quickest approach. If you need to create an empty table in advance of inserting data you can do so using the ``create-table`` command:: +Most of the time creating tables by inserting example data is the quickest approach. If you need to create an empty table in advance of inserting data you can do so using the ``create-table`` command: - $ sqlite-utils create-table mydb.db mytable id integer name text --pk=id +.. code-block:: bash + + sqlite-utils create-table mydb.db mytable id integer name text --pk=id This will create a table called ``mytable`` with two columns - an integer ``id`` column and a text ``name`` column. It will set the ``id`` column to be the primary key. You can pass as many column-name column-type pairs as you like. Valid types are ``integer``, ``text``, ``float`` and ``blob``. +Pass ``--pk`` more than once for a compound primary key that covers multiple columns. + You can specify columns that should be NOT NULL using ``--not-null colname``. You can specify default values for columns using ``--default colname defaultvalue``. -:: +.. code-block:: bash - $ sqlite-utils create-table mydb.db mytable \ + sqlite-utils create-table mydb.db mytable \ id integer \ name text \ age integer \ @@ -1112,66 +2086,137 @@ You can specify columns that should be NOT NULL using ``--not-null colname``. Yo --default is_good 1 \ --pk=id - $ sqlite-utils tables mydb.db --schema -t +.. code-block:: bash + + sqlite-utils tables mydb.db --schema -t + +.. code-block:: output + table schema ------- -------------------------------- - mytable CREATE TABLE [mytable] ( - [id] INTEGER PRIMARY KEY, - [name] TEXT NOT NULL, - [age] INTEGER NOT NULL, - [is_good] INTEGER DEFAULT '1' + mytable CREATE TABLE "mytable" ( + "id" INTEGER PRIMARY KEY, + "name" TEXT NOT NULL, + "age" INTEGER NOT NULL, + "is_good" INTEGER DEFAULT '1' ) -You can specify foreign key relationships between the tables you are creating using ``--fk colname othertable othercolumn``:: +You can specify foreign key relationships between the tables you are creating using ``--fk colname othertable othercolumn``: - $ sqlite-utils create-table books.db authors \ +.. code-block:: bash + + sqlite-utils create-table books.db authors \ id integer \ name text \ --pk=id - $ sqlite-utils create-table books.db books \ + sqlite-utils create-table books.db books \ id integer \ title text \ author_id integer \ --pk=id \ --fk author_id authors id - $ sqlite-utils tables books.db --schema -t +.. code-block:: bash + + sqlite-utils tables books.db --schema -t + +.. code-block:: output + table schema ------- ------------------------------------------------- - authors CREATE TABLE [authors] ( - [id] INTEGER PRIMARY KEY, - [name] TEXT + authors CREATE TABLE "authors" ( + "id" INTEGER PRIMARY KEY, + "name" TEXT ) - books CREATE TABLE [books] ( - [id] INTEGER PRIMARY KEY, - [title] TEXT, - [author_id] INTEGER REFERENCES [authors]([id]) + books CREATE TABLE "books" ( + "id" INTEGER PRIMARY KEY, + "title" TEXT, + "author_id" INTEGER REFERENCES "authors"("id") ) +You can create a table in `SQLite STRICT mode `__ using ``--strict``: + +.. code-block:: bash + + sqlite-utils create-table mydb.db mytable id integer name text --strict + +.. code-block:: bash + + sqlite-utils tables mydb.db --schema -t + +.. code-block:: output + + table schema + ------- ------------------------ + mytable CREATE TABLE "mytable" ( + "id" INTEGER, + "name" TEXT + ) STRICT + If a table with the same name already exists, you will get an error. You can choose to silently ignore this error with ``--ignore``, or you can replace the existing table with a new, empty table using ``--replace``. +You can also pass ``--transform`` to transform the existing table to match the new schema. See :ref:`python_api_explicit_create` in the Python library documentation for details of how this option works. + +.. note:: + In Python: :ref:`table.create() ` CLI reference: :ref:`sqlite-utils create-table ` + +.. _cli_renaming_tables: + +Renaming a table +================ + +Yo ucan rename a table using the ``rename-table`` command: + +.. code-block:: bash + + sqlite-utils rename-table mydb.db oldname newname + +Pass ``--ignore`` to ignore any errors caused by the table not existing, or the new name already being in use. + +.. note:: + In Python: :ref:`db.rename_table() ` CLI reference: :ref:`sqlite-utils rename-table ` + +.. _cli_duplicate_table: + +Duplicating tables +================== + +The ``duplicate`` command duplicates a table - creating a new table with the same schema and a copy of all of the rows: + +.. code-block:: bash + + sqlite-utils duplicate books.db authors authors_copy + +.. note:: + In Python: :ref:`table.duplicate() ` CLI reference: :ref:`sqlite-utils duplicate ` + .. _cli_drop_table: Dropping tables =============== -You can drop a table using the ``drop-table`` command:: +You can drop a table using the ``drop-table`` command: - $ sqlite-utils drop-table mydb.db mytable +.. code-block:: bash + + sqlite-utils drop-table mydb.db mytable Use ``--ignore`` to ignore the error if the table does not exist. +.. note:: + In Python: :ref:`table.drop() ` CLI reference: :ref:`sqlite-utils drop-table ` + .. _cli_transform_table: Transforming tables =================== -The ``transform`` command allows you to apply complex transformations to a table that cannot be implemented using a regular SQLite ``ALTER TABLE`` command. See :ref:`python_api_transform` for details of how this works. +The ``transform`` command allows you to apply complex transformations to a table that cannot be implemented using a regular SQLite ``ALTER TABLE`` command. See :ref:`python_api_transform` for details of how this works. By default, the ``transform`` command preserves a table's ``STRICT`` mode. -:: +.. code-block:: bash - $ sqlite-utils transform mydb.db mytable \ + sqlite-utils transform mydb.db mytable \ --drop column1 \ --rename column2 column_renamed @@ -1210,9 +2255,20 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi ``--drop-foreign-key column`` Drop the specified foreign key. -If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example:: +``--add-foreign-key column other_table other_column`` + Add a foreign key constraint to ``column`` pointing to ``other_table.other_column``. - $ sqlite-utils transform fixtures.db roadside_attractions \ +``--strict`` + Convert the table to a `SQLite STRICT table `__. The command fails if the available SQLite version does not support strict tables. If existing rows contain values that are incompatible with their declared column types the transformation fails and the original table is left unchanged. + +``--no-strict`` + Convert a strict table back to a regular non-strict table. + +If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example: + +.. code-block:: bash + + sqlite-utils transform fixtures.db roadside_attractions \ --rename pk id \ --default name Untitled \ --column-order id \ @@ -1220,16 +2276,99 @@ If you want to see the SQL that will be executed to make the change without actu --column-order latitude \ --drop address \ --sql - CREATE TABLE [roadside_attractions_new_4033a60276b9] ( - [id] INTEGER PRIMARY KEY, - [longitude] FLOAT, - [latitude] FLOAT, - [name] TEXT DEFAULT 'Untitled' + +.. code-block:: output + + CREATE TABLE "roadside_attractions_new_4033a60276b9" ( + "id" INTEGER PRIMARY KEY, + "longitude" FLOAT, + "latitude" FLOAT, + "name" TEXT DEFAULT 'Untitled' ); - INSERT INTO [roadside_attractions_new_4033a60276b9] ([longitude], [latitude], [id], [name]) - SELECT [longitude], [latitude], [pk], [name] FROM [roadside_attractions]; - DROP TABLE [roadside_attractions]; - ALTER TABLE [roadside_attractions_new_4033a60276b9] RENAME TO [roadside_attractions]; + INSERT INTO "roadside_attractions_new_4033a60276b9" ("longitude", "latitude", "id", "name") + SELECT "longitude", "latitude", "pk", "name" FROM "roadside_attractions"; + DROP TABLE "roadside_attractions"; + ALTER TABLE "roadside_attractions_new_4033a60276b9" RENAME TO "roadside_attractions"; + +.. note:: + In Python: :ref:`table.transform() ` CLI reference: :ref:`sqlite-utils transform ` + +.. _cli_transform_table_add_primary_key_to_rowid: + +Adding a primary key to a rowid table +------------------------------------- + +SQLite tables that are created without an explicit primary key are created as `rowid tables `__. They still have a numeric primary key which is available in the ``rowid`` column, but that column is not included in the output of ``select *``. Here's an example: + +.. code-block:: bash + + echo '[{"name": "Azi"}, {"name": "Suna"}]' | \ + sqlite-utils insert chickens.db chickens - + sqlite-utils schema chickens.db + +.. code-block:: output + + CREATE TABLE "chickens" ( + "name" TEXT + ); + +.. code-block:: bash + + sqlite-utils chickens.db 'select * from chickens' + +.. code-block:: output + + [{"name": "Azi"}, + {"name": "Suna"}] + +.. code-block:: bash + + sqlite-utils chickens.db 'select rowid, * from chickens' + +.. code-block:: output + + [{"rowid": 1, "name": "Azi"}, + {"rowid": 2, "name": "Suna"}] + +You can use ``sqlite-utils transform ... --pk id`` to add a primary key column called ``id`` to the table. The primary key will be created as an ``INTEGER PRIMARY KEY`` and the existing ``rowid`` values will be copied across to it. It will automatically increment as new rows are added to the table: + +.. code-block:: bash + + sqlite-utils transform chickens.db chickens --pk id + +.. code-block:: bash + + sqlite-utils schema chickens.db + +.. code-block:: output + + CREATE TABLE "chickens" ( + "id" INTEGER PRIMARY KEY, + "name" TEXT + ); + +.. code-block:: bash + + sqlite-utils chickens.db 'select * from chickens' + +.. code-block:: output + + [{"id": 1, "name": "Azi"}, + {"id": 2, "name": "Suna"}] + +.. code-block:: bash + + echo '{"name": "Cardi"}' | sqlite-utils insert chickens.db chickens - + +.. code-block:: bash + + sqlite-utils chickens.db 'select * from chickens' + +.. code-block:: output + + [{"id": 1, "name": "Azi"}, + {"id": 2, "name": "Suna"}, + {"id": 3, "name": "Cardi"}] .. _cli_extract: @@ -1240,25 +2379,30 @@ The ``sqlite-utils extract`` command can be used to extract specified columns in Take a look at the Python API documentation for :ref:`python_api_extract` for a detailed description of how this works, including examples of table schemas before and after running an extraction operation. -The command takes a database, table and one or more columns that should be extracted. To extract the ``species`` column from the ``trees`` table you would run:: +Rows where every extracted column is ``null`` are not extracted - those rows get a ``null`` value in their new foreign key column and no record is created for them in the lookup table. - $ sqlite-utils extract my.db trees species +The command takes a database, table and one or more columns that should be extracted. To extract the ``species`` column from the ``trees`` table you would run: + +.. code-block:: bash + + sqlite-utils extract my.db trees species This would produce the following schema: .. code-block:: sql CREATE TABLE "trees" ( - [id] INTEGER PRIMARY KEY, - [TreeAddress] TEXT, - [species_id] INTEGER, + "id" INTEGER PRIMARY KEY, + "TreeAddress" TEXT, + "species_id" INTEGER, FOREIGN KEY(species_id) REFERENCES species(id) - ) - - CREATE TABLE [species] ( - [id] INTEGER PRIMARY KEY, - [species] TEXT - ) + ); + CREATE TABLE "species" ( + "id" INTEGER PRIMARY KEY, + "species" TEXT + ); + CREATE UNIQUE INDEX "idx_species_species" + ON "species" ("species"); The command takes the following options: @@ -1274,7 +2418,9 @@ The command takes the following options: ``--silent`` Don't display the progress bar. -Here's a more complex example that makes use of these options. It converts `this CSV file `__ full of global power plants into SQLite, then extracts the ``country`` and ``country_long`` columns into a separate ``countries`` table:: +Here's a more complex example that makes use of these options. It converts `this CSV file `__ full of global power plants into SQLite, then extracts the ``country`` and ``country_long`` columns into a separate ``countries`` table: + +.. code-block:: bash wget 'https://github.com/wri/global-power-plant-database/blob/232a6666/output_database/global_power_plant_database.csv?raw=true' sqlite-utils insert global.db power_plants \ @@ -1289,99 +2435,140 @@ After running the above, the command ``sqlite-utils schema global.db`` reveals t .. code-block:: sql - CREATE TABLE [countries] ( - [id] INTEGER PRIMARY KEY, - [country] TEXT, - [name] TEXT + CREATE TABLE "countries" ( + "id" INTEGER PRIMARY KEY, + "country" TEXT, + "name" TEXT ); CREATE TABLE "power_plants" ( - [country_id] INTEGER, - [name] TEXT, - [gppd_idnr] TEXT, - [capacity_mw] TEXT, - [latitude] TEXT, - [longitude] TEXT, - [primary_fuel] TEXT, - [other_fuel1] TEXT, - [other_fuel2] TEXT, - [other_fuel3] TEXT, - [commissioning_year] TEXT, - [owner] TEXT, - [source] TEXT, - [url] TEXT, - [geolocation_source] TEXT, - [wepp_id] TEXT, - [year_of_capacity_data] TEXT, - [generation_gwh_2013] TEXT, - [generation_gwh_2014] TEXT, - [generation_gwh_2015] TEXT, - [generation_gwh_2016] TEXT, - [generation_gwh_2017] TEXT, - [generation_data_source] TEXT, - [estimated_generation_gwh] TEXT, - FOREIGN KEY([country_id]) REFERENCES [countries]([id]) + "country_id" INTEGER, + "name" TEXT, + "gppd_idnr" TEXT, + "capacity_mw" TEXT, + "latitude" TEXT, + "longitude" TEXT, + "primary_fuel" TEXT, + "other_fuel1" TEXT, + "other_fuel2" TEXT, + "other_fuel3" TEXT, + "commissioning_year" TEXT, + "owner" TEXT, + "source" TEXT, + "url" TEXT, + "geolocation_source" TEXT, + "wepp_id" TEXT, + "year_of_capacity_data" TEXT, + "generation_gwh_2013" TEXT, + "generation_gwh_2014" TEXT, + "generation_gwh_2015" TEXT, + "generation_gwh_2016" TEXT, + "generation_gwh_2017" TEXT, + "generation_data_source" TEXT, + "estimated_generation_gwh" TEXT, + FOREIGN KEY("country_id") REFERENCES "countries"("id") ); - CREATE UNIQUE INDEX [idx_countries_country_name] - ON [countries] ([country], [name]); + CREATE UNIQUE INDEX "idx_countries_country_name" + ON "countries" ("country", "name"); + +.. note:: + In Python: :ref:`table.extract() ` CLI reference: :ref:`sqlite-utils extract ` .. _cli_create_view: Creating views ============== -You can create a view using the ``create-view`` command:: +You can create a view using the ``create-view`` command: - $ sqlite-utils create-view mydb.db version "select sqlite_version()" +.. code-block:: bash + + sqlite-utils create-view mydb.db version "select sqlite_version()" + +.. code-block:: bash + + sqlite-utils mydb.db "select * from version" + +.. code-block:: output - $ sqlite-utils mydb.db "select * from version" [{"sqlite_version()": "3.31.1"}] Use ``--replace`` to replace an existing view of the same name, and ``--ignore`` to do nothing if a view already exists. +.. note:: + In Python: :ref:`db.create_view() ` CLI reference: :ref:`sqlite-utils create-view ` + .. _cli_drop_view: Dropping views ============== -You can drop a view using the ``drop-view`` command:: +You can drop a view using the ``drop-view`` command: - $ sqlite-utils drop-view myview +.. code-block:: bash + + sqlite-utils drop-view myview Use ``--ignore`` to ignore the error if the view does not exist. +.. note:: + In Python: :ref:`view.drop() ` CLI reference: :ref:`sqlite-utils drop-view ` + .. _cli_add_column: Adding columns ============== -You can add a column using the ``add-column`` command:: +You can add a column using the ``add-column`` command: - $ sqlite-utils add-column mydb.db mytable nameofcolumn text +.. code-block:: bash -The last argument here is the type of the column to be created. You can use one of ``text``, ``integer``, ``float`` or ``blob``. If you leave it off, ``text`` will be used. + sqlite-utils add-column mydb.db mytable nameofcolumn text -You can add a column that is a foreign key reference to another table using the ``--fk`` option:: +The last argument here is the type of the column to be created. This can be one of: - $ sqlite-utils add-column mydb.db dogs species_id --fk species +- ``text`` or ``str`` +- ``integer`` or ``int`` +- ``float`` +- ``blob`` or ``bytes`` + +This argument is optional and defaults to ``text``. + +You can add a column that is a foreign key reference to another table using the ``--fk`` option: + +.. code-block:: bash + + sqlite-utils add-column mydb.db dogs species_id --fk species This will automatically detect the name of the primary key on the species table and use that (and its type) for the new column. -You can explicitly specify the column you wish to reference using ``--fk-col``:: +You can explicitly specify the column you wish to reference using ``--fk-col``: - $ sqlite-utils add-column mydb.db dogs species_id --fk species --fk-col ref +.. code-block:: bash -You can set a ``NOT NULL DEFAULT 'x'`` constraint on the new column using ``--not-null-default``:: + sqlite-utils add-column mydb.db dogs species_id --fk species --fk-col ref - $ sqlite-utils add-column mydb.db dogs friends_count integer --not-null-default 0 +You can set a ``NOT NULL DEFAULT 'x'`` constraint on the new column using ``--not-null-default``: + +.. code-block:: bash + + sqlite-utils add-column mydb.db dogs friends_count integer --not-null-default 0 + +.. note:: + In Python: :ref:`table.add_column() ` CLI reference: :ref:`sqlite-utils add-column ` .. _cli_add_column_alter: Adding columns automatically on insert/update ============================================= -You can use the ``--alter`` option to automatically add new columns if the data you are inserting or upserting is of a different shape:: +You can use the ``--alter`` option to automatically add new columns if the data you are inserting or upserting is of a different shape: - $ sqlite-utils insert dogs.db dogs new-dogs.json --pk=id --alter +.. code-block:: bash + + sqlite-utils insert dogs.db dogs new-dogs.json --pk=id --alter + +.. note:: + In Python: :ref:`table.insert(..., alter=True) ` .. _cli_add_foreign_key: @@ -1390,63 +2577,89 @@ Adding foreign key constraints The ``add-foreign-key`` command can be used to add new foreign key references to an existing table - something which SQLite's ``ALTER TABLE`` command does not support. -To add a foreign key constraint pointing the ``books.author_id`` column to ``authors.id`` in another table, do this:: +To add a foreign key constraint pointing the ``books.author_id`` column to ``authors.id`` in another table, do this: - $ sqlite-utils add-foreign-key books.db books author_id authors id +.. code-block:: bash -If you omit the other table and other column references ``sqlite-utils`` will attempt to guess them - so the above example could instead look like this:: + sqlite-utils add-foreign-key books.db books author_id authors id - $ sqlite-utils add-foreign-key books.db books author_id +If you omit the other table and other column references ``sqlite-utils`` will attempt to guess them - so the above example could instead look like this: -Add ``--ignore`` to ignore an existing foreign key (as opposed to returning an error):: +.. code-block:: bash - $ sqlite-utils add-foreign-key books.db books author_id --ignore + sqlite-utils add-foreign-key books.db books author_id + +Add ``--ignore`` to ignore an existing foreign key (as opposed to returning an error): + +.. code-block:: bash + + sqlite-utils add-foreign-key books.db books author_id --ignore See :ref:`python_api_add_foreign_key` in the Python API documentation for further details, including how the automatic table guessing mechanism works. +.. note:: + In Python: :ref:`table.add_foreign_key() ` CLI reference: :ref:`sqlite-utils add-foreign-key ` + .. _cli_add_foreign_keys: Adding multiple foreign keys at once ------------------------------------ -Adding a foreign key requires a ``VACUUM``. On large databases this can be an expensive operation, so if you are adding multiple foreign keys you can combine them into one operation (and hence one ``VACUUM``) using ``add-foreign-keys``:: +Adding a foreign key requires a ``VACUUM``. On large databases this can be an expensive operation, so if you are adding multiple foreign keys you can combine them into one operation (and hence one ``VACUUM``) using ``add-foreign-keys``: - $ sqlite-utils add-foreign-keys books.db \ +.. code-block:: bash + + sqlite-utils add-foreign-keys books.db \ books author_id authors id \ authors country_id countries id When you are using this command each foreign key needs to be defined in full, as four arguments - the table, column, other table and other column. +.. note:: + In Python: :ref:`db.add_foreign_keys() ` CLI reference: :ref:`sqlite-utils add-foreign-keys ` + .. _cli_index_foreign_keys: Adding indexes for all foreign keys ----------------------------------- -If you want to ensure that every foreign key column in your database has a corresponding index, you can do so like this:: +If you want to ensure that every foreign key column in your database has a corresponding index, you can do so like this: - $ sqlite-utils index-foreign-keys books.db +.. code-block:: bash + + sqlite-utils index-foreign-keys books.db + +.. note:: + In Python: :ref:`db.index_foreign_keys() ` CLI reference: :ref:`sqlite-utils index-foreign-keys ` .. _cli_defaults_not_null: Setting defaults and not null constraints ========================================= -You can use the ``--not-null`` and ``--default`` options (to both ``insert`` and ``upsert``) to specify columns that should be ``NOT NULL`` or to set database defaults for one or more specific columns:: +You can use the ``--not-null`` and ``--default`` options (to both ``insert`` and ``upsert``) to specify columns that should be ``NOT NULL`` or to set database defaults for one or more specific columns: - $ sqlite-utils insert dogs.db dogs_with_scores dogs-with-scores.json \ +.. code-block:: bash + + sqlite-utils insert dogs.db dogs_with_scores dogs-with-scores.json \ --not-null=age \ --not-null=name \ --default age 2 \ --default score 5 +.. note:: + In Python: :ref:`not_null= and defaults= arguments ` + .. _cli_create_index: Creating indexes ================ -You can add an index to an existing table using the ``create-index`` command:: +You can add an index to an existing table using the ``create-index`` command: - $ sqlite-utils create-index mydb.db mytable col1 [col2...] +.. code-block:: bash + + sqlite-utils create-index mydb.db mytable col1 [col2...] This can be used to create indexes against a single column or multiple columns. @@ -1456,93 +2669,151 @@ Use the ``--unique`` option to create a unique index. Use ``--if-not-exists`` to avoid attempting to create the index if one with that name already exists. -To add an index on a column in descending order, prefix the column with a hyphen. Since this can be confused for a command-line option you need to construct that like this:: +To add an index on a column in descending order, prefix the column with a hyphen. Since this can be confused for a command-line option you need to construct that like this: - $ sqlite-utils create-index mydb.db mytable -- col1 -col2 col3 +.. code-block:: bash + + sqlite-utils create-index mydb.db mytable -- col1 -col2 col3 This will create an index on that table on ``(col1, col2 desc, col3)``. If your column names are already prefixed with a hyphen you'll need to manually execute a ``CREATE INDEX`` SQL statement to add indexes to them rather than using this tool. +Add the ``--analyze`` option to run ``ANALYZE`` against the index after it has been created. + +.. note:: + In Python: :ref:`table.create_index() ` CLI reference: :ref:`sqlite-utils create-index ` + +.. _cli_drop_index: + +Dropping indexes +================ + +You can drop an index from an existing table using the ``drop-index`` command: + +.. code-block:: bash + + sqlite-utils drop-index mydb.db mytable idx_mytable_col1 + +Use ``--ignore`` to ignore the error if the index does not exist on that table. + +.. note:: + In Python: :ref:`table.drop_index() ` CLI reference: :ref:`sqlite-utils drop-index ` + .. _cli_fts: Configuring full-text search ============================ -You can enable SQLite full-text search on a table and a set of columns like this:: +You can enable SQLite full-text search on a table and a set of columns like this: - $ sqlite-utils enable-fts mydb.db documents title summary +.. code-block:: bash -This will use SQLite's FTS5 module by default. Use ``--fts4`` if you want to use FTS4:: + sqlite-utils enable-fts mydb.db documents title summary - $ sqlite-utils enable-fts mydb.db documents title summary --fts4 +This will use SQLite's FTS5 module by default. Use ``--fts4`` if you want to use FTS4: -The ``enable-fts`` command will populate the new index with all existing documents. If you later add more documents you will need to use ``populate-fts`` to cause them to be indexed as well:: +.. code-block:: bash - $ sqlite-utils populate-fts mydb.db documents title summary + sqlite-utils enable-fts mydb.db documents title summary --fts4 -A better solution here is to use database triggers. You can set up database triggers to automatically update the full-text index using the ``--create-triggers`` option when you first run ``enable-fts``:: +The ``enable-fts`` command will populate the new index with all existing documents. If you later add more documents you will need to use ``populate-fts`` to cause them to be indexed as well: - $ sqlite-utils enable-fts mydb.db documents title summary --create-triggers +.. code-block:: bash -To set a custom FTS tokenizer, e.g. to enable Porter stemming, use ``--tokenize=``:: + sqlite-utils populate-fts mydb.db documents title summary - $ sqlite-utils populate-fts mydb.db documents title summary --tokenize=porter +A better solution here is to use database triggers. You can set up database triggers to automatically update the full-text index using the ``--create-triggers`` option when you first run ``enable-fts``: -To remove the FTS tables and triggers you created, use ``disable-fts``:: +.. code-block:: bash - $ sqlite-utils disable-fts mydb.db documents + sqlite-utils enable-fts mydb.db documents title summary --create-triggers -To rebuild one or more FTS tables (see :ref:`python_api_fts_rebuild`), use ``rebuild-fts``:: +To set a custom FTS tokenizer, e.g. to enable Porter stemming, use ``--tokenize=``: - $ sqlite-utils rebuild-fts mydb.db documents +.. code-block:: bash -You can rebuild every FTS table by running ``rebuild-fts`` without passing any table names:: + sqlite-utils populate-fts mydb.db documents title summary --tokenize=porter - $ sqlite-utils rebuild-fts mydb.db +To remove the FTS tables and triggers you created, use ``disable-fts``: + +.. code-block:: bash + + sqlite-utils disable-fts mydb.db documents + +To rebuild one or more FTS tables (see :ref:`python_api_fts_rebuild`), use ``rebuild-fts``: + +.. code-block:: bash + + sqlite-utils rebuild-fts mydb.db documents + +You can rebuild every FTS table by running ``rebuild-fts`` without passing any table names: + +.. code-block:: bash + + sqlite-utils rebuild-fts mydb.db + +.. note:: + In Python: :ref:`table.enable_fts() ` CLI reference: :ref:`sqlite-utils enable-fts ` .. _cli_search: Executing searches ================== -Once you have configured full-text search for a table, you can search it using ``sqlite-utils search``:: +Once you have configured full-text search for a table, you can search it using ``sqlite-utils search``: - $ sqlite-utils search mydb.db documents searchterm +.. code-block:: bash + + sqlite-utils search mydb.db documents searchterm This command accepts the same output options as ``sqlite-utils query``: ``--table``, ``--csv``, ``--tsv``, ``--nl`` etc. -By default it shows the most relevant matches first. You can specify a different sort order using the ``-o`` option, which can take a column or a column followed by ``desc``:: +By default it shows the most relevant matches first. You can specify a different sort order using the ``-o`` option, which can take a column or a column followed by ``desc``: + +.. code-block:: bash # Sort by rowid - $ sqlite-utils search mydb.db documents searchterm -o rowid + sqlite-utils search mydb.db documents searchterm -o rowid # Sort by created in descending order - $ sqlite-utils search mydb.db documents searchterm -o 'created desc' + sqlite-utils search mydb.db documents searchterm -o 'created desc' -You can specify a subset of columns to be returned using the ``-c`` option one or more times:: +SQLite `advanced search syntax `__ is enabled by default. To run a search with automatic quoting applied to the terms to avoid them being potentially interpreted as advanced search syntax use the ``--quote`` option. - $ sqlite-utils search mydb.db documents searchterm -c title -c created +You can specify a subset of columns to be returned using the ``-c`` option one or more times: + +.. code-block:: bash + + sqlite-utils search mydb.db documents searchterm -c title -c created By default all search results will be returned. You can use ``--limit 20`` to return just the first 20 results. -Use the ``--sql`` option to output the SQL that would be executed, rather than running the query:: +Use the ``--sql`` option to output the SQL that would be executed, rather than running the query: + +.. code-block:: bash + + sqlite-utils search mydb.db documents searchterm --sql + +.. code-block:: output - $ sqlite-utils search mydb.db documents searchterm --sql with original as ( select rowid, * - from [documents] + from "documents" ) select - [original].* + "original".* from - [original] - join [documents_fts] on [original].rowid = [documents_fts].rowid + "original" + join "documents_fts" on "original".rowid = "documents_fts".rowid where - [documents_fts] match :query + "documents_fts" match :query order by - [documents_fts].rank + "documents_fts".rank + +.. note:: + In Python: :ref:`table.search() ` CLI reference: :ref:`sqlite-utils search ` .. _cli_enable_counts: @@ -1553,26 +2824,62 @@ Enabling cached counts The ``sqlite-utils enable-counts`` command can be used to configure these triggers, either for every table in the database or for specific tables. -:: +.. code-block:: bash # Configure triggers for every table in the database - $ sqlite-utils enable-counts mydb.db + sqlite-utils enable-counts mydb.db # Configure triggers just for specific tables - $ sqlite-utils enable-counts mydb.db table1 table2 + sqlite-utils enable-counts mydb.db table1 table2 -If the ``_counts`` table ever becomes out-of-sync with the actual table counts you can repair it using the ``reset-counts`` command:: +If the ``_counts`` table ever becomes out-of-sync with the actual table counts you can repair it using the ``reset-counts`` command: - $ sqlite-utils reset-counts mydb.db +.. code-block:: bash + + sqlite-utils reset-counts mydb.db + +.. note:: + In Python: :ref:`table.enable_counts() ` CLI reference: :ref:`sqlite-utils enable-counts ` + +.. _cli_analyze: + +Optimizing index usage with ANALYZE +=================================== + +The `SQLite ANALYZE command `__ builds a table of statistics which the query planner can use to make better decisions about which indexes to use for a given query. + +You should run ``ANALYZE`` if your database is large and you do not think your indexes are being efficiently used. + +To run ``ANALYZE`` against every index in a database, use this: + +.. code-block:: bash + + sqlite-utils analyze mydb.db + +You can run it against specific tables, or against specific named indexes, by passing them as optional arguments: + +.. code-block:: bash + + sqlite-utils analyze mydb.db mytable idx_mytable_name + +You can also run ``ANALYZE`` as part of another command using the ``--analyze`` option. This is supported by the ``create-index``, ``insert`` and ``upsert`` commands. + +.. note:: + In Python: :ref:`db.analyze() ` CLI reference: :ref:`sqlite-utils analyze ` .. _cli_vacuum: Vacuum ====== -You can run VACUUM to optimize your database like so:: +You can run VACUUM to optimize your database like so: - $ sqlite-utils vacuum mydb.db +.. code-block:: bash + + sqlite-utils vacuum mydb.db + +.. note:: + In Python: :ref:`db.vacuum() ` CLI reference: :ref:`sqlite-utils vacuum ` .. _cli_optimize: @@ -1583,48 +2890,63 @@ The optimize command can dramatically reduce the size of your database if you ar If you just want to run OPTIMIZE without the VACUUM, use the ``--no-vacuum`` flag. -:: +.. code-block:: bash # Optimize all FTS tables and then VACUUM - $ sqlite-utils optimize mydb.db + sqlite-utils optimize mydb.db # Optimize but skip the VACUUM - $ sqlite-utils optimize --no-vacuum mydb.db + sqlite-utils optimize --no-vacuum mydb.db To optimize specific tables rather than every FTS table, pass those tables as extra arguments: -:: +.. code-block:: bash - $ sqlite-utils optimize mydb.db table_1 table_2 + sqlite-utils optimize mydb.db table_1 table_2 + +.. note:: + In Python: :ref:`table.optimize() ` CLI reference: :ref:`sqlite-utils optimize ` .. _cli_wal: WAL mode ======== -You can enable `Write-Ahead Logging `__ for a database file using the ``enable-wal`` command:: +You can enable `Write-Ahead Logging `__ for a database file using the ``enable-wal`` command: - $ sqlite-utils enable-wal mydb.db +.. code-block:: bash -You can disable WAL mode using ``disable-wal``:: + sqlite-utils enable-wal mydb.db - $ sqlite-utils disable-wal mydb.db +You can disable WAL mode using ``disable-wal``: + +.. code-block:: bash + + sqlite-utils disable-wal mydb.db Both of these commands accept one or more database files as arguments. +.. note:: + In Python: :ref:`db.enable_wal() and db.disable_wal() ` CLI reference: :ref:`sqlite-utils enable-wal ` + .. _cli_dump: Dumping the database to SQL =========================== -The ``dump`` command outputs a SQL dump of the schema and full contents of the specified database file:: +The ``dump`` command outputs a SQL dump of the schema and full contents of the specified database file: - $ sqlite-utils dump mydb.db +.. code-block:: bash + + sqlite-utils dump mydb.db BEGIN TRANSACTION; CREATE TABLE ... ... COMMIT; +.. note:: + In Python: :ref:`db.iterdump() ` CLI reference: :ref:`sqlite-utils dump ` + .. _cli_load_extension: Loading SQLite extensions @@ -1634,7 +2956,87 @@ Many of these commands have the ability to load additional SQLite extensions usi This option can be applied multiple times to load multiple extensions. -Since `SpatiaLite `__ is commonly used with SQLite, the value ``spatialite`` is special: it will search for SpatiaLite in the most common installation locations, saving you from needing to remember exactly where that module is located:: +Since `SpatiaLite `__ is commonly used with SQLite, the value ``spatialite`` is special: it will search for SpatiaLite in the most common installation locations, saving you from needing to remember exactly where that module is located: + +.. code-block:: bash + + sqlite-utils memory "select spatialite_version()" --load-extension=spatialite + +.. code-block:: output - $ sqlite-utils memory "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] + +.. _cli_spatialite: + +SpatiaLite helpers +================== + +`SpatiaLite `_ adds geographic capability to SQLite (similar to how PostGIS builds on PostgreSQL). The `SpatiaLite cookbook `__ is a good resource for learning what's possible with it. + +You can convert an existing table to a geographic table by adding a geometry column, use the ``sqlite-utils add-geometry-column`` command: + +.. code-block:: bash + + sqlite-utils add-geometry-column spatial.db locations geometry --type POLYGON --srid 4326 + +The table (``locations`` in the example above) must already exist before adding a geometry column. Use ``sqlite-utils create-table`` first, then ``add-geometry-column``. + +Use the ``--type`` option to specify a geometry type. By default, ``add-geometry-column`` uses a generic ``GEOMETRY``, which will work with any type, though it may not be supported by some desktop GIS applications. + +Eight (case-insensitive) types are allowed: + +* POINT +* LINESTRING +* POLYGON +* MULTIPOINT +* MULTILINESTRING +* MULTIPOLYGON +* GEOMETRYCOLLECTION +* GEOMETRY + +.. note:: + In Python: :ref:`table.add_geometry_column() ` CLI reference: :ref:`sqlite-utils add-geometry-column ` + +.. _cli_spatialite_indexes: + +Adding spatial indexes +---------------------- + +Once you have a geometry column, you can speed up bounding box queries by adding a spatial index: + +.. code-block:: bash + + sqlite-utils create-spatial-index spatial.db locations geometry + +See this `SpatiaLite Cookbook recipe `__ for examples of how to use a spatial index. + +.. note:: + In Python: :ref:`table.create_spatial_index() ` CLI reference: :ref:`sqlite-utils create-spatial-index ` + +.. _cli_install: + +Installing packages +=================== + +The :ref:`convert command ` and the :ref:`insert -\\-convert ` and :ref:`query -\\-functions ` options can be provided with a Python script that imports additional modules from the ``sqlite-utils`` environment. + +You can install packages from PyPI directly into the correct environment using ``sqlite-utils install ``. This is a wrapper around ``pip install``. + +.. code-block:: bash + + sqlite-utils install beautifulsoup4 + +Use ``-U`` to upgrade an existing package. + +.. _cli_uninstall: + +Uninstalling packages +===================== + +You can uninstall packages that were installed using ``sqlite-utils install`` with ``sqlite-utils uninstall ``: + +.. code-block:: bash + + sqlite-utils uninstall beautifulsoup4 + +Use ``-y`` to skip the request for confirmation. diff --git a/docs/conf.py b/docs/conf.py index b4c7f44..4f29b39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from subprocess import Popen, PIPE +import inspect +from pathlib import Path +from subprocess import Popen, PIPE, check_output +import sys # This file is execfile()d with the current directory set to its # containing dir. @@ -30,12 +33,69 @@ from subprocess import Popen, PIPE # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks"] +extensions = [ + "sphinx.ext.extlinks", + "sphinx.ext.autodoc", + "sphinx_copybutton", + "sphinx.ext.linkcode", +] +autodoc_member_order = "bysource" +autodoc_typehints = "description" extlinks = { - "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), + "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#%s"), } + +def _linkcode_git_ref(): + try: + return check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip() + except Exception: + return "main" + + +def linkcode_resolve(domain, info): + if domain != "py": + return None + + module_name = info.get("module") + if not module_name or module_name.split(".")[0] != "sqlite_utils": + return None + + module = sys.modules.get(module_name) + if module is None: + return None + + obj = module + for part in info.get("fullname", "").split("."): + obj = getattr(obj, part, None) + if obj is None: + return None + + if isinstance(obj, property): + obj = obj.fget + + try: + obj = inspect.unwrap(obj) + source_file = inspect.getsourcefile(obj) + _, line_number = inspect.getsourcelines(obj) + except Exception: + return None + + if source_file is None: + return None + + try: + filename = Path(source_file).resolve().relative_to(Path(__file__).parent.parent) + except ValueError: + return None + + return ( + "https://github.com/simonw/sqlite-utils/blob/" + f"{_linkcode_git_ref()}/{filename}#L{line_number}" + ) + + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -50,7 +110,7 @@ master_doc = "index" # General information about the project. project = "sqlite-utils" -copyright = "2018-2021, Simon Willison" +copyright = "2018-2022, Simon Willison" author = "Simon Willison" # The version info for the project you're documenting, acts as replacement for @@ -59,7 +119,7 @@ author = "Simon Willison" # # The short X.Y version. pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True) -git_version = pipe.stdout.read().decode("utf8") +git_version = pipe.stdout.read().decode("utf8") if pipe.stdout else "" if git_version: version = git_version.rsplit("-", 1)[0] @@ -73,7 +133,7 @@ else: # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -83,6 +143,9 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" +# Only syntax highlight of code-block is used: +highlight_language = "none" + # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -92,7 +155,8 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = "furo" +html_title = "sqlite-utils" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -105,18 +169,7 @@ html_theme = "sphinx_rtd_theme" # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - "**": [ - "relations.html", # needs 'show_related': True theme option to display - "searchbox.html", - ] -} - +html_js_files = ["js/custom.js"] # -- Options for HTMLHelp output ------------------------------------------ @@ -174,7 +227,7 @@ texinfo_documents = [ "sqlite-utils documentation", author, "sqlite-utils", - "Python utility functions for manipulating SQLite databases", + "Python library for manipulating SQLite databases", "Miscellaneous", ) ] diff --git a/docs/contributing.rst b/docs/contributing.rst index ec1a4a7..f10b02a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -4,64 +4,134 @@ Contributing ============== -To work on this library locally, first checkout the code. Then create a new virtual environment:: +Development of ``sqlite-utils`` takes place in the `sqlite-utils GitHub repository `__. + +All improvements to the software should start with an issue. Read `How I build a feature `__ for a detailed description of the recommended process for building bug fixes or enhancements. + +.. _contributing_checkout: + +Obtaining the code +================== + +To work on this library locally, first checkout the code:: git clone git@github.com:simonw/sqlite-utils cd sqlite-utils - python3 -mvenv venv - source venv/bin/activate -Or if you are using ``pipenv``:: +Use ``uv run`` to run the development version of the tool:: - pipenv shell - -Within the virtual environment running ``sqlite-utils`` should run your locally editable version of the tool. You can use ``which sqlite-utils`` to confirm that you are running the version that lives in your virtual environment. + uv run sqlite-utils --help .. _contributing_tests: Running the tests ================= -To install the dependencies and test dependencies:: +Use ``uv run`` to run the tests:: - pip install -e '.[test]' - -To run the tests:: - - pytest + uv run pytest .. _contributing_docs: Building the documentation ========================== -To build the documentation, first install the documentation dependencies:: +To build the documentation run this command:: - pip install -e '.[docs]' + uv run make livehtml --directory docs -Then run ``make livehtml`` from the ``docs/`` directory to start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file:: +This will start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file. - cd docs - make livehtml +The `cog `__ tool is used to maintain portions of the documentation. You can run it like so:: + + uv run cog -r docs/*.rst .. _contributing_linting: Linting and formatting ====================== -``sqlite-utils`` uses `Black `__ for code formatting, and `flake8 `__ and `mypy `__ for linting and type checking. +``sqlite-utils`` uses `Black `__ for code formatting, and `flake8 `__ and `mypy `__ for linting and type checking:: -Black is installed as part of ``pip install -e '.[test]'`` - you can then format your code by running it in the root of the project:: + uv run black . - black . +Linting tools can be run like this:: -To install ``mypy`` and ``flake8`` run the following:: - - pip install -e '.[flake8,mypy]' - -Both commands can then be run in the root of the project like this:: - - flake8 - mypy sqlite_utils + uv run flake8 + uv run mypy sqlite_utils All three of these tools are run by our CI mechanism against every commit and pull request. + +.. _contributing_just: + +Using Just +========== + +If you install `Just `__ you can use it to manage your local development environment. + +To run all of the tests and linters:: + + just + +To run tests, or run a specific test module or test by name:: + + just test # All tests + just test tests/test_cli_memory.py # Just this module + just test -k test_memory_no_detect_types # Just this test + +To run just the linters:: + + just lint + +To apply Black to your code:: + + just black + +To update documentation using Cog:: + + just cog + +To run the live documentation server (this will run Cog first):: + + just docs + +And to list all available commands:: + + just -l + +.. _release_process: + +Release process +=============== + +Releases are performed using tags. When a new release is published on GitHub, a `GitHub Actions workflow `__ will perform the following: + +* Run the unit tests against all supported Python versions. If the tests pass... +* Build a wheel bundle of the underlying Python source code +* Push that new wheel up to PyPI: https://pypi.org/project/sqlite-utils/ + +To deploy new releases you will need to have push access to the GitHub repository. + +``sqlite-utils`` follows `Semantic Versioning `__:: + + major.minor.patch + +We increment ``major`` for backwards-incompatible releases. + +We increment ``minor`` for new features. + +We increment ``patch`` for bugfix releass. + +To release a new version, first create a commit that updates the version number in ``pyproject.toml`` and the :ref:`the changelog ` with highlights of the new version. An example `commit can be seen here `__:: + + # Update changelog + git commit -m " Release 3.29 + + Refs #423, #458, #467, #469, #470, #471, #472, #475" -a + git push + +Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here `__. + +You can generate the list of issue references for a specific release by copying and pasting text from the release notes or GitHub changes-since-last-release view into this `Extract issue numbers from pasted text `__ tool. + +To create the tag for the release, create `a new release `__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool `__. diff --git a/docs/index.rst b/docs/index.rst index 93b0bc0..f190d47 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ sqlite-utils |version| ======================= -|PyPI| |Changelog| |CI| |License| +|PyPI| |Changelog| |CI| |License| |discord| .. |PyPI| image:: https://img.shields.io/pypi/v/sqlite-utils.svg :target: https://pypi.org/project/sqlite-utils/ @@ -12,8 +12,10 @@ :target: https://github.com/simonw/sqlite-utils/actions .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE +.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=discord + :target: https://discord.gg/Ass7bCAMDw -*Python utility functions for manipulating SQLite databases* +*CLI tool and Python library for manipulating SQLite databases* This library and command-line utility helps create SQLite databases from an existing collection of data. @@ -23,6 +25,8 @@ sqlite-utils is not intended to be a full ORM: the focus is utility helpers to m It is designed as a useful complement to `Datasette `_. +`Cleaning data with sqlite-utils and Datasette `_ provides a tutorial introduction (and accompanying ten minute video) about using this tool. + Contents -------- @@ -32,7 +36,10 @@ Contents installation cli python-api + migrations + plugins + reference + cli-reference + upgrading contributing changelog - -Take a look at `this script `_ for an example of this library in action. diff --git a/docs/installation.rst b/docs/installation.rst index f4f132e..1333f5d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -38,3 +38,53 @@ Using pipx `pipx `__ is a tool for installing Python command-line applications in their own isolated environments. You can use ``pipx`` to install the ``sqlite-utils`` command-line tool like this:: pipx install sqlite-utils + +.. _installation_sqlite3_alternatives: + +Alternatives to sqlite3 +======================= + +By default, ``sqlite-utils`` uses the ``sqlite3`` package bundled with the Python standard library. + +Depending on your operating system, this may come with some limitations. + +On some platforms the ability to load additional extensions (via ``conn.load_extension(...)`` or ``--load-extension=/path/to/extension``) may be disabled. + +You may also see the error ``sqlite3.OperationalError: table sqlite_master may not be modified`` when trying to alter an existing table. + +You can work around these limitations by installing the `pysqlite3 `__ package, which provides a drop-in replacement for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions. + +To install ``pysqlite3`` run the following: + +.. code-block:: bash + + sqlite-utils install pysqlite3 + +``pysqlite3`` does not provide an implementation of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package: + +.. code-block:: bash + + sqlite-utils install sqlite-dump + +.. _installation_completion: + +Setting up shell completion +=========================== + +You can configure shell tab completion for the ``sqlite-utils`` command using these commands. + +For ``bash``: + +.. code-block:: bash + + eval "$(_SQLITE_UTILS_COMPLETE=bash_source sqlite-utils)" + +For ``zsh``: + +.. code-block:: zsh + + eval "$(_SQLITE_UTILS_COMPLETE=zsh_source sqlite-utils)" + +Add this code to ``~/.zshrc`` or ``~/.bashrc`` to automatically run it when you start a new shell. + +See `the Click documentation `__ for more details. diff --git a/docs/migrations.rst b/docs/migrations.rst new file mode 100644 index 0000000..23aa9d8 --- /dev/null +++ b/docs/migrations.rst @@ -0,0 +1,194 @@ +.. _migrations: + +===================== + Database migrations +===================== + +``sqlite-utils`` includes a migration system for applying repeatable changes to SQLite database files. + +A migration is a Python function that receives a :class:`sqlite_utils.Database` instance and then executes Python code to modify that database - creating or transforming tables, adding indexes, inserting rows, or any other operation supported by SQLite. + +Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database. + +This means you can run the migrate operation multiple times and it will only apply migrations that have not previously been recorded. + +.. _migrations_define: + +Defining migrations +=================== + +Ordered migration sets are defined by first creating a :class:`sqlite_utils.Migrations` object. + +Individual migrations are Python functions that are then registered with that migration set. Each migration function is passed a single argument that is a :ref:`sqlite_utils.Database ` instance. + +The name passed to ``Migrations("creatures")`` identifies that set of migrations. Use a name that is unique for your project, since multiple migration sets can be applied to the same database. + +Here is a simple example of a ``migrations.py`` file which creates a table, then adds an extra column to that table in a second migration: + +.. code-block:: python + + from sqlite_utils import Migrations + + migrations = Migrations("creatures") + + @migrations() + def create_table(db): + db["creatures"].create( + {"id": int, "name": str, "species": str}, + pk="id", + ) + + @migrations() + def add_weight(db): + db["creatures"].add_column("weight", float) + +.. _migrations_python: + +Applying migrations in Python +============================= + +Once you have a ``Migrations(name)`` collection with one or more migrations registered to it, you can execute them in Python code like this: + +.. code-block:: python + + from sqlite_utils import Database + + db = Database("creatures.db") + migrations.apply(db) + +Running ``migrations.apply(db)`` repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped. + +Migration functions are applied in the order that they were registered. The function name is used as the migration name unless you pass one explicitly: + +.. code-block:: python + + @migrations(name="001_create_table") + def create_table(db): + db["creatures"].create({"id": int, "name": str}, pk="id") + +When you apply a set of migrations you can stop part way through by specifying a ``stop_before=`` migration name: + +.. code-block:: python + + migrations.apply(db, stop_before="add_weight") + +.. _migrations_transactions: + +Migrations and transactions +=========================== + +Each migration runs inside a transaction, together with the ``_sqlite_migrations`` record of it having been applied. If a migration function raises an exception, everything it did is rolled back, no record is written and the migration stays pending - so fixing the error and re-applying will run that migration again from a clean state. Migrations that completed earlier in the same ``apply()`` run stay applied. + +Some operations cannot run inside a transaction, for example ``VACUUM`` or changing the journal mode with ``db.enable_wal()``. Register migrations like these with ``transactional=False``: + +.. code-block:: python + + @migrations(transactional=False) + def compact(db): + db.execute("VACUUM") + +A migration registered with ``transactional=False`` runs without a wrapping transaction, so if it fails part way through any changes it already made will not be rolled back, and re-applying will run the whole function again. + +Avoid calling ``db.commit()`` or otherwise managing transactions manually inside a transactional migration - register the migration with ``transactional=False`` if it needs to control its own transactions. Using ``with db.atomic():`` blocks inside a migration is fine: they nest as savepoints within the migration's transaction, so the migration as a whole still commits or rolls back as a single unit. See :ref:`python_api_transactions`. + +Applying migrations using the CLI +================================= + +Run migrations using the ``sqlite-utils migrate`` command: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py + +The first argument is the database file. The remaining arguments can be paths to migration files or directories containing migration files. + +If you omit migration paths, ``sqlite-utils`` searches the current directory and subdirectories for files called ``migrations.py``: + +.. code-block:: bash + + sqlite-utils migrate creatures.db + +You can also pass a directory. Every ``migrations.py`` file in that directory tree will be considered: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/project/ + +Running the command repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped. + +Listing migrations +================== + +Use ``--list`` to show applied and pending migrations without running them. This is a read-only operation - it will not create the database file or the ``_sqlite_migrations`` table: + +.. code-block:: bash + + sqlite-utils migrate creatures.db --list + +Example output: + +.. code-block:: output + + Migrations for: creatures + + Applied: + create_table - 2026-06-09 17:23:12.048092+00:00 + add_weight - 2026-06-09 17:23:12.051249+00:00 + + Pending: + add_age + +Stopping before a migration +=========================== + +When applying migrations using the CLI, you can stop before a named migration: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py --stop-before add_weight + +This applies any pending migrations before ``add_weight`` and leaves ``add_weight`` and later migrations pending. An unqualified migration name matches in any migration set. + +You can also target a specific migration set using ``migration_set:migration_name``. This is useful if a migrations file contains more than one migration set, or if multiple sets use the same migration name: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py \ + --stop-before creatures:add_weight \ + --stop-before sales:drop_index + +The ``--stop-before`` option can be passed more than once. + +If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. Naming a migration that has already been applied is also an error - stopping before it is impossible to honor - and no pending migrations are applied. + +Verbose output +============== + +Use ``--verbose`` or ``-v`` to show the schema before and after migrations are applied, plus a unified diff when the schema changes: + +.. code-block:: bash + + sqlite-utils migrate creatures.db --verbose + +Migrating from sqlite-migrate +============================= + +This system uses the same migration table format as the older `sqlite-migrate `__ package. To use existing migration files directly with ``sqlite-utils``, update their import from ``sqlite_migrate`` to ``sqlite_utils``: + +.. code-block:: python + + from sqlite_utils import Migrations + + migration = Migrations("creatures") + + @migration() + def create_table(db): + db["creatures"].create({"id": int, "name": str}, pk="id") + +Python API +========== + +.. autoclass:: sqlite_utils.migrations.Migrations + :members: + :undoc-members: + :exclude-members: _Migration, _AppliedMigration diff --git a/docs/plugins.rst b/docs/plugins.rst new file mode 100644 index 0000000..99588ee --- /dev/null +++ b/docs/plugins.rst @@ -0,0 +1,159 @@ +.. _plugins: + +========= + Plugins +========= + +``sqlite-utils`` supports plugins, which can be used to add extra features to the software. + +Plugins can add new commands, for example ``sqlite-utils some-command ...`` + +Plugins can be installed using the ``sqlite-utils install`` command: + +.. code-block:: bash + + sqlite-utils install sqlite-utils-name-of-plugin + +You can see a JSON list of plugins that have been installed by running this: + +.. code-block:: bash + + sqlite-utils plugins + +Plugin hooks such as :ref:`plugins_hooks_prepare_connection` affect each instance of the ``Database`` class. You can opt-out of these plugins by creating that class instance like so: + +.. code-block:: python + + db = Database(memory=True, execute_plugins=False) + +.. _plugins_building: + +Building a plugin +----------------- + +Plugins are created in a directory named after the plugin. To create a "hello world" plugin, first create a ``hello-world`` directory: + +.. code-block:: bash + + mkdir hello-world + cd hello-world + +In that folder create two files. The first is a ``pyproject.toml`` file describing the plugin: + +.. code-block:: toml + + [project] + name = "sqlite-utils-hello-world" + version = "0.1" + + [project.entry-points.sqlite_utils] + hello_world = "sqlite_utils_hello_world" + +The ``[project.entry-points.sqlite_utils]`` section tells ``sqlite-utils`` which module to load when executing the plugin. + +Then create ``sqlite_utils_hello_world.py`` with the following content: + +.. code-block:: python + + import click + import sqlite_utils + + @sqlite_utils.hookimpl + def register_commands(cli): + @cli.command() + def hello_world(): + "Say hello world" + click.echo("Hello world!") + +Install the plugin in "editable" mode - so you can make changes to the code and have them picked up instantly by ``sqlite-utils`` - like this: + +.. code-block:: bash + + sqlite-utils install -e . + +Or pass the path to your plugin directory: + +.. code-block:: bash + + sqlite-utils install -e /dev/sqlite-utils-hello-world + +Now, running this should execute your new command: + +.. code-block:: bash + + sqlite-utils hello-world + +Your command will also be listed in the output of ``sqlite-utils --help``. + +See the `LLM plugin documentation `__ for tips on distributing your plugin. + +.. _plugins_hooks: + +Plugin hooks +------------ + +Plugin hooks allow ``sqlite-utils`` to be customized. + +.. _plugins_hooks_register_commands: + +register_commands(cli) +~~~~~~~~~~~~~~~~~~~~~~ + +This hook can be used to register additional commands with the ``sqlite-utils`` CLI. It is called with the ``cli`` object, which is a ``click.Group`` instance. + +Example implementation: + +.. code-block:: python + + import click + import sqlite_utils + + @sqlite_utils.hookimpl + def register_commands(cli): + @cli.command() + def hello_world(): + "Say hello world" + click.echo("Hello world!") + +New commands implemented by plugins can invoke existing commands using the `context.invoke `__ mechanism. + +As a special niche feature, if your plugin needs to import some files and then act against an in-memory database containing those files you can forward to the :ref:`sqlite-utils memory command ` and pass it ``return_db=True``: + +.. code-block:: python + + @cli.command() + @click.pass_context + @click.argument( + "paths", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True), + required=False, + nargs=-1, + ) + def show_schema_for_files(ctx, paths): + from sqlite_utils.cli import memory + db = ctx.invoke(memory, paths=paths, return_db=True) + # Now do something with that database + click.echo(db.schema) + +.. _plugins_hooks_prepare_connection: + +prepare_connection(conn) +~~~~~~~~~~~~~~~~~~~~~~~~ + +This hook is called when a new SQLite database connection is created. You can use it to `register custom SQL functions `_, aggregates and collations. For example: + +.. code-block:: python + + import sqlite_utils + + @sqlite_utils.hookimpl + def prepare_connection(conn): + conn.create_function( + "hello", 1, lambda name: f"Hello, {name}!" + ) + +This registers a SQL function called ``hello`` which takes a single argument and can be called like this: + +.. code-block:: sql + + select hello("world"); -- "Hello, world!" diff --git a/docs/python-api.rst b/docs/python-api.rst index 70baf0c..43b734d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -5,6 +5,68 @@ ============================= .. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here + +.. _python_api_getting_started: + +Getting started +=============== + +Here's how to create a new SQLite database file containing a new ``chickens`` table, populated with four records: + +.. code-block:: python + + from sqlite_utils import Database + + db = Database("chickens.db") + db.table("chickens").insert_all([{ + "name": "Azi", + "color": "blue", + }, { + "name": "Lila", + "color": "blue", + }, { + "name": "Suna", + "color": "gold", + }, { + "name": "Cardi", + "color": "black", + }]) + +The inserted rows are saved to the database file straight away - methods like ``insert_all()`` commit their own changes, so no ``commit()`` call is needed. See :ref:`python_api_transactions` for how this works. + +You can loop through those rows like this: + +.. code-block:: python + + for row in db.table("chickens").rows: + print(row) + +Which outputs the following:: + + {'name': 'Azi', 'color': 'blue'} + {'name': 'Lila', 'color': 'blue'} + {'name': 'Suna', 'color': 'gold'} + {'name': 'Cardi', 'color': 'black'} + +To run a SQL query, use :ref:`db.query() `: + +.. code-block:: python + + for row in db.query(""" + select color, count(*) + from chickens group by color + order by count(*) desc + """): + print(row) + +Which outputs:: + + {'color': 'blue', 'count(*)': 2} + {'color': 'gold', 'count(*)': 1} + {'color': 'black', 'count(*)': 1} + +.. _python_api_connect: Connecting to or creating a database ==================================== @@ -33,18 +95,73 @@ Instead of a file path you can pass in an existing SQLite connection: db = Database(sqlite3.connect("my_database.db")) +The connection must use Python's default transaction handling. Connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options are rejected with a ``sqlite_utils.db.TransactionError`` - see :ref:`python_api_transactions_modes`. + If you want to create an in-memory database, you can do so like this: .. code-block:: python db = Database(memory=True) +You can also create a named in-memory database. Unlike regular memory databases these can be accessed by multiple threads, provided at least one reference to the database still exists. `del db` will clear the database from memory. + +.. code-block:: python + + db = Database(memory_name="my_shared_database") + +After creating a ``Database`` you can use ``db.memory`` and ``db.memory_name`` to tell whether it is backed by an in-memory database and to read the shared cache name. ``db.memory`` is ``True`` for any in-memory database and ``db.memory_name`` holds the name passed to ``memory_name=``, or ``None`` otherwise. + +.. code-block:: python + + db = Database(memory_name="shared") + db.memory # True + db.memory_name # "shared" + Connections use ``PRAGMA recursive_triggers=on`` by default. If you don't want to use `recursive triggers `__ you can turn them off using: .. code-block:: python db = Database(memory=True, recursive_triggers=False) +By default, any :ref:`sqlite-utils plugins ` that implement the :ref:`plugins_hooks_prepare_connection` hook will be executed against the connection when you create the ``Database`` object. You can opt out of executing plugins using ``execute_plugins=False`` like this: + +.. code-block:: python + + db = Database(memory=True, execute_plugins=False) + +You can pass ``strict=True`` to enable `SQLite STRICT mode `__ for all tables created using this database object: + +.. code-block:: python + + db = Database("my_database.db", strict=True) + +.. _python_api_close: + +Closing a database +------------------ + +Database objects maintain a connection to the underlying SQLite database. You can explicitly close this connection using the ``.close()`` method: + +.. code-block:: python + + db = Database("my_database.db") + # ... use the database ... + db.close() + +The ``Database`` object also works as a context manager, which will automatically close the connection when the ``with`` block exits: + +.. code-block:: python + + with Database("my_database.db") as db: + db["my_table"].insert({"name": "Example"}) + # Connection is automatically closed here + +Exiting the block is equivalent to calling ``db.close()``: the connection is closed and any transaction still open at that point is rolled back. This matches SQLite's own behavior when a connection closes. + +This rarely matters in practice. Everything that writes to the database - including raw ``db.execute()`` statements - commits automatically, so a transaction can only be open here if you explicitly started one with ``db.begin()`` and have not yet committed it. In that case the decision to commit stays with you: committing automatically on exit could silently persist half-finished work, for example if your code returned early from the block. Call ``db.commit()`` when the work is complete. + +Note this differs from the ``sqlite3.Connection`` context manager in the standard library, which commits on success but does not close the connection. See :ref:`python_api_transactions` for the full transaction model. + .. _python_api_attach: Attaching additional databases @@ -67,6 +184,9 @@ You can attach an additional database using the ``.attach()`` method, providing You can reference tables in the attached database using the alias value you passed to ``db.attach(alias, filepath)`` as a prefix, for example the ``second.table_in_second`` reference in the SQL query above. +.. note:: + In the CLI: :ref:`sqlite-utils --attach ` + .. _python_api_tracing: Tracing queries @@ -92,7 +212,7 @@ You can also turn on a tracer function temporarily for a block of code using the db = Database(memory=True) # ... later with db.tracer(print): - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) This example will print queries only for the duration of the ``with`` block. @@ -113,13 +233,33 @@ The ``db.query(sql)`` function executes a SQL query and returns an iterator over .. code-block:: python db = Database(memory=True) - db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) + db.table("dogs").insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) for row in db.query("select * from dogs"): print(row) # Outputs: # {'name': 'Cleo'} # {'name': 'Pancakes'} +The SQL query is executed as soon as ``db.query()`` is called. The resulting rows are fetched lazily as you iterate, so large result sets are not loaded into memory all at once. Because execution is immediate, an error in your SQL will raise an exception straight away, and a statement such as ``INSERT ... RETURNING`` will take effect - and be committed, unless a transaction is open - even if you do not iterate over its results. + +``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. The rejected statement is rolled back, so it has no effect on the database. Use :ref:`db.execute() ` for those statements instead. + +There is one exception to the rolled-back guarantee: a ``PRAGMA`` statement that returns no rows, such as ``PRAGMA user_version = 5``, still raises a ``ValueError`` but will already have taken effect. Some PRAGMA statements refuse to run inside a transaction, so PRAGMAs are executed outside the savepoint that is used to roll back other rejected statements. Use ``db.execute()`` for PRAGMA statements that do not return rows. + +If a query returns more than one column with the same name - a join between two tables that share column names, for example - later occurrences are renamed with a numeric suffix, so every value is included in the dictionary: + +.. code-block:: python + + row = next(db.query("select 1 as id, 2 as id, 3 as id")) + print(row) + # Outputs: + # {'id': 1, 'id_2': 2, 'id_3': 3} + +A suffix that would collide with another column in the query is skipped - ``select 1 as id, 2 as id, 3 as id_2`` returns ``{'id': 1, 'id_3': 2, 'id_2': 3}``. The same renaming is applied by ``table.rows_where()`` and ``table.search()``. + +.. note:: + In the CLI: :ref:`sqlite-utils query ` + .. _python_api_execute: db.execute(sql, params) @@ -132,7 +272,7 @@ The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around .. code-block:: python db = Database(memory=True) - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) cursor = db.execute("update dogs set name = 'Cleopaws'") print(cursor.rowcount) # Outputs the number of rows affected by the update @@ -140,6 +280,9 @@ The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation `__. +.. note:: + Write statements executed this way are committed automatically, unless a transaction is already open in which case they become part of it - see :ref:`python_api_transactions_execute`. + .. _python_api_parameters: Passing parameters @@ -147,7 +290,7 @@ Passing parameters Both ``db.query()`` and ``db.execute()`` accept an optional second argument for parameters to be passed to the SQL query. -This can take the form of either a tuple/list or a dictionary, depending on the type of parameters used in the query. Values passed in this way will be correctly quoted and escaped, helping avoid XSS vulnerabilities. +This can take the form of either a tuple/list or a dictionary, depending on the type of parameters used in the query. Values passed in this way will be correctly quoted and escaped, helping avoid SQL injection vulnerabilities. ``?`` parameters in the SQL query can be filled in using a list: @@ -168,26 +311,181 @@ Named parameters using ``:name`` can be filled using a dictionary: In this example ``next()`` is used to retrieve the first result in the iterator returned by the ``db.query()`` method. +.. _python_api_transactions: + +Transactions and saving your changes +==================================== + +Every method in this library that writes to the database - ``insert()``, ``upsert()``, ``update()``, ``delete()``, ``delete_where()``, ``transform()``, ``create_table()``, ``create_index()``, ``enable_fts()`` and the rest - runs inside its own transaction and commits it before returning. Your changes are saved to disk as soon as the method call finishes: + +.. code-block:: python + + db = Database("data.db") + db.table("news").insert({"headline": "Dog wins award"}) + # The new row is already saved - no commit() required + +The same applies to raw SQL executed with :ref:`db.execute() ` - a write statement is committed as soon as it has run. + +Another way to think about this is that each sqlite-utils method call is its own unit of work. If several method calls must either all succeed or all fail, use ``db.atomic()`` to turn them into a single unit of work. + +You never need to call ``commit()``, and you do not need to close the database to persist your changes. There are exactly two situations where you need to think about transactions: + +1. You want to group several write operations together, so they either all succeed or all fail - use :ref:`db.atomic() `. +2. You are :ref:`managing a transaction yourself ` with ``db.begin()``, in which case nothing is committed until you commit - the library will never commit a transaction you opened. + +``with Database(...) as db:`` is not a transaction block. It manages the lifetime of the database connection and closes it on exit. Use ``with db.atomic():`` for a transaction. + +.. _python_api_atomic: + +Grouping changes with db.atomic() +--------------------------------- + +Use ``db.atomic()`` to group multiple operations in a single transaction: + +.. code-block:: python + + with db.atomic(): + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("dogs").insert({"id": 2, "name": "Pancakes"}) + +The transaction commits when the block exits. If an exception is raised, changes made inside the block will be rolled back. + +This matters when several operations represent a single logical change. Without ``db.atomic()``, an earlier method call remains committed if a later one fails: + +.. code-block:: python + + # These are two separate transactions + db.table("accounts").update(1, {"balance": 90}) + db.table("accounts").update(2, {"balance": 110}) + + # These updates either both succeed or both fail + with db.atomic(): + db.table("accounts").update(1, {"balance": 90}) + db.table("accounts").update(2, {"balance": 110}) + +Transactions can also improve performance. Calling ``insert()`` repeatedly outside ``db.atomic()`` creates and commits a separate transaction for every call. For bulk inserts, prefer :ref:`insert_all() `. If you need to call several different methods in a loop, wrap the loop in ``db.atomic()``: + +.. code-block:: python + + with db.atomic(): + for row in rows: + db.table("events").insert(row) + +``db.atomic()`` can be nested. Nested blocks use SQLite savepoints, so an exception in an inner block can roll back to that savepoint without rolling back the entire outer transaction: + +.. code-block:: python + + with db.atomic(): + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") + try: + with db.atomic(): + db.table("dogs").insert({"id": 2, "name": "Pancakes"}) + raise ValueError("skip this one") + except ValueError: + pass + db.table("dogs").insert({"id": 3, "name": "Marnie"}) + +The transaction is opened with a deferred ``BEGIN`` - SQLite takes the necessary locks when the first statement inside the block runs. + +.. _python_api_transactions_execute: + +Raw SQL writes with db.execute() +-------------------------------- + +Write statements executed with :ref:`db.execute() ` follow the same rule as everything else: they are committed automatically as soon as they have run. + +.. code-block:: python + + db.execute("insert into news (headline) values (?)", ["Dog wins award"]) + # Already committed + +``db.execute()`` participates in sqlite-utils transaction handling. Calling ``db.conn.execute()`` directly bypasses that policy and leaves transaction handling to Python's underlying ``sqlite3.Connection``. Prefer ``db.execute()`` unless you deliberately need the lower-level API. + +If a transaction is open - because the call happens inside a ``db.atomic()`` block, or after ``db.begin()`` - the statement becomes part of that transaction instead, and commits when the transaction commits: + +.. code-block:: python + + with db.atomic(): + db.execute("insert into news (headline) values (?)", ["Dog wins award"]) + db.execute("insert into news (headline) values (?)", ["Cat unimpressed"]) + # Both rows committed together + +One corner case: a row-returning write such as ``INSERT ... RETURNING`` executed through ``db.execute()`` cannot be auto-committed, because its rows have not been read yet - call ``db.commit()`` after fetching them, or use :ref:`db.query() ` for those statements, which executes the write and commits it immediately. + +.. _python_api_transactions_manual: + +Managing transactions yourself +------------------------------ + +You can take full manual control using the ``db.begin()``, ``db.commit()`` and ``db.rollback()`` methods: + +.. code-block:: python + + db.begin() + db.table("news").insert({"headline": "Dog wins award"}) + if all_looks_good: + db.commit() + else: + db.rollback() + +``db.begin()`` raises ``sqlite3.OperationalError`` if a transaction is already open. ``db.commit()`` and ``db.rollback()`` do nothing if there is no open transaction. + +The library will never commit a transaction you opened. If you call write methods such as ``insert()`` - or use ``db.atomic()`` - while your transaction is open, they participate in it using SQLite savepoints instead of committing: exiting an ``atomic()`` block releases its savepoint, but nothing is saved to disk until you commit the outer transaction yourself. If you roll back, their changes are rolled back too. + +Prefer ``db.atomic()`` or ``db.begin()``, ``db.commit()`` and ``db.rollback()`` over mixing sqlite-utils transaction methods with calls to ``db.conn.commit()``, ``db.conn.rollback()`` or raw transaction-control SQL. Mixing the two layers makes it much harder to tell which layer owns the current transaction. + +Some related safeguards to be aware of: + +- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. +- ``table.transform()`` raises a ``sqlite_utils.db.TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions, because the pragma cannot be turned off mid-transaction to protect those referencing rows - see :ref:`python_api_transform_foreign_keys_transactions`. +- Closing the database - explicitly with ``db.close()``, or by exiting a ``with Database(...) as db:`` block - rolls back any transaction that is still open, see :ref:`python_api_close`. + +.. _python_api_transactions_modes: + +Supported connection modes +-------------------------- + +``db.atomic()`` and the automatic per-method transactions currently require a connection using Python's legacy transaction control mode (``sqlite3.LEGACY_TRANSACTION_CONTROL`` on Python 3.12 and later). Passing a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options to ``Database()`` raises a ``sqlite_utils.db.TransactionError``. + +Connections using ``autocommit=False`` are not supported because Python keeps a transaction open continuously. sqlite-utils uses ``Connection.in_transaction`` to distinguish its own transactions from transactions opened by its caller, and that distinction is not available in this mode. + +Connections using ``autocommit=True`` are also currently rejected because sqlite-utils has not formally exposed that as a supported configuration. + .. _python_api_table: Accessing tables ================ -Tables are accessed using the indexing operator, like so: - -.. code-block:: python - - table = db["my_table"] - -If the table does not yet exist, it will be created the first time you attempt to insert or upsert data into it. - -You can also access tables using the ``.table()`` method like so: +Tables are accessed using the ``db.table()`` method, like so: .. code-block:: python table = db.table("my_table") -Using this factory function allows you to set :ref:`python_api_table_configuration`. +Using this factory function allows you to set :ref:`python_api_table_configuration`. Additional keyword arguments to ``db.table()`` will be used if a further method call causes the table to be created. + +The ``db.table()`` method will always return a :ref:`reference_db_table` instance, or raise a ``sqlite_utils.db.NoTable`` exception if the table name is actually a SQL view. + +You can also access tables or views using dictionary-style syntax, like this: + +.. code-block:: python + + table_or_view = db["my_table_or_view_name"] + +If a table accessed using either of these methods does not yet exist, it will be created the first time you attempt to insert or upsert data into it. + +.. _python_api_view: + +Accessing views +=============== + +SQL views can be accessed using the ``db.view()`` method, like so: + +.. code-block:: python + + view = db.view("my_view") + +This will return a :ref:`reference_db_view` instance, or raise a ``sqlite_utils.db.NoView`` exception if the view does not exist. .. _python_api_tables: @@ -206,6 +504,9 @@ You can also iterate through the table objects themselves using the ``.tables`` >>> db.tables [] +.. note:: + In the CLI: :ref:`sqlite-utils tables ` + .. _python_api_views: Listing views @@ -231,6 +532,9 @@ View objects are similar to Table objects, except that any attempts to insert or * ``rows_where(where, where_args, order_by, select)`` * ``drop()`` +.. note:: + In the CLI: :ref:`sqlite-utils views ` + .. _python_api_rows: Listing rows @@ -239,7 +543,7 @@ Listing rows To iterate through dictionaries for each of the rows in a table, use ``.rows``:: >>> db = sqlite_utils.Database("dogs.db") - >>> for row in db["dogs"].rows: + >>> for row in db.table("dogs").rows: ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} {'id': 2, 'age': 2, 'name': 'Pancakes'} @@ -247,26 +551,26 @@ To iterate through dictionaries for each of the rows in a table, use ``.rows``:: You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``:: >>> db = sqlite_utils.Database("dogs.db") - >>> for row in db["dogs"].rows_where("age > ?", [3]): + >>> for row in db.table("dogs").rows_where("age > ?", [3]): ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} The first argument is a fragment of SQL. The second, optional argument is values to be passed to that fragment - you can use ``?`` placeholders and pass an array, or you can use ``:named`` parameters and pass a dictionary, like this:: - >>> for row in db["dogs"].rows_where("age > :age", {"age": 3}): + >>> for row in db.table("dogs").rows_where("age > :age", {"age": 3}): ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} To return custom columns (instead of the default that uses ``select *``) pass ``select="column1, column2"``:: >>> db = sqlite_utils.Database("dogs.db") - >>> for row in db["dogs"].rows_where(select='name, age'): + >>> for row in db.table("dogs").rows_where(select='name, age'): ... print(row) {'name': 'Cleo', 'age': 4} To specify an order, use the ``order_by=`` argument:: - >>> for row in db["dogs"].rows_where("age > 1", order_by="age"): + >>> for row in db.table("dogs").rows_where("age > 1", order_by="age"): ... print(row) {'id': 2, 'age': 2, 'name': 'Pancakes'} {'id': 1, 'age': 4, 'name': 'Cleo'} @@ -275,17 +579,20 @@ You can use ``order_by="age desc"`` for descending order. You can order all records in the table by excluding the ``where`` argument:: - >>> for row in db["dogs"].rows_where(order_by="age desc"): + >>> for row in db.table("dogs").rows_where(order_by="age desc"): ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} {'id': 2, 'age': 2, 'name': 'Pancakes'} This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an OFFSET and a LIMIT for the SQL query:: - >>> for row in db["dogs"].rows_where(order_by="age desc", limit=1): + >>> for row in db.table("dogs").rows_where(order_by="age desc", limit=1): ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} +.. note:: + In the CLI: :ref:`sqlite-utils rows ` + .. _python_api_rows_count_where: Counting rows @@ -293,7 +600,7 @@ Counting rows To count the number of rows that would be returned by a where filter, use ``.count_where(where, where_args)``: - >>> db["dogs"].count_where("age > ?", [1]): + >>> db.table("dogs").count_where("age > ?", [1]) 2 .. _python_api_pks_and_rows_where: @@ -312,21 +619,21 @@ If the table is a ``rowid`` table (with no explicit primary key column) then tha :: >>> db = sqlite_utils.Database(memory=True) - >>> db["dogs"].insert({"name": "Cleo"}) - >>> for pk, row in db["dogs"].pks_and_rows_where(): + >>> db.table("dogs").insert({"name": "Cleo"}) + >>> for pk, row in db.table("dogs").pks_and_rows_where(): ... print(pk, row) 1 {'rowid': 1, 'name': 'Cleo'} - >>> db["dogs_with_pk"].insert({"id": 5, "name": "Cleo"}, pk="id") - >>> for pk, row in db["dogs_with_pk"].pks_and_rows_where(): + >>> db.table("dogs_with_pk").insert({"id": 5, "name": "Cleo"}, pk="id") + >>> for pk, row in db.table("dogs_with_pk").pks_and_rows_where(): ... print(pk, row) 5 {'id': 5, 'name': 'Cleo'} - >>> db["dogs_with_compound_pk"].insert( + >>> db.table("dogs_with_compound_pk").insert( ... {"species": "dog", "id": 3, "name": "Cleo"}, ... pk=("species", "id") ... ) - >>> for pk, row in db["dogs_with_compound_pk"].pks_and_rows_where(): + >>> for pk, row in db.table("dogs_with_compound_pk").pks_and_rows_where(): ... print(pk, row) ('dog', 3) {'species': 'dog', 'id': 3, 'name': 'Cleo'} @@ -338,12 +645,12 @@ Retrieving a specific record You can retrieve a record by its primary key using ``table.get()``:: >>> db = sqlite_utils.Database("dogs.db") - >>> print(db["dogs"].get(1)) + >>> print(db.table("dogs").get(1)) {'id': 1, 'age': 4, 'name': 'Cleo'} If the table has a compound primary key you can pass in the primary key values as a tuple:: - >>> db["compound_dogs"].get(("mixed", 3)) + >>> db.table("compound_dogs").get(("mixed", 3)) If the record does not exist a ``NotFoundError`` will be raised: @@ -352,7 +659,7 @@ If the record does not exist a ``NotFoundError`` will be raised: from sqlite_utils.db import NotFoundError try: - row = db["dogs"].get(5) + row = db.table("dogs").get(5) except NotFoundError: print("Dog not found") @@ -365,12 +672,14 @@ The ``db.schema`` property returns the full SQL schema for the database as a str >>> db = sqlite_utils.Database("dogs.db") >>> print(db.schema) - >>> print(db.schema) CREATE TABLE "dogs" ( - [id] INTEGER PRIMARY KEY, - [name] TEXT + "id" INTEGER PRIMARY KEY, + "name" TEXT ); +.. note:: + In the CLI: :ref:`sqlite-utils schema ` + .. _python_api_creating_tables: Creating tables @@ -383,8 +692,8 @@ The easiest way to create a new table is to insert a record into it: from sqlite_utils import Database import sqlite3 - db = Database(sqlite3.connect("/tmp/dogs.db")) - dogs = db["dogs"] + db = Database("dogs.db") + dogs = db.table("dogs") dogs.insert({ "name": "Cleo", "twitter": "cleopaws", @@ -428,7 +737,7 @@ If you want to explicitly set the order of the columns you can do so using the ` .. code-block:: python - db["dogs"].insert({ + db.table("dogs").insert({ "id": 1, "name": "Cleo", "twitter": "cleopaws", @@ -442,7 +751,7 @@ Column types are detected based on the example data provided. Sometimes you may .. code-block:: python - db["dogs"].insert({ + db.table("dogs").insert({ "id": 1, "name": "Cleo", "age": "5", @@ -452,11 +761,11 @@ This will create a table with the following schema: .. code-block:: sql - CREATE TABLE [dogs] ( - [id] INTEGER PRIMARY KEY, - [name] TEXT, - [age] INTEGER, - [weight] FLOAT + CREATE TABLE "dogs" ( + "id" INTEGER PRIMARY KEY, + "name" TEXT, + "age" INTEGER, + "weight" REAL ) .. _python_api_explicit_create: @@ -464,9 +773,11 @@ This will create a table with the following schema: Explicitly creating a table --------------------------- -You can directly create a new table without inserting any data into it using the ``.create()`` method:: +You can directly create a new table without inserting any data into it using the ``.create()`` method: - db["cats"].create({ +.. code-block:: python + + db.table("cats").create({ "id": int, "name": str, "weight": float, @@ -476,6 +787,50 @@ The first argument here is a dictionary specifying the columns you would like to This method takes optional arguments ``pk=``, ``column_order=``, ``foreign_keys=``, ``not_null=set()`` and ``defaults=dict()`` - explained below. +A ``sqlite_utils.utils.sqlite3.OperationalError`` will be raised if a table of that name already exists. + +You can pass ``ignore=True`` to ignore that error. You can also use ``if_not_exists=True`` to use the SQL ``CREATE TABLE IF NOT EXISTS`` pattern to achieve the same effect: + +.. code-block:: python + + db.table("cats").create({ + "id": int, + "name": str, + }, pk="id", if_not_exists=True) + +To drop and replace any existing table of that name, pass ``replace=True``. This is a **dangerous operation** that will result in loss of existing data in the table. + +You can also pass ``transform=True`` to have any existing tables :ref:`transformed ` to match your new table specification. This is a **dangerous operation** as it will drop columns that are no longer listed in your call to ``.create()``, so be careful when running this. + +.. code-block:: python + + db.table("cats").create({ + "id": int, + "name": str, + "weight": float, + }, pk="id", transform=True) + +The ``transform=True`` option will update the table schema if any of the following have changed: + +- The specified columns or their types +- The specified primary key +- The order of the columns, defined using ``column_order=`` +- The ``not_null=`` or ``defaults=`` arguments + +Changes to ``foreign_keys=`` are not currently detected and applied by ``transform=True``. + +You can pass ``strict=True`` to create a table in ``STRICT`` mode: + +.. code-block:: python + + db.table("cats").create({ + "id": int, + "name": str, + }, strict=True) + +.. note:: + In the CLI: :ref:`sqlite-utils create-table ` + .. _python_api_compound_primary_keys: Compound primary keys @@ -485,7 +840,7 @@ If you want to create a table with a compound primary key that spans multiple co .. code-block:: python - db["cats"].create({ + db.table("cats").create({ "id": int, "breed": str, "name": str, @@ -525,25 +880,80 @@ You can leave off the third item in the tuple to have the referenced column auto .. code-block:: python - db["authors"].insert_all([ + db.table("authors").insert_all([ {"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"} ], pk="id") - db["books"].insert_all([ + db.table("books").insert_all([ {"title": "Hedgehogs of the world", "author_id": 1}, {"title": "How to train your wolf", "author_id": 2}, ], foreign_keys=[ ("author_id", "authors") ]) +.. _python_api_compound_foreign_keys: + +Compound foreign keys +~~~~~~~~~~~~~~~~~~~~~ + +To create a compound (multi-column) foreign key, use tuples of column names in place of the single column names: + +.. code-block:: python + + db.table("courses").create({ + "course_code": str, + "campus_name": str, + "dept_code": str, + }, pk="course_code", foreign_keys=[ + (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) + ]) + +This creates a table-level constraint: + +.. code-block:: sql + + CREATE TABLE "courses" ( + "course_code" TEXT PRIMARY KEY, + "campus_name" TEXT, + "dept_code" TEXT, + FOREIGN KEY ("campus_name", "dept_code") REFERENCES "departments"("campus_name", "dept_code") + ) + +As with single columns, you can leave off the tuple of other columns to reference the compound primary key of the other table: + +.. code-block:: python + + foreign_keys=[ + (("campus_name", "dept_code"), "departments") + ] + +To specify ``ON DELETE`` or ``ON UPDATE`` actions, pass ``ForeignKey`` objects instead: + +.. code-block:: python + + from sqlite_utils.db import ForeignKey + + db.table("books").create({ + "id": int, + "author_id": int, + }, pk="id", foreign_keys=[ + ForeignKey( + table="books", column="author_id", + other_table="authors", other_column="id", + on_delete="CASCADE", + ) + ]) + +Foreign key actions are preserved by :ref:`table.transform() ` - prior to sqlite-utils 4.0 they were silently dropped when a table was transformed. + .. _python_api_table_configuration: Table configuration options -=========================== +--------------------------- The ``.insert()``, ``.upsert()``, ``.insert_all()`` and ``.upsert_all()`` methods each take a number of keyword arguments, some of which influence what happens should they cause a table to be created and some of which affect the behavior of those methods. -You can set default values for these methods by accessing the table through the ``db.table(...)`` method (instead of using ``db["table_name"]``), like so: +You can set default values for these methods by accessing the table through the ``db.table(...)`` method (instead of using ``db.table("table_name")``), like so: .. code-block:: python @@ -556,12 +966,12 @@ You can set default values for these methods by accessing the table through the # Now you can call .insert() like so: table.insert({"id": 1, "name": "Tracy", "score": 5}) -The configuration options that can be specified in this way are ``pk``, ``foreign_keys``, ``column_order``, ``not_null``, ``defaults``, ``batch_size``, ``hash_id``, ``alter``, ``ignore``, ``replace``, ``extracts``, ``conversions``, ``columns``. These are all documented below. +The configuration options that can be specified in this way are ``pk``, ``foreign_keys``, ``column_order``, ``not_null``, ``defaults``, ``batch_size``, ``hash_id``, ``hash_id_columns``, ``alter``, ``ignore``, ``replace``, ``extracts``, ``conversions``, ``columns``, ``strict``. These are all documented below. .. _python_api_defaults_not_null: Setting defaults and not null constraints -========================================= +----------------------------------------- Each of the methods that can cause a table to be created take optional arguments ``not_null=set()`` and ``defaults=dict()``. The methods that take these optional arguments are: @@ -580,26 +990,68 @@ Here's an example that uses these features: .. code-block:: python - db["authors"].insert_all( + db.table("authors").insert_all( [{"id": 1, "name": "Sally", "score": 2}], pk="id", not_null={"name", "score"}, defaults={"score": 1}, ) - db["authors"].insert({"name": "Dharma"}) + db.table("authors").insert({"name": "Dharma"}) - list(db["authors"].rows) + list(db.table("authors").rows) # Outputs: # [{'id': 1, 'name': 'Sally', 'score': 2}, # {'id': 3, 'name': 'Dharma', 'score': 1}] - print(db["authors"].schema) + print(db.table("authors").schema) # Outputs: - # CREATE TABLE [authors] ( - # [id] INTEGER PRIMARY KEY, - # [name] TEXT NOT NULL, - # [score] INTEGER NOT NULL DEFAULT 1 + # CREATE TABLE "authors" ( + # "id" INTEGER PRIMARY KEY, + # "name" TEXT NOT NULL, + # "score" INTEGER NOT NULL DEFAULT 1 # ) + +.. note:: + In the CLI: :ref:`sqlite-utils insert --not-null and --default ` + +.. _python_api_rename_table: + +Renaming a table +================ + +The ``db.rename_table(old_name, new_name)`` method can be used to rename a table: + +.. code-block:: python + + db.rename_table("my_table", "new_name_for_my_table") + +This executes the following SQL: + +.. code-block:: sql + + ALTER TABLE [my_table] RENAME TO [new_name_for_my_table] + +.. note:: + In the CLI: :ref:`sqlite-utils rename-table ` + +.. _python_api_duplicate: + +Duplicating tables +================== + +The ``table.duplicate()`` method creates a copy of the table, copying both the table schema and all of the rows in that table: + +.. code-block:: python + + db.table("authors").duplicate("authors_copy") + +The new ``authors_copy`` table will now contain a duplicate copy of the data from ``authors``. + +This method raises ``sqlite_utils.db.NoTable`` if the table does not exist. + +.. note:: + In the CLI: :ref:`sqlite-utils duplicate ` + .. _python_api_bulk_inserts: Bulk inserts @@ -611,7 +1063,7 @@ Use it like this: .. code-block:: python - db["dogs"].insert_all([{ + db.table("dogs").insert_all([{ "id": 1, "name": "Cleo", "twitter": "cleopaws", @@ -631,24 +1083,81 @@ The function can accept an iterator or generator of rows and will commit them ac .. code-block:: python - db["big_table"].insert_all(({ + db.table("big_table").insert_all(({ "id": 1, "name": "Name {}".format(i), } for i in range(10000)), batch_size=1000) You can skip inserting any records that have a primary key that already exists using ``ignore=True``. This works with both ``.insert({...}, ignore=True)`` and ``.insert_all([...], ignore=True)``. -You can delete all the existing rows in the table before inserting the new -records using ``truncate=True``. This is useful if you want to replace the data in the table. +You can delete all the existing rows in the table before inserting the new records using ``truncate=True``. This is useful if you want to replace the data in the table. + +Pass ``analyze=True`` to run ``ANALYZE`` against the table after inserting the new records. + +.. note:: + In the CLI: :ref:`sqlite-utils insert ` + +.. _python_api_insert_lists: + +Inserting data from a list or tuple iterator +-------------------------------------------- + +As an alternative to passing an iterator of dictionaries, you can pass an iterator of lists or tuples. The first item yielded by the iterator must be a list or tuple of string column names, and subsequent items should be lists or tuples of values: + +.. code-block:: python + + db["creatures"].insert_all([ + ["name", "species"], + ["Cleo", "dog"], + ["Lila", "chicken"], + ["Bants", "chicken"], + ]) + +This also works with generators: + +.. code-block:: python + + def creatures(): + yield "id", "name", "city" + yield 1, "Cleo", "San Francisco" + yield 2, "Lila", "Los Angeles" + + db["creatures"].insert_all(creatures()) + +Tuples and lists are both supported. .. _python_api_insert_replace: Insert-replacing data ===================== -If you want to insert a record or replace an existing record with the same primary key, using the ``replace=True`` argument to ``.insert()`` or ``.insert_all()``:: +If you try to insert data using a primary key that already exists, the ``.insert()`` or ``.insert_all()`` method will raise a ``sqlite3.IntegrityError`` exception. - db["dogs"].insert_all([{ +This example that catches that exception: + +.. code-block:: python + + from sqlite_utils.utils import sqlite3 + + try: + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") + except sqlite3.IntegrityError: + print("Record already exists with that primary key") + +Importing from ``sqlite_utils.utils.sqlite3`` ensures your code continues to work even if you are using the ``pysqlite3`` library instead of the Python standard library ``sqlite3`` module. + +Use the ``ignore=True`` parameter to ignore this error: + +.. code-block:: python + + # This fails silently if a record with id=1 already exists + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id", ignore=True) + +To replace any existing records that have a matching primary key, use the ``replace=True`` parameter to ``.insert()`` or ``.insert_all()``: + +.. code-block:: python + + db.table("dogs").insert_all([{ "id": 1, "name": "Cleo", "twitter": "cleopaws", @@ -663,7 +1172,10 @@ If you want to insert a record or replace an existing record with the same prima }], pk="id", replace=True) .. note:: - Prior to sqlite-utils 2.x the ``.upsert()`` and ``.upsert_all()`` methods did this. See :ref:`python_api_upsert` for the new behaviour of those methods in 2.x. + Prior to sqlite-utils 2.0 the ``.upsert()`` and ``.upsert_all()`` methods worked the same way as ``.insert(replace=True)`` does today. See :ref:`python_api_upsert` for the new behaviour of those methods introduced in 2.0. + +.. note:: + In the CLI: :ref:`sqlite-utils insert --replace ` .. _python_api_update: @@ -673,21 +1185,21 @@ Updating a specific record You can update a record by its primary key using ``table.update()``:: >>> db = sqlite_utils.Database("dogs.db") - >>> print(db["dogs"].get(1)) + >>> print(db.table("dogs").get(1)) {'id': 1, 'age': 4, 'name': 'Cleo'} - >>> db["dogs"].update(1, {"age": 5}) - >>> print(db["dogs"].get(1)) + >>> db.table("dogs").update(1, {"age": 5}) + >>> print(db.table("dogs").get(1)) {'id': 1, 'age': 5, 'name': 'Cleo'} The first argument to ``update()`` is the primary key. This can be a single value, or a tuple if that table has a compound primary key:: - >>> db["compound_dogs"].update((5, 3), {"name": "Updated"}) + >>> db.table("compound_dogs").update((5, 3), {"name": "Updated"}) The second argument is a dictionary of columns that should be updated, along with their new values. You can cause any missing columns to be added automatically using ``alter=True``:: - >>> db["dogs"].update(1, {"breed": "Mutt"}, alter=True) + >>> db.table("dogs").update(1, {"breed": "Mutt"}, alter=True) .. _python_api_delete: @@ -697,11 +1209,11 @@ Deleting a specific record You can delete a record using ``table.delete()``:: >>> db = sqlite_utils.Database("dogs.db") - >>> db["dogs"].delete(1) + >>> db.table("dogs").delete(1) The ``delete()`` method takes the primary key of the record. This can be a tuple of values if the row has a compound primary key:: - >>> db["compound_dogs"].delete((5, 3)) + >>> db.table("compound_dogs").delete((5, 3)) .. _python_api_delete_where: @@ -712,10 +1224,12 @@ You can delete all records in a table that match a specific WHERE statement usin >>> db = sqlite_utils.Database("dogs.db") >>> # Delete every dog with age less than 3 - >>> db["dogs"].delete_where("age < ?", [3]) + >>> db.table("dogs").delete_where("age < ?", [3]) Calling ``table.delete_where()`` with no other arguments will delete every row in the table. +Pass ``analyze=True`` to run ``ANALYZE`` against the table after deleting the rows. + .. _python_api_upsert: Upserting data @@ -727,7 +1241,7 @@ For example, given the dogs database you could upsert the record for Cleo like s .. code-block:: python - db["dogs"].upsert({ + db.table("dogs").upsert({ "id": 1, "name": "Cleo", "twitter": "cleopaws", @@ -743,9 +1257,21 @@ Note that the ``pk`` and ``column_order`` parameters here are optional if you ar An ``upsert_all()`` method is also available, which behaves like ``insert_all()`` but performs upserts instead. +Every record passed to ``upsert()`` or ``upsert_all()`` must include a value for each primary key column - a record without one could never match an existing row, so a ``sqlite_utils.db.PrimaryKeyRequired`` exception is raised instead of quietly inserting a new row. + .. note:: ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 `__ for details of this change. +.. note:: + In the CLI: :ref:`sqlite-utils upsert ` + +.. _python_api_old_upsert: + +Alternative upserts using INSERT OR IGNORE +------------------------------------------ + +Upserts use ``INSERT INTO ... ON CONFLICT SET``. Prior to ``sqlite-utils 4.0`` these used a sequence of ``INSERT OR IGNORE`` followed by an ``UPDATE``. This older method is still used for SQLite 3.23.1 and earlier. You can force the older implementation by passing ``use_old_upsert=True`` to the ``Database()`` constructor. + .. _python_api_convert: Converting data in columns @@ -759,19 +1285,19 @@ To transform a specific column to uppercase, you would use the following: .. code-block:: python - db["dogs"].convert("name", lambda value: value.upper()) + db.table("dogs").convert("name", lambda value: value.upper()) You can pass a list of columns, in which case the transformation will be applied to each one: .. code-block:: python - db["dogs"].convert(["name", "twitter"], lambda value: value.upper()) + db.table("dogs").convert(["name", "twitter"], lambda value: value.upper()) To save the output to of the transformation to a different column, use the ``output=`` parameter: .. code-block:: python - db["dogs"].convert("name", lambda value: value.upper(), output="name_upper") + db.table("dogs").convert("name", lambda value: value.upper(), output="name_upper") This will add the new column, if it does not already exist. You can pass ``output_type=int`` or some other type to control the type of the new column - otherwise it will default to text. @@ -805,7 +1331,7 @@ A useful pattern when populating large tables in to break common values out into Creating lookup tables explicitly --------------------------------- -Calling ``db["Species"].lookup({"name": "Palm"})`` creates a table called ``Species`` (if one does not already exist) with two columns: ``id`` and ``name``. It sets up a unique constraint on the ``name`` column to guarantee it will not contain duplicate rows. It then inserts a new row with the ``name`` set to ``Palm`` and returns the new integer primary key value. +Calling ``db.table("Species").lookup({"name": "Palm"})`` creates a table called ``Species`` (if one does not already exist) with two columns: ``id`` and ``name``. It sets up a unique constraint on the ``name`` column to guarantee it will not contain duplicate rows. It then inserts a new row with the ``name`` set to ``Palm`` and returns the new integer primary key value. If the ``Species`` table already exists, it will insert the new row and return the primary key. If a row with that ``name`` already exists, it will return the corresponding primary key value directly. @@ -815,15 +1341,39 @@ If you pass in a dictionary with multiple values, both values will be used to in .. code-block:: python - db["Trees"].insert({ + db.table("Trees").insert({ "latitude": 49.1265976, "longitude": 2.5496218, - "species": db["Species"].lookup({ + "species": db.table("Species").lookup({ "common_name": "Common Juniper", "latin_name": "Juniperus communis" }) }) +The ``.lookup()`` method has an optional second argument which can be used to populate other columns in the table but only if the row does not exist yet. These columns will not be included in the unique index. + +To create a species record with a note on when it was first seen, you can use this: + +.. code-block:: python + + db.table("Species").lookup({"name": "Palm"}, {"first_seen": "2021-03-04"}) + +The first time this is called the record will be created for ``name="Palm"``. Any subsequent calls with that name will ignore the second argument, even if it includes different values. + +``None`` values are matched correctly: calling ``.lookup()`` a second time with the same values will return the primary key of the existing row even if some of those values are ``None``. + +``.lookup()`` also accepts keyword arguments, which are passed through to the :ref:`insert() method ` and can be used to influence the shape of the created table. Supported parameters are: + +- ``pk`` - which defaults to ``id`` +- ``foreign_keys`` +- ``column_order`` +- ``not_null`` +- ``defaults`` +- ``extracts`` +- ``conversions`` +- ``columns`` +- ``strict`` + .. _python_api_extracts: Populating lookup tables automatically during insert/upsert @@ -851,12 +1401,14 @@ To extract the ``species`` column out to a separate ``Species`` table, you can d trees = db.table("Trees", extracts=["species"]) # Using .insert() directly - db["Trees"].insert({ + db.table("Trees").insert({ "latitude": 49.1265976, "longitude": 2.5496218, "species": "Common Juniper" }, extracts={"species": "Species"}) +``None`` values are not extracted: no record is created for them in the lookup table and the column value stays ``null``. + .. _python_api_m2m: Working with many-to-many relationships @@ -868,7 +1420,7 @@ Here's how to create two new records and connect them via a many-to-many table i .. code-block:: python - db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id").m2m( "humans", {"id": 1, "name": "Natalie"}, pk="id" ) @@ -878,7 +1430,7 @@ The ``.m2m()`` method executes against the last record that was affected by ``.i .. code-block:: python - db["dogs"].update(1).m2m( + db.table("dogs").update(1).m2m( "humans", {"id": 2, "name": "Simon"}, pk="id" ) @@ -935,17 +1487,17 @@ You can inspect the database to see the results like this:: >>> db.table_names() ['dogs', 'characteristics', 'characteristics_dogs'] - >>> list(db["dogs"].rows) + >>> list(db.table("dogs").rows) [{'id': 1, 'name': 'Cleo'}] - >>> list(db["characteristics"].rows) + >>> list(db.table("characteristics").rows) [{'id': 1, 'name': 'Playful'}, {'id': 2, 'name': 'Opinionated'}] - >>> list(db["characteristics_dogs"].rows) + >>> list(db.table("characteristics_dogs").rows) [{'characteristics_id': 1, 'dogs_id': 1}, {'characteristics_id': 2, 'dogs_id': 1}] - >>> print(db["characteristics_dogs"].schema) - CREATE TABLE [characteristics_dogs] ( - [characteristics_id] INTEGER REFERENCES [characteristics]([id]), - [dogs_id] INTEGER REFERENCES [dogs]([id]), - PRIMARY KEY ([characteristics_id], [dogs_id]) + >>> print(db.table("characteristics_dogs").schema) + CREATE TABLE "characteristics_dogs" ( + "characteristics_id" INTEGER REFERENCES "characteristics"("id"), + "dogs_id" INTEGER REFERENCES "dogs"("id"), + PRIMARY KEY ("characteristics_id", "dogs_id") ) .. _python_api_analyze_column: @@ -953,7 +1505,26 @@ You can inspect the database to see the results like this:: Analyzing a column ================== -The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` method is used by the :ref:`analyze-tables ` CLI command. It returns a ``ColumnDetails`` named tuple with the following fields: +The ``table.analyze_column(column)`` method is used by the :ref:`analyze-tables ` CLI command. + +It takes the following arguments and options: + +``column`` - required + The name of the column to analyze + +``common_limit`` + The number of most common values to return. Defaults to 10. + +``value_truncate`` + If set to an integer, values longer than this will be truncated to this length. Defaults to None. + +``most_common`` + If set to False, the ``most_common`` field of the returned ``ColumnDetails`` will be set to None. Defaults to True. + +``least_common`` + If set to False, the ``least_common`` field of the returned ``ColumnDetails`` will be set to None. Defaults to True. + +And returns a ``ColumnDetails`` named tuple with the following fields: ``table`` The name of the table @@ -962,7 +1533,7 @@ The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` metho The name of the column ``total_rows`` - The total number of rows in the table` + The total number of rows in the table ``num_null`` The number of rows for which this column is null @@ -979,10 +1550,6 @@ The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` metho ``least_common`` The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) -``N`` defaults to 10, or you can pass a custom ``N`` using the ``common_limit`` parameter. - -You can use the ``value_truncate`` parameter to truncate values in the ``most_common`` and ``least_common`` lists to a specified number of characters. - .. _python_api_add_column: Adding columns @@ -992,21 +1559,21 @@ You can add a new column to a table using the ``.add_column(col_name, col_type)` .. code-block:: python - db["dogs"].add_column("instagram", str) - db["dogs"].add_column("weight", float) - db["dogs"].add_column("dob", datetime.date) - db["dogs"].add_column("image", "BLOB") - db["dogs"].add_column("website") # str by default + db.table("dogs").add_column("instagram", str) + db.table("dogs").add_column("weight", float) + db.table("dogs").add_column("dob", datetime.date) + db.table("dogs").add_column("image", "BLOB") + db.table("dogs").add_column("website") # str by default You can specify the ``col_type`` argument either using a SQLite type as a string, or by directly passing a Python type e.g. ``str`` or ``float``. The ``col_type`` is optional - if you omit it the type of ``TEXT`` will be used. -SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"`` or ``"BLOB"``. +SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"`` or ``"BLOB"``. If you pass a Python type, it will be mapped to SQLite types as shown here:: - float: "FLOAT" + float: "REAL" int: "INTEGER" bool: "INTEGER" str: "TEXT" @@ -1014,6 +1581,7 @@ If you pass a Python type, it will be mapped to SQLite types as shown here:: datetime.datetime: "TEXT" datetime.date: "TEXT" datetime.time: "TEXT" + datetime.timedelta: "TEXT" # If numpy is installed np.int8: "INTEGER" @@ -1024,15 +1592,15 @@ If you pass a Python type, it will be mapped to SQLite types as shown here:: np.uint16: "INTEGER" np.uint32: "INTEGER" np.uint64: "INTEGER" - np.float16: "FLOAT" - np.float32: "FLOAT" - np.float64: "FLOAT" + np.float16: "REAL" + np.float32: "REAL" + np.float64: "REAL" You can also add a column that is a foreign key reference to another table using the ``fk`` parameter: .. code-block:: python - db["dogs"].add_column("species_id", fk="species") + db.table("dogs").add_column("species_id", fk="species") This will automatically detect the name of the primary key on the species table and use that (and its type) for the new column. @@ -1040,13 +1608,16 @@ You can explicitly specify the column you wish to reference using ``fk_col``: .. code-block:: python - db["dogs"].add_column("species_id", fk="species", fk_col="ref") + db.table("dogs").add_column("species_id", fk="species", fk_col="ref") You can set a ``NOT NULL DEFAULT 'x'`` constraint on the new column using ``not_null_default``: .. code-block:: python - db["dogs"].add_column("friends_count", int, not_null_default=0) + db.table("dogs").add_column("friends_count", int, not_null_default=0) + +.. note:: + In the CLI: :ref:`sqlite-utils add-column ` .. _python_api_add_column_alter: @@ -1057,13 +1628,13 @@ You can insert or update data that includes new columns and have the table autom .. code-block:: python - db["new_table"].insert({"name": "Gareth"}) + db.table("new_table").insert({"name": "Gareth"}) # This will throw an exception: - db["new_table"].insert({"name": "Gareth", "age": 32}) + db.table("new_table").insert({"name": "Gareth", "age": 32}) # This will succeed and add a new "age" integer column: - db["new_table"].insert({"name": "Gareth", "age": 32}, alter=True) + db.table("new_table").insert({"name": "Gareth", "age": 32}, alter=True) # You can see confirm the new column like so: - print(db["new_table"].columns_dict) + print(db.table("new_table").columns_dict) # Outputs this: # {'name': , 'age': } @@ -1071,6 +1642,9 @@ You can insert or update data that includes new columns and have the table autom new_table = db.table("new_table", alter=True) new_table.insert({"name": "Gareth", "age": 32, "shoe_size": 11}) +.. note:: + In the CLI: :ref:`sqlite-utils insert --alter ` + .. _python_api_add_foreign_key: Adding foreign key constraints @@ -1078,23 +1652,23 @@ Adding foreign key constraints The SQLite ``ALTER TABLE`` statement doesn't have the ability to add foreign key references to an existing column. -It's possible to add these references through very careful manipulation of SQLite's ``sqlite_master`` table, using ``PRAGMA writable_schema``. +The ``add_foreign_key()`` method here is a convenient wrapper around :ref:`table.transform() `. -``sqlite-utils`` can do this for you, though there is a significant risk of data corruption if something goes wrong so it is advisable to create a fresh copy of your database file before attempting this. +It's also possible to add foreign keys by directly updating the `sqlite_master` table. The `sqlite-utils-fast-fks `__ plugin implements this pattern, using code that was included with ``sqlite-utils`` prior to version 3.35. Here's an example of this mechanism in action: .. code-block:: python - db["authors"].insert_all([ + db.table("authors").insert_all([ {"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"} ], pk="id") - db["books"].insert_all([ + db.table("books").insert_all([ {"title": "Hedgehogs of the world", "author_id": 1}, {"title": "How to train your wolf", "author_id": 2}, ]) - db["books"].add_foreign_key("author_id", "authors", "id") + db.table("books").add_foreign_key("author_id", "authors", "id") The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you omit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules: @@ -1107,18 +1681,37 @@ To ignore the case where the key already exists, use ``ignore=True``: .. code-block:: python - db["books"].add_foreign_key("author_id", "authors", "id", ignore=True) + db.table("books").add_foreign_key("author_id", "authors", "id", ignore=True) + +To add a compound foreign key, pass tuples of columns: + +.. code-block:: python + + db.table("courses").add_foreign_key( + ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") + ) + +As with single columns, omitting the other columns will use the compound primary key of the other table. ``other_table`` must always be specified for a compound foreign key. + +Use ``on_delete=`` and ``on_update=`` to specify ``ON DELETE`` and ``ON UPDATE`` actions for the foreign key: + +.. code-block:: python + + db.table("books").add_foreign_key( + "author_id", "authors", "id", on_delete="CASCADE" + ) + +This creates a foreign key with an ``ON DELETE CASCADE`` clause, so deleting an author will also delete their books (provided foreign key enforcement is enabled with ``PRAGMA foreign_keys = ON``). Valid actions are ``"SET NULL"``, ``"SET DEFAULT"``, ``"CASCADE"``, ``"RESTRICT"`` and the default ``"NO ACTION"``. + +.. note:: + In the CLI: :ref:`sqlite-utils add-foreign-key ` .. _python_api_add_foreign_keys: Adding multiple foreign key constraints at once ----------------------------------------------- -The final step in adding a new foreign key to a SQLite database is to run ``VACUUM``, to ensure the new foreign key is available in future introspection queries. - -``VACUUM`` against a large (multi-GB) database can take several minutes or longer. If you are adding multiple foreign keys using ``table.add_foreign_key(...)`` these can quickly add up. - -Instead, you can use ``db.add_foreign_keys(...)`` to add multiple foreign keys within a single transaction. This method takes a list of four-tuples, each one specifying a ``table``, ``column``, ``other_table`` and ``other_column``. +You can use ``db.add_foreign_keys(...)`` to add multiple foreign keys in one go. This method takes a list of four-tuples, each one specifying a ``table``, ``column``, ``other_table`` and ``other_column``. Here's an example adding two foreign keys at once: @@ -1131,6 +1724,11 @@ Here's an example adding two foreign keys at once: This method runs the same checks as ``.add_foreign_keys()`` and will raise ``sqlite_utils.db.AlterError`` if those checks fail. +Foreign keys that already exist are silently skipped, so repeated calls are idempotent - but only if they match exactly. Requesting a foreign key that exists with different ``ON DELETE``/``ON UPDATE`` actions raises ``AlterError``: use ``table.transform()`` to change the actions of an existing foreign key. + +.. note:: + In the CLI: :ref:`sqlite-utils add-foreign-keys ` + .. _python_api_index_foreign_keys: Adding indexes for all foreign keys @@ -1142,6 +1740,11 @@ If you want to ensure that every foreign key column in your database has a corre db.index_foreign_keys() +Compound foreign keys get a single composite index across their columns. + +.. note:: + In the CLI: :ref:`sqlite-utils index-foreign-keys ` + .. _python_api_drop: Dropping a table or view @@ -1151,20 +1754,27 @@ You can drop a table or view using the ``.drop()`` method: .. code-block:: python - db["my_table"].drop() + db.table("my_table").drop() + # Or for a view: + db.view("my_view").drop() + # Or for either: + db["table_or_view_name"].drop() Pass ``ignore=True`` if you want to ignore the error caused by the table or view not existing. .. code-block:: python - db["my_table"].drop(ignore=True) + db.table("my_table").drop(ignore=True) + +.. note:: + In the CLI: :ref:`sqlite-utils drop-table ` and :ref:`sqlite-utils drop-view ` .. _python_api_transform: Transforming a table ==================== -The SQLite ``ALTER TABLE`` statement is limited. It can add columns and rename tables, but it cannot drop columns, change column types, change ``NOT NULL`` status or change the primary key for a table. +The SQLite ``ALTER TABLE`` statement is limited. It can add and drop columns and rename tables, but it cannot change column types, change ``NOT NULL`` status or change the primary key for a table. The ``table.transform()`` method can do all of these things, by implementing a multi-step pattern `described in the SQLite documentation `__: @@ -1177,6 +1787,21 @@ The ``table.transform()`` method can do all of these things, by implementing a m The ``.transform()`` method takes a number of parameters, all of which are optional. +As a bonus, calling ``.transform()`` will reformat the schema for the table that is stored in SQLite to make it more readable. This works even if you call it without any arguments. + +To keep the original table around instead of dropping it, pass the ``keep_table=`` option and specify the name of the table you would like it to be renamed to: + +.. code-block:: python + + table.transform(types={"age": int}, keep_table="original_table") + +This method raises a ``sqlite_utils.db.TransformError`` exception if the table cannot be transformed, usually because there are existing constraints or indexes that are incompatible with modifications to the columns. + +.. note:: + In the CLI: :ref:`sqlite-utils transform ` + +.. _python_api_transform_alter_column_types: + Altering column types --------------------- @@ -1189,6 +1814,31 @@ To alter the type of a column, use the ``types=`` argument: See :ref:`python_api_add_column` for a list of available types. +.. _python_api_transform_strict: + +Changing strict mode +-------------------- + +The optional ``strict=`` parameter can change whether a table uses `SQLite STRICT mode `__. Pass ``strict=True`` to convert a regular table to a strict table: + +.. code-block:: python + + table.transform(strict=True) + +Pass ``strict=False`` to convert a strict table back to a regular non-strict table: + +.. code-block:: python + + table.transform(strict=False) + +The default is ``strict=None``, which preserves the table's existing strict mode. + +Passing ``strict=True`` raises ``sqlite_utils.db.TransformError`` if the available SQLite version does not support strict tables. + +Converting to a strict table validates all existing rows as they are copied into the replacement table. If a value is incompatible with its declared column type, SQLite raises ``sqlite3.IntegrityError`` and the transformation is rolled back, leaving the original table and its data unchanged. + +.. _python_api_transform_rename_columns: + Renaming columns ---------------- @@ -1199,6 +1849,8 @@ The ``rename=`` parameter can rename columns: # Rename 'age' to 'initial_age': table.transform(rename={"age": "initial_age"}) +.. _python_api_transform_drop_columns: + Dropping columns ---------------- @@ -1209,6 +1861,8 @@ To drop columns, pass them in the ``drop=`` set: # Drop the 'age' column: table.transform(drop={"age"}) +.. _python_api_transform_change_primary_keys: + Changing primary keys --------------------- @@ -1219,6 +1873,8 @@ To change the primary key for a table, use ``pk=``. This can be passed a single # Make `user_id` the new primary key table.transform(pk="user_id") +.. _python_api_transform_change_not_null: + Changing not null status ------------------------ @@ -1239,6 +1895,8 @@ If you want to take existing ``NOT NULL`` columns and change them to allow null # Make age allow NULL and switch weight to being NOT NULL: table.transform(not_null={"age": False, "weight": True}) +.. _python_api_transform_alter_column_defaults: + Altering column defaults ------------------------ @@ -1252,6 +1910,8 @@ The ``defaults=`` parameter can be used to set or change the defaults for differ # Now remove the default from that column: table.transform(defaults={"age": None}) +.. _python_api_transform_change_column_order: + Changing column order --------------------- @@ -1262,6 +1922,47 @@ The ``column_order=`` parameter can be used to change the order of the columns. # Change column order table.transform(column_order=("name", "age", "id") +.. _python_api_transform_add_foreign_key_constraints: + +Adding foreign key constraints +------------------------------ + +You can add one or more foreign key constraints to a table using the ``add_foreign_keys=`` parameter: + +.. code-block:: python + + db.table("places").transform( + add_foreign_keys=( + ("country", "country", "id"), + ("continent", "continent", "id") + ) + ) + +This accepts the same arguments described in :ref:`specifying foreign keys ` - so you can specify them as a full tuple of ``(column, other_table, other_column)``, or you can take a shortcut and pass just the name of the column, provided the table can be automatically derived from the column name: + +.. code-block:: python + + db.table("places").transform( + add_foreign_keys=(("country", "continent")) + ) + +.. _python_api_transform_replace_foreign_key_constraints: + +Replacing foreign key constraints +--------------------------------- + +The ``foreign_keys=`` parameter is similar to to ``add_foreign_keys=`` but can be be used to replace all foreign key constraints on a table, dropping any that are not explicitly mentioned: + +.. code-block:: python + + db.table("places").transform( + foreign_keys=( + ("continent", "continent", "id"), + ) + ) + +.. _python_api_transform_drop_foreign_key_constraints: + Dropping foreign key constraints -------------------------------- @@ -1271,10 +1972,20 @@ This example drops two foreign keys - the one from ``places.country`` to ``count .. code-block:: python - db["places"].transform( + db.table("places").transform( drop_foreign_keys=("country", "continent") ) +A bare column name drops any foreign key that column participates in, including compound foreign keys. To target a compound foreign key precisely, pass a tuple of its columns: + +.. code-block:: python + + db.table("courses").transform( + drop_foreign_keys=[("campus_name", "dept_code")] + ) + +Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint. + .. _python_api_transform_sql: Custom transformations with .transform_sql() @@ -1286,6 +1997,36 @@ If you want to do something more advanced, you can call the ``table.transform_sq This method will return a list of SQL statements that should be executed to implement the change. You can then make modifications to that SQL - or add additional SQL statements - before executing it yourself. +.. _python_api_transform_foreign_keys_transactions: + +Foreign keys and transactions +----------------------------- + +Because ``.transform()`` drops the old table, running it with ``PRAGMA foreign_keys`` enabled could fire ``ON DELETE`` actions on any tables that reference it - an inbound ``ON DELETE CASCADE`` foreign key would silently delete those referencing rows. To prevent this, ``.transform()`` turns ``PRAGMA foreign_keys`` off for the duration of the operation and restores it afterwards, running ``PRAGMA foreign_key_check`` before committing. + +``PRAGMA foreign_keys`` cannot be changed inside a transaction, so this protection is impossible if you call ``.transform()`` while a transaction is already open - for example inside a ``with db.atomic():`` block or after ``db.begin()``. If ``PRAGMA foreign_keys`` is on and another table references the table being transformed with a destructive ``ON DELETE`` action - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT`` - the method will refuse to run and raise a ``sqlite_utils.db.TransactionError``: + +.. code-block:: python + + from sqlite_utils.db import TransactionError + + try: + with db.atomic(): + db["authors"].transform(types={"id": str}) + except TransactionError as ex: + print("Could not transform in transaction:", ex) + +To transform such a table either call ``.transform()`` outside of the transaction, or execute ``PRAGMA foreign_keys = off`` before opening it: + +.. code-block:: python + + db.execute("PRAGMA foreign_keys = off") + with db.atomic(): + db["authors"].transform(types={"id": str}) + db.execute("PRAGMA foreign_keys = on") + +Tables referenced by foreign keys without a destructive action (the default ``NO ACTION``, or ``RESTRICT``) can still be transformed inside a transaction - sqlite-utils uses ``PRAGMA defer_foreign_keys`` to postpone the foreign key checks until the transaction commits. + .. _python_api_extract: Extracting columns into a separate table @@ -1310,26 +2051,26 @@ The schema of the above table is: .. code-block:: sql - CREATE TABLE [Trees] ( - [id] INTEGER PRIMARY KEY, - [TreeAddress] TEXT, - [Species] TEXT + CREATE TABLE "Trees" ( + "id" INTEGER PRIMARY KEY, + "TreeAddress" TEXT, + "Species" TEXT ) Here's how to extract the ``Species`` column using ``.extract()``: .. code-block:: python - db["Trees"].extract("Species") + db.table("Trees").extract("Species") After running this code the table schema now looks like this: .. code-block:: sql CREATE TABLE "Trees" ( - [id] INTEGER PRIMARY KEY, - [TreeAddress] TEXT, - [Species_id] INTEGER, + "id" INTEGER PRIMARY KEY, + "TreeAddress" TEXT, + "Species_id" INTEGER, FOREIGN KEY(Species_id) REFERENCES Species(id) ) @@ -1337,9 +2078,9 @@ A new ``Species`` table will have been created with the following schema: .. code-block:: sql - CREATE TABLE [Species] ( - [id] INTEGER PRIMARY KEY, - [Species] TEXT + CREATE TABLE "Species" ( + "id" INTEGER PRIMARY KEY, + "Species" TEXT ) The ``.extract()`` method defaults to creating a table with the same name as the column that was extracted, and adding a foreign key column called ``tablename_id``. @@ -1348,22 +2089,22 @@ You can specify a custom table name using ``table=``, and a custom foreign key n .. code-block:: python - db["Trees"].extract("Species", table="tree_species", fk_column="tree_species_id") + db.table("Trees").extract("Species", table="tree_species", fk_column="tree_species_id") The resulting schema looks like this: .. code-block:: sql CREATE TABLE "Trees" ( - [id] INTEGER PRIMARY KEY, - [TreeAddress] TEXT, - [tree_species_id] INTEGER, + "id" INTEGER PRIMARY KEY, + "TreeAddress" TEXT, + "tree_species_id" INTEGER, FOREIGN KEY(tree_species_id) REFERENCES tree_species(id) ) - CREATE TABLE [tree_species] ( - [id] INTEGER PRIMARY KEY, - [Species] TEXT + CREATE TABLE "tree_species" ( + "id" INTEGER PRIMARY KEY, + "Species" TEXT ) You can also extract multiple columns into the same external table. Say for example you have a table like this: @@ -1381,51 +2122,51 @@ You can pass ``["CommonName", "LatinName"]`` to ``.extract()`` to extract both o .. code-block:: python - db["Trees"].extract(["CommonName", "LatinName"]) + db.table("Trees").extract(["CommonName", "LatinName"]) This produces the following schema: .. code-block:: sql CREATE TABLE "Trees" ( - [id] INTEGER PRIMARY KEY, - [TreeAddress] TEXT, - [CommonName_LatinName_id] INTEGER, + "id" INTEGER PRIMARY KEY, + "TreeAddress" TEXT, + "CommonName_LatinName_id" INTEGER, FOREIGN KEY(CommonName_LatinName_id) REFERENCES CommonName_LatinName(id) ) - CREATE TABLE [CommonName_LatinName] ( - [id] INTEGER PRIMARY KEY, - [CommonName] TEXT, - [LatinName] TEXT + CREATE TABLE "CommonName_LatinName" ( + "id" INTEGER PRIMARY KEY, + "CommonName" TEXT, + "LatinName" TEXT ) The table name ``CommonName_LatinName`` is derived from the extract columns. You can use ``table=`` and ``fk_column=`` to specify custom names like this: .. code-block:: python - db["Trees"].extract(["CommonName", "LatinName"], table="Species", fk_column="species_id") + db.table("Trees").extract(["CommonName", "LatinName"], table="Species", fk_column="species_id") This produces the following schema: .. code-block:: sql CREATE TABLE "Trees" ( - [id] INTEGER PRIMARY KEY, - [TreeAddress] TEXT, - [species_id] INTEGER, + "id" INTEGER PRIMARY KEY, + "TreeAddress" TEXT, + "species_id" INTEGER, FOREIGN KEY(species_id) REFERENCES Species(id) ) - CREATE TABLE [Species] ( - [id] INTEGER PRIMARY KEY, - [CommonName] TEXT, - [LatinName] TEXT + CREATE TABLE "Species" ( + "id" INTEGER PRIMARY KEY, + "CommonName" TEXT, + "LatinName" TEXT ) You can use the ``rename=`` argument to rename columns in the lookup table. To create a ``Species`` table with columns called ``name`` and ``latin`` you can do this: .. code-block:: python - db["Trees"].extract( + db.table("Trees").extract( ["CommonName", "LatinName"], table="Species", fk_column="species_id", @@ -1436,12 +2177,17 @@ This produces a lookup table like so: .. code-block:: sql - CREATE TABLE [Species] ( - [id] INTEGER PRIMARY KEY, - [name] TEXT, - [latin] TEXT + CREATE TABLE "Species" ( + "id" INTEGER PRIMARY KEY, + "name" TEXT, + "latin" TEXT ) +Rows where every extracted column is ``null`` are not extracted: no record is created for them in the lookup table and their foreign key column is left as ``null``. When extracting multiple columns, rows where at least one of the extracted columns has a value will be extracted as usual. + +.. note:: + In the CLI: :ref:`sqlite-utils extract ` + .. _python_api_hash: Setting an ID based on the hash of the row contents @@ -1454,7 +2200,7 @@ In these cases, a useful technique is to create an ID that is derived from the s ``sqlite-utils`` can do this for you using the ``hash_id=`` option. For example:: db = sqlite_utils.Database("dogs.db") - db["dogs"].upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id") + db.table("dogs").upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id") print(list(db["dogs])) Outputs:: @@ -1463,12 +2209,23 @@ Outputs:: If you are going to use that ID straight away, you can access it using ``last_pk``:: - dog_id = db["dogs"].upsert({ + dog_id = db.table("dogs").upsert({ "name": "Cleo", "twitter": "cleopaws" }, hash_id="id").last_pk # dog_id is now "f501265970505d9825d8d9f590bfab3519fb20b1" +The hash will be created using all of the column values. To create a hash using a subset of the columns, pass the ``hash_id_columns=`` parameter:: + + db.table("dogs").upsert( + {"name": "Cleo", "twitter": "cleopaws", "age": 7}, + hash_id_columns=("name", "twitter") + ) + +The ``hash_id=`` parameter is optional if you specify ``hash_id_columns=`` - it will default to putting the hash in a column called ``id``. + +You can manually calculate these hashes using the :ref:`hash_record(record, keys=...) ` utility function. + .. _python_api_create_view: Creating views @@ -1492,6 +2249,9 @@ You can pass ``ignore=True`` to silently ignore an existing view and do nothing, select * from dogs where is_good_dog = 1 """, replace=True) +.. note:: + In the CLI: :ref:`sqlite-utils create-view ` + Storing JSON ============ @@ -1501,7 +2261,7 @@ For example: .. code-block:: python - db["niche_museums"].insert({ + db.table("niche_museums").insert({ "name": "The Bigfoot Discovery Museum", "url": "http://bigfootdiscoveryproject.com/" "hours": { @@ -1537,11 +2297,11 @@ You can specify an upper case conversion for a specific column like so: .. code-block:: python - db["example"].insert({ + db.table("example").insert({ "name": "The Bigfoot Discovery Museum" }, conversions={"name": "upper(?)"}) - # list(db["example"].rows) now returns: + # list(db.table("example").rows) now returns: # [{'name': 'THE BIGFOOT DISCOVERY MUSEUM'}] The dictionary key is the column name to be converted. The value is the SQL fragment to use, with a ``?`` placeholder for the original value. @@ -1553,48 +2313,77 @@ A more useful example: if you are working with `SpatiaLite `__ and depends on the `Shapely `__ and `HTTPX `__ Python libraries. + +.. _python_api_sqlite_version: + +Checking the SQLite version +=========================== + +The ``db.sqlite_version`` property returns a tuple of integers representing the version of SQLite used for that database object:: + + >>> db.sqlite_version + (3, 36, 0) + +.. _python_api_itedump: + +Dumping the database to SQL +=========================== + +The ``db.iterdump()`` method returns a sequence of SQL strings representing a complete dump of the database. Use it like this: + +.. code-block:: python + + full_sql = "".join(db.iterdump()) + +This uses the `sqlite3.Connection.iterdump() `__ method. + +If you are using ``pysqlite3`` the underlying method may be missing. If you install the `sqlite-dump `__ package then the ``db.iterdump()`` method will use that implementation instead: + +.. code-block:: bash + + pip install sqlite-dump + +.. note:: + In the CLI: :ref:`sqlite-utils dump ` + .. _python_api_introspection: -Introspection -============= +Introspecting tables and views +============================== If you have loaded an existing table or view, you can use introspection to find out more about it:: - >>> db["PlantType"] + >>> db.table("PlantType")
+ ### db.view("NameOfView") + .. _python_api_introspection_exists: @@ -1603,9 +2392,11 @@ If you have loaded an existing table or view, you can use introspection to find The ``.exists()`` method can be used to find out if a table exists or not:: - >>> db["PlantType"].exists() + >>> db.table("PlantType").exists() True - >>> db["PlantType2"].exists() + >>> db.table("PlantType2").exists() + False + >>> db["table_or_view_name"].exists() False .. _python_api_introspection_count: @@ -1615,9 +2406,9 @@ The ``.exists()`` method can be used to find out if a table exists or not:: The ``.count`` property shows the current number of rows (``select count(*) from table``):: - >>> db["PlantType"].count + >>> db.table("PlantType").count 3 - >>> db["Street_Tree_List"].count + >>> db.table("Street_Tree_List").count 189144 This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.count_where()`` instead of accessing the property. @@ -1631,7 +2422,7 @@ The ``.columns`` property shows the columns in the table or view. It returns a l :: - >>> db["PlantType"].columns + >>> db.table("PlantType").columns [Column(cid=0, name='id', type='INTEGER', notnull=0, default_value=None, is_pk=1), Column(cid=1, name='value', type='TEXT', notnull=0, default_value=None, is_pk=0)] @@ -1642,9 +2433,19 @@ The ``.columns`` property shows the columns in the table or view. It returns a l The ``.columns_dict`` property returns a dictionary version of the columns with just the names and Python types:: - >>> db["PlantType"].columns_dict + >>> db.table("PlantType").columns_dict {'id': , 'value': } +.. _python_api_introspection_default_values: + +.default_values +--------------- + +The ``.default_values`` property returns a dictionary of default values for each column that has a default:: + + >>> db.table("table_with_defaults").default_values + {'score': 5} + .. _python_api_introspection_pks: .pks @@ -1652,7 +2453,7 @@ The ``.columns_dict`` property returns a dictionary version of the columns with The ``.pks`` property returns a list of strings naming the primary key columns for the table:: - >>> db["PlantType"].pks + >>> db.table("PlantType").pks ['id'] If a table has no primary keys but is a `rowid table `__, this property will return ``['rowid']``. @@ -1664,7 +2465,7 @@ If a table has no primary keys but is a `rowid table >> db["PlantType"].use_rowid + >>> db.table("PlantType").use_rowid False @@ -1673,17 +2474,39 @@ Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly .foreign_keys ------------- -The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views. +The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey`` objects. It is not available on views. + +Each ``ForeignKey`` has the following attributes: + +``table`` + The table the foreign key is defined on. +``column`` + The column on this table, or ``None`` for a compound foreign key. +``other_table`` + The table being referenced. +``other_column`` + The referenced column, or ``None`` for a compound foreign key. +``columns`` + A tuple of the columns on this table, always populated (a one-item tuple for single-column foreign keys). +``other_columns`` + A tuple of the referenced columns. +``is_compound`` + ``True`` if this is a compound (multi-column) foreign key. +``on_delete`` + The ``ON DELETE`` action, e.g. ``"CASCADE"`` - ``"NO ACTION"`` if not set. +``on_update`` + The ``ON UPDATE`` action - ``"NO ACTION"`` if not set. + +``ForeignKey`` was a ``namedtuple`` prior to sqlite-utils 4.0. It is now a dataclass and can no longer be unpacked or indexed as a tuple - access its fields by name instead. See :ref:`upgrading_3_to_4` for details. :: - >>> db["Street_Tree_List"].foreign_keys - [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qSiteInfo', other_table='qSiteInfo', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qSpecies', other_table='qSpecies', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'), - ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')] + >>> db.table("Street_Tree_List").foreign_keys + [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=('qLegalStatus',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), + ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=('qCareAssistant',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), + ...] + +Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` tuples. .. _python_api_introspection_schema: @@ -1692,7 +2515,7 @@ The ``.foreign_keys`` property returns any foreign key relationships for the tab The ``.schema`` property outputs the table's schema as a SQL string:: - >>> print(db["Street_Tree_List"].schema) + >>> print(db.table("Street_Tree_List").schema) CREATE TABLE "Street_Tree_List" ( "TreeID" INTEGER, "qLegalStatus" INTEGER, @@ -1713,12 +2536,24 @@ The ``.schema`` property outputs the table's schema as a SQL string:: "Longitude" REAL, "Location" TEXT , - FOREIGN KEY ("PlantType") REFERENCES [PlantType](id), - FOREIGN KEY ("qCaretaker") REFERENCES [qCaretaker](id), - FOREIGN KEY ("qSpecies") REFERENCES [qSpecies](id), - FOREIGN KEY ("qSiteInfo") REFERENCES [qSiteInfo](id), - FOREIGN KEY ("qCareAssistant") REFERENCES [qCareAssistant](id), - FOREIGN KEY ("qLegalStatus") REFERENCES [qLegalStatus](id)) + FOREIGN KEY ("PlantType") REFERENCES "PlantType"(id), + FOREIGN KEY ("qCaretaker") REFERENCES "qCaretaker"(id), + FOREIGN KEY ("qSpecies") REFERENCES "qSpecies"(id), + FOREIGN KEY ("qSiteInfo") REFERENCES "qSiteInfo"(id), + FOREIGN KEY ("qCareAssistant") REFERENCES "qCareAssistant"(id), + FOREIGN KEY ("qLegalStatus") REFERENCES "qLegalStatus"(id)) + +.. _python_api_introspection_strict: + +.strict +------- + +The ``.strict`` property identifies if the table is a `SQLite STRICT table `__. + +:: + + >>> db.table("ny_times_us_counties").strict + False .. _python_api_introspection_indexes: @@ -1729,7 +2564,7 @@ The ``.indexes`` property returns all indexes created for a table, as a list of :: - >>> db["Street_Tree_List"].indexes + >>> db.table("Street_Tree_List").indexes [Index(seq=0, name='"Street_Tree_List_qLegalStatus"', unique=0, origin='c', partial=0, columns=['qLegalStatus']), Index(seq=1, name='"Street_Tree_List_qCareAssistant"', unique=0, origin='c', partial=0, columns=['qCareAssistant']), Index(seq=2, name='"Street_Tree_List_qSiteInfo"', unique=0, origin='c', partial=0, columns=['qSiteInfo']), @@ -1737,6 +2572,9 @@ The ``.indexes`` property returns all indexes created for a table, as a list of Index(seq=4, name='"Street_Tree_List_qCaretaker"', unique=0, origin='c', partial=0, columns=['qCaretaker']), Index(seq=5, name='"Street_Tree_List_PlantType"', unique=0, origin='c', partial=0, columns=['PlantType'])] +.. note:: + In the CLI: :ref:`sqlite-utils indexes ` + .. _python_api_introspection_xindexes: .xindexes @@ -1746,7 +2584,7 @@ The ``.xindexes`` property returns more detailed information about the indexes o :: - >>> db["ny_times_us_counties"].xindexes + >>> db.table("ny_times_us_counties").xindexes [ XIndex( name='idx_ny_times_us_counties_date', @@ -1773,12 +2611,15 @@ The ``.triggers`` property lists database triggers. It can be used on both datab :: - >>> db["authors"].triggers + >>> db.table("authors").triggers [Trigger(name='authors_ai', table='authors', sql='CREATE TRIGGER [authors_ai] AFTER INSERT...'), Trigger(name='authors_ad', table='authors', sql="CREATE TRIGGER [authors_ad] AFTER DELETE..."), Trigger(name='authors_au', table='authors', sql="CREATE TRIGGER [authors_au] AFTER UPDATE")] >>> db.triggers - ... similar output to db["authors"].triggers + ... similar output to db.table("authors").triggers + +.. note:: + In the CLI: :ref:`sqlite-utils triggers ` .. _python_api_introspection_triggers_dict: @@ -1789,7 +2630,7 @@ The ``.triggers_dict`` property returns the triggers for that table as a diction :: - >>> db["authors"].triggers_dict + >>> db.table("authors").triggers_dict {'authors_ai': 'CREATE TRIGGER [authors_ai] AFTER INSERT...', 'authors_ad': 'CREATE TRIGGER [authors_ad] AFTER DELETE...', 'authors_au': 'CREATE TRIGGER [authors_au] AFTER UPDATE'} @@ -1812,7 +2653,7 @@ The ``detect_fts()`` method returns the associated SQLite FTS table name, if one :: - >>> db["authors"].detect_fts() + >>> db.table("authors").detect_fts() "authors_fts" .. _python_api_introspection_virtual_table_using: @@ -1822,8 +2663,8 @@ The ``detect_fts()`` method returns the associated SQLite FTS table name, if one The ``.virtual_table_using`` property reveals if a table is a virtual table. It returns ``None`` for regular tables and the upper case version of the type of virtual table otherwise. For example:: - >>> db["authors"].enable_fts(["name"]) - >>> db["authors_fts"].virtual_table_using + >>> db.table("authors").enable_fts(["name"]) + >>> db.table("authors_fts").virtual_table_using "FTS5" .. _python_api_introspection_has_counts_triggers: @@ -1835,28 +2676,47 @@ The ``.has_counts_triggers`` property shows if a table has been configured with :: - >>> db["authors"].has_counts_triggers + >>> db.table("authors").has_counts_triggers False - >>> db["authors"].enable_counts() - >>> db["authors"].has_counts_triggers + >>> db.table("authors").enable_counts() + >>> db.table("authors").has_counts_triggers + True + +.. _python_api_introspection_supports_strict: + +db.supports_strict +------------------ + +This property on the database object returns ``True`` if the available SQLite version supports `STRICT mode `__, which was added in SQLite 3.37.0 (on 2021-11-27). + +:: + + >>> db.supports_strict True .. _python_api_fts: -Enabling full-text search -========================= +Full-text search +================ + +SQLite includes bundled extensions that implement `powerful full-text search `__. + +.. _python_api_fts_enable: + +Enabling full-text search for a table +------------------------------------- You can enable full-text search on a table using ``.enable_fts(columns)``: .. code-block:: python - db["dogs"].enable_fts(["name", "twitter"]) + db.table("dogs").enable_fts(["name", "twitter"]) You can then run searches using the ``.search()`` method: .. code-block:: python - rows = list(db["dogs"].search("cleo")) + rows = list(db.table("dogs").search("cleo")) This method returns a generator that can be looped over to get dictionaries for each row, similar to :ref:`python_api_rows`. @@ -1864,32 +2724,32 @@ If you insert additional records into the table you will need to refresh the sea .. code-block:: python - db["dogs"].insert({ + db.table("dogs").insert({ "id": 2, "name": "Marnie", "twitter": "MarnieTheDog", "age": 16, "is_good_dog": True, }, pk="id") - db["dogs"].populate_fts(["name", "twitter"]) + db.table("dogs").populate_fts(["name", "twitter"]) A better solution is to use database triggers. You can set up database triggers to automatically update the full-text index using ``create_triggers=True``: .. code-block:: python - db["dogs"].enable_fts(["name", "twitter"], create_triggers=True) + db.table("dogs").enable_fts(["name", "twitter"], create_triggers=True) ``.enable_fts()`` defaults to using `FTS5 `__. If you wish to use `FTS4 `__ instead, use the following: .. code-block:: python - db["dogs"].enable_fts(["name", "twitter"], fts_version="FTS4") + db.table("dogs").enable_fts(["name", "twitter"], fts_version="FTS4") You can customize the tokenizer configured for the table using the ``tokenize=`` parameter. For example, to enable Porter stemming, where English words like "running" will match stemmed alternatives such as "run", use ``tokenize="porter"``: .. code-block:: python - db["articles"].enable_fts(["headline", "body"], tokenize="porter") + db.table("articles").enable_fts(["headline", "body"], tokenize="porter") The SQLite documentation has more on `FTS5 tokenizers `__ and `FTS4 tokenizers `__. ``porter`` is a valid option for both. @@ -1899,7 +2759,7 @@ You can replace the existing table with a new configuration using ``replace=True .. code-block:: python - db["articles"].enable_fts(["headline"], tokenize="porter", replace=True) + db.table("articles").enable_fts(["headline"], tokenize="porter", replace=True) This will have no effect if the FTS table already exists, otherwise it will drop and recreate the table with the new settings. This takes into consideration the columns, the tokenizer, the FTS version used and whether or not the table has triggers. @@ -1907,7 +2767,22 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab .. code-block:: python - db["dogs"].disable_fts() + db.table("dogs").disable_fts() + +.. note:: + In the CLI: :ref:`sqlite-utils enable-fts ` + +.. _python_api_quote_fts: + +Quoting characters for use in search +------------------------------------ + +SQLite supports `advanced search query syntax `__. In some situations you may wish to disable this, since characters such as ``.`` may have special meaning that causes errors when searching for strings provided by your users. + +The ``db.quote_fts(query)`` method returns the query with SQLite full-text search quoting applied such that the query should be safe to use in a search:: + + db.quote_fts("Search term.") + # Returns: '"Search" "term."' .. _python_api_fts_search: @@ -1918,7 +2793,7 @@ The ``table.search(q)`` method returns a generator over Python dictionaries repr .. code-block:: python - for article in db["articles"].search("jquery"): + for article in db.table("articles").search("jquery"): print(article) The ``.search()`` method also accepts the following optional parameters: @@ -1935,28 +2810,45 @@ The ``.search()`` method also accepts the following optional parameters: ``offset`` integer Offset to use along side the limit parameter. -To return just the title and published columns for three matches for ``"dog"`` ordered by ``published`` with the most recent first, use the following: +``where`` string + Extra SQL fragment for the WHERE clause + +``where_args`` dictionary + Arguments to use for ``:param`` placeholders in the extra WHERE clause + +``include_rank`` bool + If set a ``rank`` column will be included with the BM25 ranking score - for FTS5 tables only. + +``quote`` bool + Apply :ref:`FTS quoting rules ` to the search query, disabling advanced query syntax in a way that avoids surprising errors. + +To return just the title and published columns for three matches for ``"dog"`` where the ``id`` is greater than 10 ordered by ``published`` with the most recent first, use the following: .. code-block:: python - for article in db["articles"].search( + for article in db.table("articles").search( "dog", order_by="published desc", limit=3, + where="id > :min_id", + where_args={"min_id": 10}, columns=["title", "published"] ): print(article) +.. note:: + In the CLI: :ref:`sqlite-utils search ` + .. _python_api_fts_search_sql: Building SQL queries with table.search_sql() -------------------------------------------- -You can generate the SQL query that would be used for a search using the ``table.search_sql()`` method. It takes the same arguments as ``table.search()`` with the exception of the search query itself, since the returned SQL includes a parameter that can be used for the search. +You can generate the SQL query that would be used for a search using the ``table.search_sql()`` method. It takes the same arguments as ``table.search()``, with the exception of the search query and the ``where_args`` parameter, since those should be provided when the returned SQL is executed. .. code-block:: python - print(db["articles"].search_sql(columns=["title", "author"])) + print(db.table("articles").search_sql(columns=["title", "author"])) Outputs: @@ -2019,18 +2911,21 @@ You can rebuild a table using the ``table.rebuild_fts()`` method. This is useful .. code-block:: python - db["dogs"].rebuild_fts() + db.table("dogs").rebuild_fts() This method can be called on a table that has been configured for full-text search - ``dogs`` in this instance - or directly on a ``_fts`` table: .. code-block:: python - db["dogs_fts"].rebuild_fts() + db.table("dogs_fts").rebuild_fts() This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("rebuild"); +.. note:: + In the CLI: :ref:`sqlite-utils rebuild-fts ` + .. _python_api_fts_optimize: Optimizing a full-text search table @@ -2040,12 +2935,15 @@ Once you have populated a FTS table you can optimize it to dramatically reduce i .. code-block:: python - db["dogs"].optimize() + db.table("dogs").optimize() This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("optimize"); +.. note:: + In the CLI: :ref:`sqlite-utils optimize ` + .. _python_api_cached_table_counts: Cached table counts using triggers @@ -2057,15 +2955,15 @@ The ``table.enable_counts()`` method can be used to configure triggers to contin .. code-block:: python - db["dogs"].enable_counts() + db.table("dogs").enable_counts() This will create the ``_counts`` table if it does not already exist, with the following schema: .. code-block:: sql - CREATE TABLE [_counts] ( - [table] TEXT PRIMARY KEY, - [count] INTEGER DEFAULT 0 + CREATE TABLE "_counts" ( + "table" TEXT PRIMARY KEY, + "count" INTEGER DEFAULT 0 ) You can enable cached counts for every table in a database (except for virtual tables and the ``_counts`` table itself) using the database ``enable_counts()`` method: @@ -2108,6 +3006,9 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y db.reset_counts() +.. note:: + In the CLI: :ref:`sqlite-utils enable-counts ` + .. _python_api_create_index: Creating indexes @@ -2117,13 +3018,15 @@ You can create an index on a table using the ``.create_index(columns)`` method. .. code-block:: python - db["dogs"].create_index(["is_good_dog"]) + db.table("dogs").create_index(["is_good_dog"]) -By default the index will be named ``idx_{table-name}_{columns}`` - if you want to customize the name of the created index you can pass the ``index_name`` parameter: +By default the index will be named ``idx_{table-name}_{columns}``. If you pass ``find_unique_name=True`` and the automatically derived name already exists, an available name will be found by incrementing a suffix number, for example ``idx_items_title_2``. + +You can customize the name of the created index by passing the ``index_name`` parameter: .. code-block:: python - db["dogs"].create_index( + db.table("dogs").create_index( ["is_good_dog", "age"], index_name="good_dogs_by_age" ) @@ -2134,7 +3037,7 @@ To create an index in descending order for a column, wrap the column name in ``d from sqlite_utils.db import DescIndex - db["dogs"].create_index( + db.table("dogs").create_index( ["is_good_dog", DescIndex("age")], index_name="good_dogs_by_age" ) @@ -2143,10 +3046,53 @@ You can create a unique index by passing ``unique=True``: .. code-block:: python - db["dogs"].create_index(["name"], unique=True) + db.table("dogs").create_index(["name"], unique=True) Use ``if_not_exists=True`` to do nothing if an index with that name already exists. +Pass ``analyze=True`` to run ``ANALYZE`` against the new index after creating it. + +You can drop an index from a table using ``.drop_index(index_name)``: + +.. code-block:: python + + db.table("dogs").drop_index("idx_dogs_name") + +Use ``ignore=True`` to ignore the error if the index does not exist. + +.. note:: + In the CLI: :ref:`sqlite-utils create-index ` and :ref:`sqlite-utils drop-index ` + +.. _python_api_analyze: + +Optimizing index usage with ANALYZE +=================================== + +The `SQLite ANALYZE command `__ builds a table of statistics which the query planner can use to make better decisions about which indexes to use for a given query. + +You should run ``ANALYZE`` if your database is large and you do not think your indexes are being efficiently used. + +To run ``ANALYZE`` against every index in a database, use this: + +.. code-block:: python + + db.analyze() + +To run it just against a specific named index, pass the name of the index to that method: + +.. code-block:: python + + db.analyze("idx_countries_country_name") + +To run against all indexes attached to a specific table, you can either pass the table name to ``db.analyze(...)`` or you can call the method directly on the table, like this: + +.. code-block:: python + + db.table("dogs").analyze() + +.. note:: + In the CLI: :ref:`sqlite-utils analyze ` + .. _python_api_vacuum: Vacuum @@ -2158,6 +3104,9 @@ You can optimize your database by running VACUUM against it like so: Database("my_database.db").vacuum() +.. note:: + In the CLI: :ref:`sqlite-utils vacuum ` + .. _python_api_wal: WAL mode @@ -2175,6 +3124,8 @@ You can disable WAL mode using ``.disable_wal()``: Database("my_database.db").disable_wal() +The journal mode can only be changed outside of a transaction. Calling either method while a transaction is open - inside a ``db.atomic()`` block, for example - raises a ``sqlite_utils.db.TransactionError``, unless the database is already in the requested mode in which case the call is a no-op. + You can check the current journal mode for a database using the ``journal_mode`` property: .. code-block:: python @@ -2183,6 +3134,9 @@ You can check the current journal mode for a database using the ``journal_mode`` This will usually be ``wal`` or ``delete`` (meaning WAL is disabled), but can have other values - see the `PRAGMA journal_mode `__ documentation. +.. note:: + In the CLI: :ref:`sqlite-utils enable-wal and disable-wal ` + .. _python_api_suggest_column_types: Suggesting column types @@ -2196,7 +3150,7 @@ That table ``.create()`` method takes a dictionary mapping column names to the P .. code-block:: python - db["cats"].create({ + db.table("cats").create({ "id": int, "name": str, "weight": float, @@ -2234,43 +3188,23 @@ For example: # Create the table db = Database("cats.db") - db["cats"].create(types, pk="id") + db.table("cats").create(types, pk="id") # Insert the records - db["cats"].insert_all(cats) + db.table("cats").insert_all(cats) - # list(db["cats"].rows) now returns: + # list(db.table("cats").rows) now returns: # [{"id": 1, "name": "Snowflake", "age": None, "thumbnail": None} # {"id": 2, "name": "Crabtree", "age": 4, "thumbnail": None}] # The table schema looks like this: - # print(db["cats"].schema) - # CREATE TABLE [cats] ( - # [id] INTEGER PRIMARY KEY, - # [name] TEXT, - # [age] INTEGER, - # [thumbnail] BLOB + # print(db.table("cats").schema) + # CREATE TABLE "cats" ( + # "id" INTEGER PRIMARY KEY, + # "name" TEXT, + # "age" INTEGER, + # "thumbnail" BLOB # ) -.. _find_spatialite: - -Finding SpatiaLite -================== - -The ``find_spatialite()`` function searches for the `SpatiaLite `__ SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. - -You can use it in code like this: - -.. code-block:: python - - from sqlite_utils import Database - from sqlite_utils.utils import find_spatialite - - db = Database("mydb.db") - spatialite = find_spatialite() - if spatialite: - db.conn.enable_load_extension(True) - db.conn.load_extension(spatialite) - .. _python_api_register_function: Registering custom SQL functions @@ -2303,7 +3237,17 @@ You can also use the method as a function decorator like so: print(db.execute('select reverse_string("hello")').fetchone()[0]) -Python 3.8 added the ability to register `deterministic SQLite functions `__, allowing you to indicate that a function will return the exact same result for any given inputs and hence allowing SQLite to apply some performance optimizations. You can mark a function as deterministic using ``deterministic=True``, like this: +By default, the name of the Python function will be used as the name of the SQL function. You can customize this with the ``name=`` keyword argument: + +.. code-block:: python + + @db.register_function(name="rev") + def reverse_string(s): + return "".join(reversed(list(s))) + + print(db.execute('select rev("hello")').fetchone()[0]) + +If a function will return the exact same result for any given inputs you can register it as a `deterministic SQLite function `__ allowing SQLite to apply some performance optimizations: .. code-block:: python @@ -2311,8 +3255,6 @@ Python 3.8 added the ability to register `deterministic SQLite functions ` + .. _python_api_quote: Quoting strings for use in SQL @@ -2351,3 +3296,150 @@ If that option isn't relevant to your use-case you can to quote a string for use "'hello'" >>> db.quote("hello'this'has'quotes") "'hello''this''has''quotes'" + +.. _python_api_rows_from_file: + +Reading rows from a file +======================== + +The ``sqlite_utils.utils.rows_from_file()`` helper function can read rows (a sequence of dictionaries) from CSV, TSV, JSON or newline-delimited JSON files. + +.. autofunction:: sqlite_utils.utils.rows_from_file + :noindex: + +.. _python_api_maximize_csv_field_size_limit: + +Setting the maximum CSV field size limit +======================================== + +Sometimes when working with CSV files that include extremely long fields you may see an error that looks like this:: + + _csv.Error: field larger than field limit (131072) + +The Python standard library ``csv`` module enforces a field size limit. You can increase that limit using the ``csv.field_size_limit(new_limit)`` method (`documented here `__) but if you don't want to pick a new level you may instead want to increase it to the maximum possible. + +The maximum possible value for this is not documented, and varies between systems. + +Calling ``sqlite_utils.utils.maximize_csv_field_size_limit()`` will set the value to the highest possible for the current system: + +.. code-block:: python + + from sqlite_utils.utils import maximize_csv_field_size_limit + + maximize_csv_field_size_limit() + + +If you need to reset to the original value after calling this function you can do so like this: + +.. code-block:: python + + from sqlite_utils.utils import ORIGINAL_CSV_FIELD_SIZE_LIMIT + import csv + + csv.field_size_limit(ORIGINAL_CSV_FIELD_SIZE_LIMIT) + +.. _python_api_typetracker: + +Detecting column types using TypeTracker +======================================== + +Sometimes you may find yourself working with data that lacks type information - data from a CSV file for example. + +The ``TypeTracker`` class can be used to try to automatically identify the most likely types for data that is initially represented as strings. + +Consider this example: + +.. code-block:: python + + import csv, io + + csv_file = io.StringIO("id,name\n1,Cleo\n2,Cardi") + rows = list(csv.DictReader(csv_file)) + + # rows is now this: + # [{'id': '1', 'name': 'Cleo'}, {'id': '2', 'name': 'Cardi'}] + +If we insert this data directly into a table we will get a schema that is entirely ``TEXT`` columns: + +.. code-block:: python + + from sqlite_utils import Database + + db = Database(memory=True) + db.table("creatures").insert_all(rows) + print(db.schema) + # Outputs: + # CREATE TABLE "creatures" ( + # "id" TEXT, + # "name" TEXT + # ); + +We can detect the best column types using a ``TypeTracker`` instance: + +.. code-block:: python + + from sqlite_utils.utils import TypeTracker + + tracker = TypeTracker() + db.table("creatures2").insert_all(tracker.wrap(rows)) + print(tracker.types) + # Outputs {'id': 'integer', 'name': 'text'} + +We can then apply those types to our new table using the :ref:`table.transform() ` method: + +.. code-block:: python + + db.table("creatures2").transform(types=tracker.types) + print(db.table("creatures2").schema) + # Outputs: + # CREATE TABLE "creatures2" ( + # "id" INTEGER, + # "name" TEXT + # ); + +.. _python_api_gis: + +SpatiaLite helpers +================== + +`SpatiaLite `__ is a geographic extension to SQLite (similar to PostgreSQL + PostGIS). Using requires finding, loading and initializing the extension, adding geometry columns to existing tables and optionally creating spatial indexes. The utilities here help streamline that setup. + +.. _python_api_gis_init_spatialite: + +Initialize SpatiaLite +--------------------- + +.. automethod:: sqlite_utils.db.Database.init_spatialite + :noindex: + +.. note:: + In the CLI: :ref:`sqlite-utils create-database --init-spatialite ` + +.. _python_api_gis_find_spatialite: + +Finding SpatiaLite +------------------ + +.. autofunction:: sqlite_utils.utils.find_spatialite + +.. _python_api_gis_add_geometry_column: + +Adding geometry columns +----------------------- + +.. automethod:: sqlite_utils.db.Table.add_geometry_column + :noindex: + +.. note:: + In the CLI: :ref:`sqlite-utils add-geometry-column ` + +.. _python_api_gis_create_spatial_index: + +Creating a spatial index +------------------------ + +.. automethod:: sqlite_utils.db.Table.create_spatial_index + :noindex: + +.. note:: + In the CLI: :ref:`sqlite-utils create-spatial-index ` diff --git a/docs/reference.rst b/docs/reference.rst new file mode 100644 index 0000000..a9fdf29 --- /dev/null +++ b/docs/reference.rst @@ -0,0 +1,117 @@ +.. _reference: + +=============== + API reference +=============== + +.. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here + +.. _reference_db_database: + +sqlite_utils.db.Database +======================== + +.. autoclass:: sqlite_utils.db.Database + :members: + :undoc-members: + :special-members: __getitem__ + :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys + +.. _reference_db_queryable: + +sqlite_utils.db.Queryable +========================= + +:ref:`Table ` and :ref:`View ` are both subclasses of ``Queryable``, providing access to the following methods: + +.. autoclass:: sqlite_utils.db.Queryable + :members: + :undoc-members: + :exclude-members: execute_count + +.. _reference_db_table: + +sqlite_utils.db.Table +===================== + +.. autoclass:: sqlite_utils.db.Table + :members: + :undoc-members: + :show-inheritance: + :exclude-members: guess_foreign_column, value_or_default, build_insert_queries_and_params, insert_chunk, add_missing_columns + +.. _reference_db_view: + +sqlite_utils.db.View +==================== + +.. autoclass:: sqlite_utils.db.View + :members: + :undoc-members: + :show-inheritance: + +.. _reference_db_other: + +Other +===== + +.. _reference_db_other_column: + +sqlite_utils.db.Column +---------------------- + +.. autoclass:: sqlite_utils.db.Column + +.. _reference_db_other_column_details: + +sqlite_utils.db.ColumnDetails +----------------------------- + +.. autoclass:: sqlite_utils.db.ColumnDetails + +.. _reference_db_other_foreign_key: + +sqlite_utils.db.ForeignKey +-------------------------- + +.. autoclass:: sqlite_utils.db.ForeignKey + +sqlite_utils.utils +================== + +.. _reference_utils_hash_record: + +sqlite_utils.utils.hash_record +------------------------------ + +.. autofunction:: sqlite_utils.utils.hash_record + +.. _reference_utils_rows_from_file: + +sqlite_utils.utils.rows_from_file +--------------------------------- + +.. autofunction:: sqlite_utils.utils.rows_from_file + +.. _reference_utils_typetracker: + +sqlite_utils.utils.TypeTracker +------------------------------ + +.. autoclass:: sqlite_utils.utils.TypeTracker + :members: wrap, types + +.. _reference_utils_chunks: + +sqlite_utils.utils.chunks +------------------------- + +.. autofunction:: sqlite_utils.utils.chunks + +.. _reference_utils_flatten: + +sqlite_utils.utils.flatten +-------------------------- + +.. autofunction:: sqlite_utils.utils.flatten diff --git a/docs/tutorial.ipynb b/docs/tutorial.ipynb index aa22461..bade686 100644 --- a/docs/tutorial.ipynb +++ b/docs/tutorial.ipynb @@ -39,10 +39,8 @@ "Requirement already satisfied: sqlite-fts4 in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (1.0.1)\n", "Requirement already satisfied: click in /Users/simon/Library/Python/3.9/lib/python/site-packages (from sqlite_utils) (7.1.2)\n", "Requirement already satisfied: tabulate in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (0.8.7)\n", - "Requirement already satisfied: dateutils in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from sqlite_utils) (0.6.12)\n", - "Requirement already satisfied: python-dateutil in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2.8.1)\n", - "Requirement already satisfied: pytz in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2021.1)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from python-dateutil->dateutils->sqlite_utils) (1.16.0)\n", + "Requirement already satisfied: python-dateutil in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-package (from sqlite-utils) (2.8.1)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-package (from python-dateutil->sqlite-utils) (1.16.0)\n", "\u001b[33mWARNING: You are using pip version 21.1.1; however, version 21.2.2 is available.\n", "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n" @@ -182,10 +180,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "CREATE TABLE [creatures] (\n", - " [name] TEXT,\n", - " [species] TEXT,\n", - " [age] FLOAT\n", + "CREATE TABLE \"creatures\" (\n", + " \"name\" TEXT,\n", + " \"species\" TEXT,\n", + " \"age\" FLOAT\n", ")\n" ] } @@ -536,11 +534,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "CREATE TABLE [creatures] (\n", - " [id] INTEGER PRIMARY KEY,\n", - " [name] TEXT,\n", - " [species] TEXT,\n", - " [age] FLOAT\n", + "CREATE TABLE \"creatures\" (\n", + " \"id\" INTEGER PRIMARY KEY,\n", + " \"name\" TEXT,\n", + " \"species\" TEXT,\n", + " \"age\" FLOAT\n", ")\n" ] } @@ -931,11 +929,11 @@ "output_type": "stream", "text": [ "CREATE TABLE \"creatures\" (\n", - " [id] INTEGER PRIMARY KEY,\n", - " [name] TEXT,\n", - " [species_id] INTEGER,\n", - " [age] FLOAT,\n", - " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n", + " \"id\" INTEGER PRIMARY KEY,\n", + " \"name\" TEXT,\n", + " \"species_id\" INTEGER,\n", + " \"age\" FLOAT,\n", + " FOREIGN KEY(\"species_id\") REFERENCES \"species\"(\"id\")\n", ")\n", "[{'id': 1, 'name': 'Cleo', 'species_id': 1, 'age': 6.0}, {'id': 2, 'name': 'Lila', 'species_id': 2, 'age': 0.8}, {'id': 3, 'name': 'Bants', 'species_id': 2, 'age': 0.8}, {'id': 4, 'name': 'Azi', 'species_id': 2, 'age': 0.8}, {'id': 5, 'name': 'Snowy', 'species_id': 2, 'age': 0.9}, {'id': 6, 'name': 'Blue', 'species_id': 2, 'age': 0.9}]\n" ] @@ -964,9 +962,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "CREATE TABLE [species] (\n", - " [id] INTEGER PRIMARY KEY,\n", - " [species] TEXT\n", + "CREATE TABLE \"species\" (\n", + " \"id\" INTEGER PRIMARY KEY,\n", + " \"species\" TEXT\n", ")\n", "[{'id': 1, 'species': 'dog'}, {'id': 2, 'species': 'chicken'}]\n" ] @@ -1050,4 +1048,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/docs/upgrading.rst b/docs/upgrading.rst new file mode 100644 index 0000000..92b582a --- /dev/null +++ b/docs/upgrading.rst @@ -0,0 +1,146 @@ +.. _upgrading: + +=========== + Upgrading +=========== + +This page describes the changes you may need to make to your own code or scripts when upgrading between major versions of ``sqlite-utils``. + +For the full list of changes in every release see the :ref:`changelog`. + +.. _upgrading_3_to_4: + +Upgrading from 3.x to 4.0 +========================= + +Requirements +------------ + +- Python 3.10 or higher is required. +- The ``click`` dependency must be version 8.3.1 or later. + +Command-line changes +-------------------- + +**Type detection is now the default for CSV and TSV imports.** ``sqlite-utils insert`` and ``sqlite-utils upsert`` now detect column types when importing CSV or TSV data - previously every column was created as ``TEXT`` unless you passed ``--detect-types``. To restore the old behavior pass the new ``--no-detect-types`` flag: + +.. code-block:: bash + + sqlite-utils insert data.db rows data.csv --csv --no-detect-types + +Two related things have been removed: + +- The ``SQLITE_UTILS_DETECT_TYPES`` environment variable. +- The old ``-d/--detect-types`` flag itself. Since detection is now the default the flag did nothing - remove it from any scripts that used it. + +**The convert command no longer skips falsey values.** ``sqlite-utils convert`` previously skipped values that evaluated to ``False`` (empty strings, ``0``) unless you passed ``--no-skip-false``. All values are now converted and the ``--no-skip-false`` flag has been removed. + +**drop-table and drop-view check the object type.** ``sqlite-utils drop-table`` now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. If you relied on that (unlikely), use the matching command instead. + +**sqlite-utils tui has moved to a plugin.** The optional terminal interface is now provided by the `sqlite-utils-tui `__ plugin: + +.. code-block:: bash + + sqlite-utils install sqlite-utils-tui + +Python API changes +------------------ + +**db.query() now rejects SQL that does not return rows.** This is likely the most common change you will need to make to existing code. ``db.query()`` used to accept any SQL statement - passing one that returns no rows, such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause or a ``CREATE TABLE``, did nothing at all, silently. Those statements now raise a ``ValueError``, and are rolled back so they have no effect on the database. Transaction control statements (``BEGIN``, ``COMMIT``, ``END``, ``ROLLBACK``, ``SAVEPOINT``, ``RELEASE``) plus ``VACUUM``, ``ATTACH`` and ``DETACH`` are also rejected with a ``ValueError``, without being executed at all. Use ``db.execute()`` for statements that do not return rows: + +.. code-block:: python + + # 3.x accepted this but silently did nothing: + db.query("update dogs set name = 'Cleopaws'") + + # In 4.0 use execute() for SQL that does not return rows: + db.execute("update dogs set name = 'Cleopaws'") + +**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily, but errors in your SQL raise at the ``db.query()`` call site rather than on first iteration, and a write with a ``RETURNING`` clause takes effect even if you never iterate over its results. + +**db.table() no longer returns views.** ``db.table(name)`` now raises a ``sqlite_utils.db.NoTable`` exception if ``name`` is a SQL view. Use the new ``db.view(name)`` method for views: + +.. code-block:: python + + table = db.table("my_table") + view = db.view("my_view") + +``db["name"]`` still returns either a ``Table`` or a ``View`` depending on what exists in the database. + +**Upserts use INSERT ... ON CONFLICT.** Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax rather than the previous ``INSERT OR IGNORE`` followed by ``UPDATE``. If your code depends on the old behavior, pass ``use_old_upsert=True`` to the ``Database()`` constructor - see :ref:`python_api_old_upsert`. + +**Upsert records must include their primary keys.** ``table.upsert()`` and ``table.upsert_all()`` now raise ``sqlite_utils.db.PrimaryKeyRequired`` if a record is missing a value for any primary key column (or has ``None`` for one). Previously such records were quietly inserted as new rows. Relatedly, ``pk=`` is now optional when the table already exists with a primary key - it is detected automatically. + +**Floating point columns are now REAL.** Auto-detected floating point columns are created with the correct SQLite type ``REAL`` instead of ``FLOAT``. Code that inspects column types should expect ``REAL``. + +**Generated schemas use double quotes.** Tables created by this library now wrap table and column names in ``"double-quotes"`` where they previously used ``[square-braces]``. If you compare ``table.schema`` strings against expected values you will need to update them. + +**table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values. + +**Null values are no longer extracted into lookup tables.** ``table.extract()`` and the ``sqlite-utils extract`` command leave rows alone if every extracted column is ``null`` - the new foreign key column is left as ``null`` instead of pointing at an all-``null`` record in the lookup table. The ``extracts=`` insert option similarly keeps ``None`` values as ``null``. Relatedly, ``table.lookup()`` now compares values using ``IS`` so that looking up a value containing ``None`` returns the existing matching row - previously it inserted a duplicate row on every call. + +**ensure_autocommit_off() is now ensure_autocommit_on().** The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``. The old name described the opposite of what the method did: it temporarily puts the connection into driver-level autocommit mode (by setting ``isolation_level = None``), so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. The behavior is unchanged - update any calls to use the new name. + +**View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. + +**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained new fields - ``columns``, ``other_columns``, ``is_compound``, ``on_delete`` and ``on_update`` - so that compound (multi-column) foreign keys and foreign key actions can be represented. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: + +.. code-block:: python + + # 3.x - tuple unpacking, no longer works: + for table, column, other_table, other_column in db["courses"].foreign_keys: + ... + + # 4.0 - access fields by name: + for fk in db["courses"].foreign_keys: + fk.table, fk.column, fk.other_table, fk.other_column + +Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. Like the old namedtuple, ``ForeignKey`` instances are immutable and hashable - they can be collected into sets and used as dictionary keys. Note that equality now includes the ``on_delete`` and ``on_update`` actions: a ``ForeignKey`` with ``ON DELETE CASCADE`` is not equal to one without. + +Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` tuples instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are one-item tuples. + +Two related behavior changes to ``table.transform()``: compound foreign keys now survive a transform (previously they were split into separate single-column keys), and ``ON DELETE``/``ON UPDATE`` actions such as ``ON DELETE CASCADE`` are now preserved (previously they were silently stripped from the schema). + +**Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``. + +**Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: + +- Write statements executed with raw ``db.execute()`` calls now commit automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that nothing committed - if your code used ``db.execute()`` for writes and relied on ``db.conn.rollback()`` to undo them, open an explicit transaction with the new ``db.begin()`` method first. +- Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead. +- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, instead of silently committing it. +- Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - a transaction you explicitly opened with ``db.begin()`` and did not commit is rolled back. +- ``Database()`` rejects connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options, raising ``sqlite_utils.db.TransactionError``. On those connections every write the library made was silently discarded when the connection closed. + +Packaging changes +----------------- + +- ``sqlite-utils`` now uses ``pyproject.toml`` in place of ``setup.py``. +- ``pip`` is now a runtime dependency, used by the ``sqlite-utils install`` and ``uninstall`` commands. + +New features to be aware of +--------------------------- + +Not breaking changes, but new in 4.0 and worth knowing about when you upgrade: + +- A :ref:`database migrations system `, incorporating the functionality of the ``sqlite-migrate`` plugin. If you used that plugin, the built-in system reads the same ``_sqlite_migrations`` table - your applied migrations will not run again. Update your migration files to use ``from sqlite_utils import Migrations``. +- :ref:`db.atomic() ` for nested transaction support. +- ``table.insert_all()`` and ``table.upsert_all()`` accept an iterator of lists or tuples as an alternative to dictionaries - see :ref:`python_api_insert_lists`. + +.. _upgrading_2_to_3: + +Upgrading from 2.x to 3.0 +========================= + +The 3.0 release redesigned search. The breaking changes were minor: + +- ``table.search()`` returns a generator of dictionaries, sorted by relevance. It previously returned a list of tuples sorted by ``rowid``. +- The ``-c`` shortcut for ``--csv`` and the ``-f`` shortcut for ``--fmt`` were removed from the CLI - use the full option names. + +.. _upgrading_1_to_2: + +Upgrading from 1.x to 2.0 +========================= + +The 2.0 release changed the meaning of *upsert*. In 1.x, ``table.upsert()`` and ``table.upsert_all()`` actually performed ``INSERT OR REPLACE`` operations - entirely replacing the existing row. Since 2.0 an upsert updates only the columns you provide, leaving other columns untouched. + +If you want the 1.x behavior, use ``table.insert(..., replace=True)`` or ``table.insert_all(..., replace=True)`` instead. diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..2f6a875 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,32 @@ +[mypy] +python_version = 3.10 +warn_return_any = False +warn_unused_configs = True +warn_redundant_casts = False +warn_unused_ignores = False +check_untyped_defs = True +disallow_untyped_defs = False +disallow_incomplete_defs = False +no_implicit_optional = True +strict_equality = True + +[mypy-sqlite_utils.cli] +ignore_errors = True + +[mypy-pysqlite3.*] +ignore_missing_imports = True + +[mypy-sqlite_dump.*] +ignore_missing_imports = True + +[mypy-sqlite_fts4.*] +ignore_missing_imports = True + +[mypy-pandas.*] +ignore_missing_imports = True + +[mypy-numpy.*] +ignore_missing_imports = True + +[mypy-tests.*] +ignore_errors = True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..003322c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,85 @@ +[project] +name = "sqlite-utils" +version = "4.1.1" +description = "CLI tool and Python library for manipulating SQLite databases" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "Simon Willison" }, +] +license = "Apache-2.0" +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Database", +] + +dependencies = [ + "click>=8.3.1", + "click-default-group>=1.2.3", + "pluggy", + "python-dateutil", + "sqlite-fts4", + "tabulate", + "pip", +] + +[dependency-groups] +dev = [ + "black>=26.3.1", + "click>=8.4.2", + "cogapp", + "hypothesis", + "pytest", + # mypy + "data-science-types", + "mypy", + "types-click", + "types-pluggy", + "types-python-dateutil", + "types-tabulate", + # flake8 + "flake8", + "flake8-pyproject", + "ty>=0.0.37", + # For stable cog: + "tabulate>=0.10.0", +] +docs = [ + "codespell", + "furo", + "pygments-csv-lexer", + "sphinx-autobuild", + "sphinx-copybutton", +] + +[project.urls] +Homepage = "https://github.com/simonw/sqlite-utils" +Documentation = "https://sqlite-utils.datasette.io/en/stable/" +Changelog = "https://sqlite-utils.datasette.io/en/stable/changelog.html" +Issues = "https://github.com/simonw/sqlite-utils/issues" +CI = "https://github.com/simonw/sqlite-utils/actions" + +[project.scripts] +sqlite-utils = "sqlite_utils.cli:cli" + +[build-system] +# setuptools 77+ is needed for the PEP 639 license = "Apache-2.0" expression +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[tool.flake8] +max-line-length = 160 +# Black compatibility, E203 whitespace before ':': +extend-ignore = ["E203"] +extend-exclude = [".venv", "build", "dist", "docs", "sqlite_utils.egg-info"] + +[tool.setuptools.package-data] +sqlite_utils = ["py.typed"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 7a88d6f..0000000 --- a/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -max-line-length = 160 -extend-ignore = E203 # for Black diff --git a/setup.py b/setup.py deleted file mode 100644 index c8c58ac..0000000 --- a/setup.py +++ /dev/null @@ -1,65 +0,0 @@ -from setuptools import setup, find_packages -import io -import os - -VERSION = "3.15" - - -def get_long_description(): - with io.open( - os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"), - encoding="utf8", - ) as fp: - return fp.read() - - -setup( - name="sqlite-utils", - description="CLI tool and Python utility functions for manipulating SQLite databases", - long_description=get_long_description(), - long_description_content_type="text/markdown", - author="Simon Willison", - version=VERSION, - license="Apache License, Version 2.0", - packages=find_packages(exclude=["tests", "tests.*"]), - install_requires=[ - "sqlite-fts4", - "click", - "click-default-group", - "tabulate", - "dateutils", - ], - setup_requires=["pytest-runner"], - extras_require={ - "test": ["pytest", "black", "hypothesis"], - "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], - "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"], - "flake8": ["flake8"], - }, - entry_points=""" - [console_scripts] - sqlite-utils=sqlite_utils.cli:cli - """, - tests_require=["sqlite-utils[test]"], - url="https://github.com/simonw/sqlite-utils", - project_urls={ - "Documentation": "https://sqlite-utils.datasette.io/en/stable/", - "Changelog": "https://sqlite-utils.datasette.io/en/stable/changelog.html", - "Source code": "https://github.com/simonw/sqlite-utils", - "Issues": "https://github.com/simonw/sqlite-utils/issues", - "CI": "https://github.com/simonw/sqlite-utils/actions", - }, - python_requires=">=3.6", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "Intended Audience :: End Users/Desktop", - "Topic :: Database", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - ], -) diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index 614f984..58ee7ab 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -1,4 +1,7 @@ -from .db import Database from .utils import suggest_column_types +from .hookspecs import hookimpl +from .hookspecs import hookspec +from .db import Database +from .migrations import Migrations -__all__ = ["Database", "suggest_column_types"] +__all__ = ["Database", "Migrations", "suggest_column_types", "hookimpl", "hookspec"] diff --git a/sqlite_utils/__main__.py b/sqlite_utils/__main__.py new file mode 100644 index 0000000..98dcca0 --- /dev/null +++ b/sqlite_utils/__main__.py @@ -0,0 +1,4 @@ +from .cli import cli + +if __name__ == "__main__": + cli() diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index d94229d..e0b8969 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,11 +1,26 @@ import base64 +import difflib +from typing import Any import click -from click_default_group import DefaultGroup # type: ignore -from datetime import datetime +from click_default_group import DefaultGroup +from datetime import datetime, timezone import hashlib import pathlib +from runpy import run_module import sqlite_utils -from sqlite_utils.db import AlterError, BadMultiValues, DescIndex +from sqlite_utils.db import ( + AlterError, + BadMultiValues, + DEFAULT, + DescIndex, + InvalidColumns, + NoTable, + NoView, + PrimaryKeyRequired, + quote_identifier, +) +from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins +from sqlite_utils.utils import maximize_csv_field_size_limit from sqlite_utils import recipes import textwrap import inspect @@ -13,12 +28,18 @@ import io import itertools import json import os +import pdb import sys import csv as csv_std import tabulate from .utils import ( + OperationalError, + _compile_code, + chunks, + dedupe_keys, file_progress, find_spatialite, + flatten as _flatten, sqlite3, decode_base64_values, progressbar, @@ -29,7 +50,28 @@ from .utils import ( CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) -VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") + +def _register_db_for_cleanup(db): + """Register a database to be closed when the Click context is cleaned up.""" + ctx = click.get_current_context(silent=True) + if ctx is None: + return + if "_databases_to_close" not in ctx.meta: + ctx.meta["_databases_to_close"] = [] + ctx.call_on_close(lambda: _close_databases(ctx)) + ctx.meta["_databases_to_close"].append(db) + + +def _close_databases(ctx): + """Close all databases registered for cleanup.""" + for db in ctx.meta.get("_databases_to_close", []): + try: + db.close() + except Exception: + pass + + +VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "REAL", "BLOB") UNICODE_ERROR = """ {} @@ -43,17 +85,15 @@ If you do not know the encoding, running 'file filename.csv' may tell you. It's often worth trying: --encoding=latin-1 """.strip() +maximize_csv_field_size_limit() -# Increase CSV field size limit to maximum possible -# https://stackoverflow.com/a/15063941 -field_size_limit = sys.maxsize -while True: - try: - csv_std.field_size_limit(field_size_limit) - break - except OverflowError: - field_size_limit = int(field_size_limit / 10) +class CaseInsensitiveChoice(click.Choice): + def __init__(self, choices): + super().__init__([choice.lower() for choice in choices]) + + def convert(self, value, param, ctx): + return super().convert(value.lower(), param, ctx) def output_options(fn): @@ -73,14 +113,19 @@ def output_options(fn): ), click.option("--csv", is_flag=True, help="Output CSV"), click.option("--tsv", is_flag=True, help="Output TSV"), - click.option("--no-headers", is_flag=True, help="Omit CSV headers"), - click.option("-t", "--table", is_flag=True, help="Output as a table"), + click.option( + "--no-headers", + is_flag=True, + help="Omit headers from CSV/TSV and table/--fmt output", + ), + click.option( + "-t", "--table", is_flag=True, help="Output as a formatted table" + ), click.option( "--fmt", help="Table format - one of {}".format( ", ".join(tabulate.tabulate_formats) ), - default="simple", ), click.option( "--json-cols", @@ -88,6 +133,13 @@ def output_options(fn): is_flag=True, default=False, ), + click.option( + "--ascii", + "ascii_", + help="Escape non-ASCII characters in JSON output as \\uXXXX", + is_flag=True, + default=False, + ), ) ): fn = decorator(fn) @@ -98,7 +150,18 @@ def load_extension_option(fn): return click.option( "--load-extension", multiple=True, - help="SQLite extensions to load", + help="Path to SQLite extension, with optional :entrypoint", + )(fn) + + +def functions_option(fn): + return click.option( + "--functions", + help=( + "Python code or a file path defining custom SQL functions; " + "can be used multiple times" + ), + multiple=True, )(fn) @@ -156,13 +219,21 @@ def tables( table, fmt, json_cols, + ascii_, columns, schema, load_extension, views=False, ): - """List the tables in the database""" + """List the tables in the database + + Example: + + \b + sqlite-utils tables trees.db + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) headers = ["view" if views else "table"] if counts: @@ -172,27 +243,35 @@ def tables( if schema: headers.append("schema") + method = db.view if views else db.table + def _iter(): if views: items = db.view_names() else: items = db.table_names(fts4=fts4, fts5=fts5) for name in items: - row = [name] + row: list[Any] = [name] if counts: - row.append(db[name].count) + row.append(method(name).count) if columns: - cols = [c.name for c in db[name].columns] + cols = [c.name for c in method(name).columns] if csv: row.append("\n".join(cols)) else: row.append(cols) if schema: - row.append(db[name].schema) + row.append(method(name).schema) yield row - if table: - print(tabulate.tabulate(_iter(), headers=headers, tablefmt=fmt)) + if table or fmt: + print( + tabulate.tabulate( + _iter(), + headers=() if no_headers else headers, + tablefmt=fmt or "simple", + ) + ) elif csv or tsv: writer = csv_std.writer(sys.stdout, dialect="excel-tab" if tsv else "excel") if not no_headers: @@ -200,7 +279,7 @@ def tables( for row in _iter(): writer.writerow(row) else: - for line in output_rows(_iter(), headers, nl, arrays, json_cols): + for line in output_rows(_iter(), headers, nl, arrays, json_cols, ascii_): click.echo(line) @@ -238,11 +317,19 @@ def views( table, fmt, json_cols, + ascii_, columns, schema, load_extension, ): - """List the views in the database""" + """List the views in the database + + Example: + + \b + sqlite-utils views trees.db + """ + assert tables.callback is not None tables.callback( path=path, fts4=False, @@ -256,6 +343,7 @@ def views( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, columns=columns, schema=schema, load_extension=load_extension, @@ -273,14 +361,21 @@ def views( @click.option("--no-vacuum", help="Don't run VACUUM", default=False, is_flag=True) @load_extension_option def optimize(path, tables, no_vacuum, load_extension): - """Optimize all full-text search tables and then run VACUUM - should shrink the database file""" + """Optimize all full-text search tables and then run VACUUM - should shrink the database file + + Example: + + \b + sqlite-utils optimize chickens.db + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) if not tables: tables = db.table_names(fts4=True) + db.table_names(fts5=True) with db.conn: for table in tables: - db[table].optimize() + db.table(table).optimize() if not no_vacuum: db.vacuum() @@ -294,14 +389,48 @@ def optimize(path, tables, no_vacuum, load_extension): @click.argument("tables", nargs=-1) @load_extension_option def rebuild_fts(path, tables, load_extension): - """Rebuild all or specific full-text search tables""" + """Rebuild all or specific full-text search tables + + Example: + + \b + sqlite-utils rebuild-fts chickens.db chickens + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) if not tables: tables = db.table_names(fts4=True) + db.table_names(fts5=True) with db.conn: for table in tables: - db[table].rebuild_fts() + db.table(table).rebuild_fts() + + +@cli.command() +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("names", nargs=-1) +def analyze(path, names): + """Run ANALYZE against the whole database, or against specific named indexes and tables + + Example: + + \b + sqlite-utils analyze chickens.db + """ + db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) + try: + if names: + for name in names: + db.analyze(name) + else: + db.analyze() + except OperationalError as e: + raise click.ClickException(str(e)) @cli.command() @@ -311,8 +440,16 @@ def rebuild_fts(path, tables, load_extension): required=True, ) def vacuum(path): - """Run VACUUM against the database""" - sqlite_utils.Database(path).vacuum() + """Run VACUUM against the database + + Example: + + \b + sqlite-utils vacuum chickens.db + """ + db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) + db.vacuum() @cli.command() @@ -323,10 +460,17 @@ def vacuum(path): ) @load_extension_option def dump(path, load_extension): - """Output a SQL dump of the schema and full contents of the database""" + """Output a SQL dump of the schema and full contents of the database + + Example: + + \b + sqlite-utils dump chickens.db + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) - for line in db.conn.iterdump(): + for line in db.iterdump(): click.echo(line) @@ -341,7 +485,8 @@ def dump(path, load_extension): @click.argument( "col_type", type=click.Choice( - ["integer", "float", "blob", "text", "INTEGER", "FLOAT", "BLOB", "TEXT"] + ["integer", "int", "float", "real", "text", "str", "blob", "bytes"], + case_sensitive=False, ), required=False, ) @@ -360,16 +505,40 @@ def dump(path, load_extension): required=False, help="Add NOT NULL DEFAULT 'TEXT' constraint", ) +@click.option( + "--ignore", + is_flag=True, + help="If column already exists, do nothing", +) @load_extension_option def add_column( - path, table, col_name, col_type, fk, fk_col, not_null_default, load_extension + path, + table, + col_name, + col_type, + fk, + fk_col, + not_null_default, + ignore, + load_extension, ): - "Add a column to the specified table" + """Add a column to the specified table + + Example: + + \b + sqlite-utils add-column chickens.db chickens weight float + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) - db[table].add_column( - col_name, col_type, fk=fk, fk_col=fk_col, not_null_default=not_null_default - ) + try: + db.table(table).add_column( + col_name, col_type, fk=fk, fk_col=fk_col, not_null_default=not_null_default + ) + except OperationalError as ex: + if not ignore: + raise click.ClickException(str(ex)) @cli.command(name="add-foreign-key") @@ -392,18 +561,21 @@ def add_foreign_key( path, table, column, other_table, other_column, ignore, load_extension ): """ - Add a new foreign key constraint to an existing table. Example usage: + Add a new foreign key constraint to an existing table - $ sqlite-utils add-foreign-key my.db books author_id authors id + Example: - WARNING: Could corrupt your database! Back up your database file first. + sqlite-utils add-foreign-key my.db books author_id authors id """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) try: - db[table].add_foreign_key(column, other_table, other_column, ignore=ignore) + db.table(table).add_foreign_key( + column, other_table, other_column, ignore=ignore + ) except AlterError as e: - raise click.ClickException(e) + raise click.ClickException(str(e)) @cli.command(name="add-foreign-keys") @@ -416,14 +588,17 @@ def add_foreign_key( @load_extension_option def add_foreign_keys(path, foreign_key, load_extension): """ - Add multiple new foreign key constraints to a database. Example usage: + Add multiple new foreign key constraints to a database + + Example: \b - sqlite-utils add-foreign-keys my.db \\ - books author_id authors id \\ - authors country_id countries id + sqlite-utils add-foreign-keys my.db \\ + books author_id authors id \\ + authors country_id countries id """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) if len(foreign_key) % 4 != 0: raise click.ClickException( @@ -435,7 +610,7 @@ def add_foreign_keys(path, foreign_key, load_extension): try: db.add_foreign_keys(tuples) except AlterError as e: - raise click.ClickException(e) + raise click.ClickException(str(e)) @cli.command(name="index-foreign-keys") @@ -447,9 +622,15 @@ def add_foreign_keys(path, foreign_key, load_extension): @load_extension_option def index_foreign_keys(path, load_extension): """ - Ensure every foreign key column has an index on it. + Ensure every foreign key column has an index on it + + Example: + + \b + sqlite-utils index-foreign-keys chickens.db """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) db.index_foreign_keys() @@ -466,18 +647,35 @@ def index_foreign_keys(path, load_extension): @click.option("--unique", help="Make this a unique index", default=False, is_flag=True) @click.option( "--if-not-exists", + "--ignore", help="Ignore if index already exists", default=False, is_flag=True, ) +@click.option( + "--analyze", + help="Run ANALYZE after creating the index", + is_flag=True, +) @load_extension_option -def create_index(path, table, column, name, unique, if_not_exists, load_extension): +def create_index( + path, table, column, name, unique, if_not_exists, analyze, load_extension +): """ - Add an index to the specified table covering the specified columns. - Use "sqlite-utils create-index mydb -- -column" to specify descending - order for a column. + Add an index to the specified table for the specified columns + + Example: + + \b + sqlite-utils create-index chickens.db chickens name + + To create an index in descending order: + + \b + sqlite-utils create-index chickens.db chickens -- -name """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) # Treat -prefix as descending for columns columns = [] @@ -485,11 +683,43 @@ def create_index(path, table, column, name, unique, if_not_exists, load_extensio if col.startswith("-"): col = DescIndex(col[1:]) columns.append(col) - db[table].create_index( - columns, index_name=name, unique=unique, if_not_exists=if_not_exists + db.table(table).create_index( + columns, + index_name=name, + unique=unique, + if_not_exists=if_not_exists, + analyze=analyze, ) +@cli.command(name="drop-index") +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.argument("index") +@click.option("--ignore", help="Ignore if index does not exist", is_flag=True) +@load_extension_option +def drop_index(path, table, index, ignore, load_extension): + """ + Drop an index by index name from the specified table + + Example: + + \b + sqlite-utils drop-index chickens.db chickens idx_chickens_name + """ + db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) + _load_extensions(db, load_extension) + try: + db.table(table).drop_index(index, ignore=ignore) + except OperationalError as ex: + raise click.ClickException(str(ex)) + + @cli.command(name="enable-fts") @click.argument( "path", @@ -507,11 +737,22 @@ def create_index(path, table, column, name, unique, if_not_exists, load_extensio default=False, is_flag=True, ) +@click.option( + "--replace", + is_flag=True, + help="Replace existing FTS configuration if it exists", +) @load_extension_option def enable_fts( - path, table, column, fts4, fts5, tokenize, create_triggers, load_extension + path, table, column, fts4, fts5, tokenize, create_triggers, replace, load_extension ): - "Enable full-text search for specific table and columns" + """Enable full-text search for specific table and columns + + Example: + + \b + sqlite-utils enable-fts chickens.db chickens name + """ fts_version = "FTS5" if fts4 and fts5: click.echo("Can only use one of --fts4 or --fts5", err=True) @@ -520,13 +761,18 @@ def enable_fts( fts_version = "FTS4" db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) - db[table].enable_fts( - column, - fts_version=fts_version, - tokenize=tokenize, - create_triggers=create_triggers, - ) + try: + db.table(table).enable_fts( + column, + fts_version=fts_version, + tokenize=tokenize, + create_triggers=create_triggers, + replace=replace, + ) + except (NoTable, OperationalError) as ex: + raise click.ClickException(str(ex)) @cli.command(name="populate-fts") @@ -539,10 +785,17 @@ def enable_fts( @click.argument("column", nargs=-1, required=True) @load_extension_option def populate_fts(path, table, column, load_extension): - "Re-populate full-text search for specific table and columns" + """Re-populate full-text search for specific table and columns + + Example: + + \b + sqlite-utils populate-fts chickens.db chickens name + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) - db[table].populate_fts(column) + db.table(table).populate_fts(column) @cli.command(name="disable-fts") @@ -554,10 +807,17 @@ def populate_fts(path, table, column, load_extension): @click.argument("table") @load_extension_option def disable_fts(path, table, load_extension): - "Disable full-text search for specific table" + """Disable full-text search for specific table + + Example: + + \b + sqlite-utils disable-fts chickens.db chickens + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) - db[table].disable_fts() + db.table(table).disable_fts() @cli.command(name="enable-wal") @@ -569,9 +829,16 @@ def disable_fts(path, table, load_extension): ) @load_extension_option def enable_wal(path, load_extension): - "Enable WAL for database files" + """Enable WAL for database files + + Example: + + \b + sqlite-utils enable-wal chickens.db + """ for path_ in path: db = sqlite_utils.Database(path_) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) db.enable_wal() @@ -585,9 +852,16 @@ def enable_wal(path, load_extension): ) @load_extension_option def disable_wal(path, load_extension): - "Disable WAL for database files" + """Disable WAL for database files + + Example: + + \b + sqlite-utils disable-wal chickens.db + """ for path_ in path: db = sqlite_utils.Database(path_) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) db.disable_wal() @@ -601,8 +875,15 @@ def disable_wal(path, load_extension): @click.argument("tables", nargs=-1) @load_extension_option def enable_counts(path, tables, load_extension): - "Configure triggers to update a _counts table with row counts" + """Configure triggers to update a _counts table with row counts + + Example: + + \b + sqlite-utils enable-counts chickens.db + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) if not tables: db.enable_counts() @@ -612,7 +893,7 @@ def enable_counts(path, tables, load_extension): if bad_tables: raise click.ClickException("Invalid tables: {}".format(bad_tables)) for table in tables: - db[table].enable_counts() + db.table(table).enable_counts() @cli.command(name="reset-counts") @@ -623,89 +904,167 @@ def enable_counts(path, tables, load_extension): ) @load_extension_option def reset_counts(path, load_extension): - "Reset calculated counts in the _counts table" + """Reset calculated counts in the _counts table + + Example: + + \b + sqlite-utils reset-counts chickens.db + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) db.reset_counts() -def insert_upsert_options(fn): - for decorator in reversed( - ( - click.argument( - "path", - type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), - required=True, - ), - click.argument("table"), - click.argument("json_file", type=click.File("rb"), required=True), - click.option( - "--pk", help="Columns to use as the primary key, e.g. id", multiple=True - ), - click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), - click.option("--flatten", is_flag=True, help="Flatten nested JSON objects"), - click.option("-c", "--csv", is_flag=True, help="Expect CSV"), - click.option("--tsv", is_flag=True, help="Expect TSV"), - click.option("--delimiter", help="Delimiter to use for CSV files"), - click.option("--quotechar", help="Quote character to use for CSV/TSV"), - click.option( - "--sniff", is_flag=True, help="Detect delimiter and quote character" - ), - click.option( - "--no-headers", is_flag=True, help="CSV file has no header row" - ), - click.option( - "--batch-size", type=int, default=100, help="Commit every X records" - ), - click.option( - "--alter", - is_flag=True, - help="Alter existing table to add any missing columns", - ), - click.option( - "--not-null", - multiple=True, - help="Columns that should be created as NOT NULL", - ), - click.option( - "--default", - multiple=True, - type=(str, str), - help="Default value that should be set for a column", - ), - click.option( - "--encoding", - help="Character encoding for input, defaults to utf-8", - ), - click.option( - "-d", - "--detect-types", - is_flag=True, - envvar="SQLITE_UTILS_DETECT_TYPES", - help="Detect types for columns in CSV/TSV data", - ), - load_extension_option, - click.option("--silent", is_flag=True, help="Do not show progress bar"), - ) - ): +_import_options = ( + click.option( + "--flatten", + is_flag=True, + help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}', + ), + click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), + click.option("-c", "--csv", is_flag=True, help="Expect CSV input"), + click.option("--tsv", is_flag=True, help="Expect TSV input"), + click.option("--empty-null", is_flag=True, help="Treat empty strings as NULL"), + click.option( + "--lines", + is_flag=True, + help="Treat each line as a single value called 'line'", + ), + click.option( + "--text", + is_flag=True, + help="Treat input as a single value called 'text'", + ), + click.option("--convert", help="Python code to convert each item"), + click.option( + "--import", + "imports", + type=str, + multiple=True, + help="Python modules to import", + ), + click.option("--delimiter", help="Delimiter to use for CSV files"), + click.option("--quotechar", help="Quote character to use for CSV/TSV"), + click.option("--sniff", is_flag=True, help="Detect delimiter and quote character"), + click.option("--no-headers", is_flag=True, help="CSV file has no header row"), + click.option( + "--encoding", + help="Character encoding for input, defaults to utf-8", + ), +) + + +def import_options(fn): + for decorator in reversed(_import_options): fn = decorator(fn) return fn +def insert_upsert_options(*, require_pk=False): + def inner(fn): + for decorator in reversed( + ( + click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, + ), + click.argument("table"), + click.argument( + "file", type=click.File("rb", lazy=True), required=False + ), + click.option( + "--pk", + help="Columns to use as the primary key, e.g. id", + multiple=True, + required=require_pk, + ), + click.option( + "--code", + help="Python code defining a rows() function or iterable of rows to insert", + ), + ) + + _import_options + + ( + click.option( + "--batch-size", type=int, default=100, help="Commit every X records" + ), + click.option("--stop-after", type=int, help="Stop after X records"), + click.option( + "--alter", + is_flag=True, + help="Alter existing table to add any missing columns", + ), + click.option( + "--not-null", + multiple=True, + help="Columns that should be created as NOT NULL", + ), + click.option( + "--default", + multiple=True, + type=(str, str), + help="Default value that should be set for a column", + ), + click.option( + "--type", + "types", + type=( + str, + click.Choice(list(VALID_COLUMN_TYPES), case_sensitive=False), + ), + multiple=True, + help="Column types to use when creating the table", + ), + click.option( + "--no-detect-types", + is_flag=True, + help="Treat all CSV/TSV columns as TEXT", + ), + click.option( + "--analyze", + is_flag=True, + help="Run ANALYZE at the end of this operation", + ), + load_extension_option, + click.option("--silent", is_flag=True, help="Do not show progress bar"), + click.option( + "--strict", + is_flag=True, + default=False, + help="Apply STRICT mode to created table", + ), + ) + ): + fn = decorator(fn) + return fn + + return inner + + def insert_upsert_implementation( path, table, - json_file, + file, pk, - nl, flatten, + nl, csv, tsv, + empty_null, + lines, + text, + convert, + imports, delimiter, quotechar, sniff, no_headers, + encoding, batch_size, + stop_after, alter, upsert, ignore=False, @@ -713,109 +1072,245 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, - encoding=None, - detect_types=None, + types=None, + no_detect_types=False, + analyze=False, load_extension=None, silent=False, + bulk_sql=None, + functions=None, + strict=False, + code=None, ): db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) - if delimiter or quotechar or sniff or no_headers: + _maybe_register_functions(db, functions) + column_type_overrides = {column: ctype.upper() for column, ctype in (types or [])} + + def _insert_docs(docs, tracker=None): + extra_kwargs = { + "ignore": ignore, + "replace": replace, + "truncate": truncate, + "analyze": analyze, + "strict": strict, + } + if not_null: + extra_kwargs["not_null"] = set(not_null) + if default: + extra_kwargs["defaults"] = dict(default) + if column_type_overrides: + extra_kwargs["columns"] = column_type_overrides + if upsert: + extra_kwargs["upsert"] = upsert + + # docs should all be dictionaries + docs = (verify_is_dict(doc) for doc in docs) + + # Apply {"$base64": true, ...} decoding, if needed + docs = (decode_base64_values(doc) for doc in docs) + + # For bulk_sql= we use cursor.executemany() instead + if bulk_sql: + if batch_size: + doc_chunks = chunks(docs, batch_size) + else: + doc_chunks = [docs] + for doc_chunk in doc_chunks: + with db.atomic(): + db.conn.cursor().executemany(bulk_sql, doc_chunk) + return + + # table_names() rather than db.table(), which raises NoTable for + # views before the error handling below can deal with them + table_existed_before_insert = table in db.table_names() + try: + db.table(table).insert_all( + docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs + ) + except (NoTable, InvalidColumns, PrimaryKeyRequired) as e: + raise click.ClickException(str(e)) + except Exception as e: + if ( + isinstance(e, OperationalError) + and e.args + and ( + "has no column named" in e.args[0] or "no such column" in e.args[0] + ) + ): + raise click.ClickException( + "{}\n\nTry using --alter to add additional columns".format( + e.args[0] + ) + ) + # If we can find sql= and parameters= arguments, show those + variables = _find_variables(e.__traceback__, ["sql", "parameters"]) + if "sql" in variables and "parameters" in variables: + raise click.ClickException( + "{}\n\nsql = {}\nparameters = {}".format( + str(e), variables["sql"], variables["parameters"] + ) + ) + else: + raise + # Apply detected types only to a table this command created - + # transforming a pre-existing table would rewrite its column types + # and corrupt values such as TEXT zip codes with leading zeros + if ( + tracker is not None + and not table_existed_before_insert + and db.table(table).exists() + ): + detected_types = tracker.types + detected_types.update(column_type_overrides) + db.table(table).transform(types=detected_types) + + if code is not None: + if file is not None: + raise click.ClickException("--code cannot be used with a FILE argument") + if any( + [ + flatten, + nl, + csv, + tsv, + empty_null, + lines, + text, + convert, + sniff, + no_headers, + delimiter, + quotechar, + encoding, + ] + ): + raise click.ClickException( + "--code cannot be used with input format options" + ) + _insert_docs(_rows_from_code(code)) + return + + if file is None: + raise click.ClickException( + "Provide either a FILE argument or --code to specify rows to insert" + ) + + if (delimiter or quotechar or sniff or no_headers) and not tsv: csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") if (csv or tsv) and flatten: raise click.ClickException("--flatten cannot be used with --csv or --tsv") + if empty_null and not (csv or tsv): + raise click.ClickException("--empty-null can only be used with --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") if pk and len(pk) == 1: pk = pk[0] encoding = encoding or "utf-8-sig" - buffered = io.BufferedReader(json_file, buffer_size=4096) - decoded = io.TextIOWrapper(buffered, encoding=encoding) + + # The --sniff option needs us to buffer the file to peek ahead + sniff_buffer = None + decoded_buffer = None + if sniff: + sniff_buffer = io.BufferedReader(file, buffer_size=4096) + decoded_buffer = io.TextIOWrapper(sniff_buffer, encoding=encoding) + else: + decoded_buffer = io.TextIOWrapper(file, encoding=encoding) + tracker = None - if csv or tsv: - if sniff: - # Read first 2048 bytes and use that to detect - first_bytes = buffered.peek(2048) - dialect = csv_std.Sniffer().sniff(first_bytes.decode(encoding, "ignore")) - else: - dialect = "excel-tab" if tsv else "excel" - with file_progress(decoded, silent=silent) as decoded: - csv_reader_args = {"dialect": dialect} + with file_progress(decoded_buffer, silent=silent) as decoded: + if csv or tsv: + if sniff: + # Read first 2048 bytes and use that to detect + assert sniff_buffer is not None + first_bytes = sniff_buffer.peek(2048) + dialect = csv_std.Sniffer().sniff( + first_bytes.decode(encoding, "ignore") + ) + else: + dialect = "excel-tab" if tsv else "excel" + csv_reader_args: dict[str, Any] = {"dialect": dialect} if delimiter: csv_reader_args["delimiter"] = delimiter if quotechar: csv_reader_args["quotechar"] = quotechar - reader = csv_std.reader(decoded, **csv_reader_args) + reader = csv_std.reader(decoded, **csv_reader_args) # type: ignore first_row = next(reader) if no_headers: headers = ["untitled_{}".format(i + 1) for i in range(len(first_row))] reader = itertools.chain([first_row], reader) else: headers = first_row - docs = (dict(zip(headers, row)) for row in reader) - if detect_types: + if empty_null: + docs = ( + dict(zip(headers, [None if cell == "" else cell for cell in row])) + for row in reader + ) + else: + docs = (dict(zip(headers, row)) for row in reader) + # Type detection is the default, unless --no-detect-types is passed + if not no_detect_types: tracker = TypeTracker() docs = tracker.wrap(docs) - else: - try: - if nl: - docs = (json.loads(line) for line in decoded) - else: - docs = json.load(decoded) - if isinstance(docs, dict): - docs = [docs] - except json.decoder.JSONDecodeError: - raise click.ClickException( - "Invalid JSON - use --csv for CSV or --tsv for TSV files" - ) - if flatten: - docs = (dict(_flatten(doc)) for doc in docs) - - extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} - if not_null: - extra_kwargs["not_null"] = set(not_null) - if default: - extra_kwargs["defaults"] = dict(default) - if upsert: - extra_kwargs["upsert"] = upsert - # Apply {"$base64": true, ...} decoding, if needed - docs = (decode_base64_values(doc) for doc in docs) - try: - db[table].insert_all( - docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs - ) - except Exception as e: - if ( - isinstance(e, sqlite3.OperationalError) - and e.args - and "has no column named" in e.args[0] - ): - raise click.ClickException( - "{}\n\nTry using --alter to add additional columns".format(e.args[0]) - ) - # If we can find sql= and parameters= arguments, show those - variables = _find_variables(e.__traceback__, ["sql", "parameters"]) - if "sql" in variables and "parameters" in variables: - raise click.ClickException( - "{}\n\nsql = {}\nparameters = {}".format( - str(e), variables["sql"], variables["parameters"] + elif lines: + docs = ({"line": line.strip()} for line in decoded) + elif text: + docs = ({"text": decoded.read()},) + else: + try: + if nl: + docs = (json.loads(line) for line in decoded if line.strip()) + else: + docs = json.load(decoded) + if isinstance(docs, dict): + docs = [docs] + except json.decoder.JSONDecodeError as ex: + raise click.ClickException( + "Invalid JSON - use --csv for CSV or --tsv for TSV files\n\nJSON error: {}".format( + ex + ) ) - ) - else: - raise - if tracker is not None: - db[table].transform(types=tracker.types) + if flatten: + docs = (_flatten(doc) for doc in docs) + if stop_after: + docs = itertools.islice(docs, stop_after) -def _flatten(d): - for key, value in d.items(): - if isinstance(value, dict): - for key2, value2 in _flatten(value): - yield key + "_" + key2, value2 - else: - yield key, value + if convert: + variable = "row" + if lines: + variable = "line" + elif text: + variable = "text" + fn = _compile_code(convert, imports, variable=variable) + if lines: + docs = (fn(doc["line"]) for doc in docs) + elif text: + # Special case: this is allowed to be an iterable + text_value = list(docs)[0]["text"] + fn_return = fn(text_value) + if isinstance(fn_return, dict): + docs = [fn_return] + else: + try: + docs = iter(fn_return) + except TypeError: + raise click.ClickException( + "--convert must return dict or iterator" + ) + else: + docs = (fn(doc) or doc for doc in docs) + + _insert_docs(docs, tracker=tracker) + + # Clean up open file-like objects + if sniff_buffer: + sniff_buffer.close() + if decoded_buffer: + decoded_buffer.close() def _find_variables(tb, vars): @@ -831,7 +1326,7 @@ def _find_variables(tb, vars): @cli.command() -@insert_upsert_options +@insert_upsert_options() @click.option( "--ignore", is_flag=True, default=False, help="Ignore records if pk already exists" ) @@ -850,20 +1345,28 @@ def _find_variables(tb, vars): def insert( path, table, - json_file, + file, pk, - nl, + code, flatten, + nl, csv, tsv, + empty_null, + lines, + text, + convert, + imports, delimiter, quotechar, sniff, no_headers, - batch_size, - alter, encoding, - detect_types, + batch_size, + stop_after, + alter, + no_detect_types, + analyze, load_extension, silent, ignore, @@ -871,100 +1374,318 @@ def insert( truncate, not_null, default, + types, + strict, ): """ - Insert records from JSON file into a table, creating the table if it + Insert records from FILE into a table, creating the table if it does not already exist. - Input should be a JSON array of objects, unless --nl or --csv is used. + Example: + + echo '{"name": "Lila"}' | sqlite-utils insert data.db chickens - + + By default the input is expected to be a JSON object or array of objects. + + \b + - Use --nl for newline-delimited JSON objects + - Use --csv or --tsv for comma-separated or tab-separated input + - Use --lines to write each incoming line to a column called "line" + - Use --text to write the entire input to a column called "text" + + Use --type column-name type to override the type automatically chosen + when the table is created. + + You can also use --convert to pass a fragment of Python code that will + be used to convert each input. + + Your Python code will be passed a "row" variable representing the + imported row, and can return a modified row. + + This example uses just the name, latitude and longitude columns from + a CSV file, converting name to upper case and latitude and longitude + to floating point numbers: + + \b + sqlite-utils insert plants.db plants plants.csv --csv --convert ' + return { + "name": row["name"].upper(), + "latitude": float(row["latitude"]), + "longitude": float(row["longitude"]), + }' + + If you are using --lines your code will be passed a "line" variable, + and for --text a "text" variable. + + When using --text your function can return an iterator of rows to + insert. This example inserts one record per word in the input: + + \b + echo 'A bunch of words' | sqlite-utils insert words.db words - \\ + --text --convert '({"word": w} for w in text.split())' + + Instead of a FILE you can use --code to provide a block of Python code + that defines the rows to insert, as either a rows() function that yields + dictionaries or a "rows" iterable. --code can also be a path to a .py file: + + \b + sqlite-utils insert data.db creatures --code ' + def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} + ' --pk id """ try: insert_upsert_implementation( path, table, - json_file, + file, pk, - nl, flatten, + nl, csv, tsv, + empty_null, + lines, + text, + convert, + imports, delimiter, quotechar, sniff, no_headers, + encoding, batch_size, + stop_after, alter=alter, upsert=False, ignore=ignore, replace=replace, truncate=truncate, - encoding=encoding, - detect_types=detect_types, + no_detect_types=no_detect_types, + analyze=analyze, load_extension=load_extension, silent=silent, not_null=not_null, default=default, + types=types, + strict=strict, + code=code, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) @cli.command() -@insert_upsert_options +@insert_upsert_options() def upsert( path, table, - json_file, + file, pk, - nl, + code, flatten, + nl, csv, tsv, + empty_null, + lines, + text, + convert, + imports, batch_size, + stop_after, delimiter, quotechar, sniff, no_headers, + encoding, alter, not_null, default, - encoding, - detect_types, + types, + no_detect_types, + analyze, load_extension, silent, + strict, ): """ Upsert records based on their primary key. Works like 'insert' but if an incoming record has a primary key that matches an existing record the existing record will be updated. + + If the table already exists and has a primary key, --pk can be omitted. + + Use --type column-name type to override the type automatically chosen + when the table is created. + + Example: + + \b + echo '[ + {"id": 1, "name": "Lila"}, + {"id": 2, "name": "Suna"} + ]' | sqlite-utils upsert data.db chickens - --pk id """ try: insert_upsert_implementation( path, table, - json_file, + file, pk, - nl, flatten, + nl, csv, tsv, + empty_null, + lines, + text, + convert, + imports, delimiter, quotechar, sniff, no_headers, + encoding, batch_size, + stop_after, alter=alter, upsert=True, not_null=not_null, default=default, - encoding=encoding, + types=types, + no_detect_types=no_detect_types, + analyze=analyze, load_extension=load_extension, silent=silent, + strict=strict, + code=code, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("sql") +@click.argument("file", type=click.File("rb"), required=True) +@click.option("--batch-size", type=int, default=100, help="Commit every X records") +@functions_option +@import_options +@load_extension_option +def bulk( + path, + sql, + file, + batch_size, + functions, + flatten, + nl, + csv, + tsv, + empty_null, + lines, + text, + convert, + imports, + delimiter, + quotechar, + sniff, + no_headers, + encoding, + load_extension, +): + """ + Execute parameterized SQL against the provided list of documents. + + Example: + + \b + echo '[ + {"id": 1, "name": "Lila2"}, + {"id": 2, "name": "Suna2"} + ]' | sqlite-utils bulk data.db ' + update chickens set name = :name where id = :id + ' - + """ + try: + insert_upsert_implementation( + path=path, + table=None, + file=file, + pk=None, + flatten=flatten, + nl=nl, + csv=csv, + tsv=tsv, + empty_null=empty_null, + lines=lines, + text=text, + convert=convert, + imports=imports, + delimiter=delimiter, + quotechar=quotechar, + sniff=sniff, + no_headers=no_headers, + encoding=encoding, + batch_size=batch_size, + stop_after=None, + alter=False, + upsert=False, + not_null=set(), + default={}, + no_detect_types=True, + load_extension=load_extension, + silent=False, + bulk_sql=sql, + functions=functions, + ) + except (OperationalError, sqlite3.IntegrityError) as e: + raise click.ClickException(str(e)) + + +@cli.command(name="create-database") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.option( + "--enable-wal", is_flag=True, help="Enable WAL mode on the created database" +) +@click.option( + "--init-spatialite", is_flag=True, help="Enable SpatiaLite on the created database" +) +@load_extension_option +def create_database(path, enable_wal, init_spatialite, load_extension): + """Create a new empty database file + + Example: + + \b + sqlite-utils create-database trees.db + """ + db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) + if enable_wal: + db.enable_wal() + + # load spatialite or another extension from a custom location + if load_extension: + _load_extensions(db, load_extension) + + # load spatialite from expected locations and initialize metadata + if init_spatialite: + db.init_spatialite() + + db.vacuum() + + @cli.command(name="create-table") @click.argument( "path", @@ -973,7 +1694,7 @@ def upsert( ) @click.argument("table") @click.argument("columns", nargs=-1, required=True) -@click.option("--pk", help="Column to use as primary key") +@click.option("pks", "--pk", help="Column to use as primary key", multiple=True) @click.option( "--not-null", multiple=True, @@ -1001,22 +1722,46 @@ def upsert( is_flag=True, help="If table already exists, replace it", ) +@click.option( + "--transform", + is_flag=True, + help="If table already exists, try to transform the schema", +) @load_extension_option +@click.option( + "--strict", + is_flag=True, + help="Apply STRICT mode to created table", +) def create_table( - path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension + path, + table, + columns, + pks, + not_null, + default, + fk, + ignore, + replace, + transform, + load_extension, + strict, ): """ Add a table with the specified columns. Columns should be specified using name, type pairs, for example: \b - sqlite-utils create-table my.db people \\ - id integer \\ - name text \\ - height float \\ - photo blob --pk id + sqlite-utils create-table my.db people \\ + id integer \\ + name text \\ + height real \\ + photo blob --pk id + + Valid column types are text, integer, real, float and blob. """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) if len(columns) % 2 == 1: raise click.ClickException( @@ -1034,21 +1779,75 @@ def create_table( coltypes[name] = ctype.upper() # Does table already exist? if table in db.table_names(): - if ignore: - return - elif replace: - db[table].drop() - else: + if not ignore and not replace and not transform: raise click.ClickException( 'Table "{}" already exists. Use --replace to delete and replace it.'.format( table ) ) - db[table].create( - coltypes, pk=pk, not_null=not_null, defaults=dict(default), foreign_keys=fk + db.table(table).create( + coltypes, + pk=pks[0] if len(pks) == 1 else pks, + not_null=not_null, + defaults=dict(default), + foreign_keys=fk, + ignore=ignore, + replace=replace, + transform=transform, + strict=strict, ) +@cli.command(name="duplicate") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.argument("new_table") +@click.option("--ignore", is_flag=True, help="If table does not exist, do nothing") +@load_extension_option +def duplicate(path, table, new_table, ignore, load_extension): + """ + Create a duplicate of this table, copying across the schema and all row data. + """ + db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) + _load_extensions(db, load_extension) + try: + db.table(table).duplicate(new_table) + except NoTable: + if not ignore: + raise click.ClickException('Table "{}" does not exist'.format(table)) + + +@cli.command(name="rename-table") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.argument("new_name") +@click.option("--ignore", is_flag=True, help="If table does not exist, do nothing") +@load_extension_option +def rename_table(path, table, new_name, ignore, load_extension): + """ + Rename this table. + """ + db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) + _load_extensions(db, load_extension) + try: + db.rename_table(table, new_name) + except sqlite3.OperationalError as ex: + if not ignore: + raise click.ClickException( + 'Table "{}" could not be renamed. {}'.format(table, str(ex)) + ) + + @cli.command(name="drop-table") @click.argument( "path", @@ -1056,15 +1855,28 @@ def create_table( required=True, ) @click.argument("table") -@click.option("--ignore", is_flag=True) +@click.option("--ignore", is_flag=True, help="If table does not exist, do nothing") @load_extension_option def drop_table(path, table, ignore, load_extension): - "Drop the specified table" + """Drop the specified table + + Example: + + \b + sqlite-utils drop-table chickens.db chickens + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) try: - db[table].drop(ignore=ignore) - except sqlite3.OperationalError: + db.table(table).drop(ignore=ignore) + except NoTable: + # A view exists with this name + if not ignore: + raise click.ClickException( + '"{}" is a view, not a table - use drop-view to drop it'.format(table) + ) + except OperationalError: raise click.ClickException('Table "{}" does not exist'.format(table)) @@ -1088,15 +1900,23 @@ def drop_table(path, table, ignore, load_extension): ) @load_extension_option def create_view(path, view, select, ignore, replace, load_extension): - "Create a view for the provided SELECT query" + """Create a view for the provided SELECT query + + Example: + + \b + sqlite-utils create-view chickens.db heavy_chickens \\ + 'select * from chickens where weight > 3' + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) # Does view already exist? if view in db.view_names(): if ignore: return elif replace: - db[view].drop() + db.view(view).drop() else: raise click.ClickException( 'View "{}" already exists. Use --replace to delete and replace it.'.format( @@ -1113,15 +1933,28 @@ def create_view(path, view, select, ignore, replace, load_extension): required=True, ) @click.argument("view") -@click.option("--ignore", is_flag=True) +@click.option("--ignore", is_flag=True, help="If view does not exist, do nothing") @load_extension_option def drop_view(path, view, ignore, load_extension): - "Drop the specified view" + """Drop the specified view + + Example: + + \b + sqlite-utils drop-view chickens.db heavy_chickens + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) try: - db[view].drop(ignore=ignore) - except sqlite3.OperationalError: + db.view(view).drop(ignore=ignore) + except NoView: + if ignore: + return + if view in db.table_names(): + raise click.ClickException( + '"{}" is a table, not a view - use drop-table to drop it'.format(view) + ) raise click.ClickException('View "{}" does not exist'.format(view)) @@ -1140,6 +1973,7 @@ def drop_view(path, view, ignore, load_extension): ) @output_options @click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") +@click.option("--raw-lines", is_flag=True, help="Raw output, first column of each row") @click.option( "-p", "--param", @@ -1147,6 +1981,7 @@ def drop_view(path, view, ignore, load_extension): type=(str, str), help="Named :parameters for SQL query", ) +@functions_option @load_extension_option def query( path, @@ -1160,19 +1995,54 @@ def query( table, fmt, json_cols, + ascii_, raw, + raw_lines, param, load_extension, + functions, ): - "Execute SQL query and return the results as JSON" + """Execute SQL query and return the results as JSON + + Example: + + \b + sqlite-utils data.db \\ + "select * from chickens where age > :age" \\ + -p age 1 + + Pass "-" as the SQL to read the query from standard input: + + \b + echo "select * from chickens" | sqlite-utils data.db - + """ + if sql == "-": + # Read SQL from standard input + sql = sys.stdin.read() db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) for alias, attach_path in attach: db.attach(alias, attach_path) _load_extensions(db, load_extension) db.register_fts4_bm25() + _maybe_register_functions(db, functions) + _execute_query( - db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + db, + sql, + param, + raw, + raw_lines, + table, + csv, + tsv, + no_headers, + fmt, + nl, + arrays, + json_cols, + ascii_, ) @@ -1184,14 +2054,21 @@ def query( nargs=-1, ) @click.argument("sql") +@functions_option @click.option( "--attach", type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), multiple=True, help="Additional databases to attach - specify alias and filepath", ) +@click.option( + "--flatten", + is_flag=True, + help='Flatten nested JSON objects, so {"foo": {"bar": 1}} becomes {"foo_bar": 1}', +) @output_options @click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") +@click.option("--raw-lines", is_flag=True, help="Raw output, first column of each row") @click.option( "-p", "--param", @@ -1216,11 +2093,18 @@ def query( type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), help="Save in-memory database to this file", ) +@click.option( + "--analyze", + is_flag=True, + help="Analyze resulting tables and output results", +) @load_extension_option def memory( paths, sql, + functions, attach, + flatten, nl, arrays, csv, @@ -1229,14 +2113,18 @@ def memory( table, fmt, json_cols, + ascii_, raw, + raw_lines, param, encoding, no_detect_types, schema, dump, save, + analyze, load_extension, + return_db=False, ): """Execute SQL query against an in-memory database, optionally populated by imported data @@ -1265,42 +2153,68 @@ def memory( sqlite-utils memory animals.csv --schema """ db = sqlite_utils.Database(memory=True) - # If --dump or --save used but no paths detected, assume SQL query is a path: - if (dump or save or schema) and not paths: + if not return_db: + _register_db_for_cleanup(db) + + # If --dump or --save or --analyze used but no paths detected, assume SQL query is a path: + if (dump or save or schema or analyze) and not paths: paths = [sql] sql = None + stem_counts = {} for i, path in enumerate(paths): # Path may have a :format suffix + fp = None + should_close_fp = False if ":" in path and path.rsplit(":", 1)[-1].upper() in Format.__members__: path, suffix = path.rsplit(":", 1) format = Format[suffix.upper()] else: format = None if path in ("-", "stdin"): - csv_fp = sys.stdin.buffer - csv_table = "stdin" + fp = sys.stdin.buffer + file_table = "stdin" else: - csv_path = pathlib.Path(path) - csv_table = csv_path.stem - csv_fp = csv_path.open("rb") - rows, format_used = rows_from_file(csv_fp, format=format, encoding=encoding) - tracker = None - if format_used in (Format.CSV, Format.TSV) and not no_detect_types: - tracker = TypeTracker() - rows = tracker.wrap(rows) - db[csv_table].insert_all(rows, alter=True) - if tracker is not None: - db[csv_table].transform(types=tracker.types) - # Add convenient t / t1 / t2 views - view_names = ["t{}".format(i + 1)] - if i == 0: - view_names.append("t") - for view_name in view_names: - if not db[view_name].exists(): - db.create_view(view_name, "select * from [{}]".format(csv_table)) + file_path = pathlib.Path(path) + stem = file_path.stem + if stem_counts.get(stem): + file_table = "{}_{}".format(stem, stem_counts[stem]) + else: + file_table = stem + stem_counts[stem] = stem_counts.get(stem, 1) + 1 + fp = file_path.open("rb") + should_close_fp = True + try: + rows, format_used = rows_from_file(fp, format=format, encoding=encoding) + tracker = None + if format_used in (Format.CSV, Format.TSV) and not no_detect_types: + tracker = TypeTracker() + rows = tracker.wrap(rows) + if flatten: + rows = (_flatten(row) for row in rows) + + db.table(file_table).insert_all(rows, alter=True) + if tracker is not None and db.table(file_table).exists(): + db.table(file_table).transform(types=tracker.types) + # Add convenient t / t1 / t2 views + view_names = ["t{}".format(i + 1)] + if i == 0: + view_names.append("t") + for view_name in view_names: + if not db[view_name].exists(): + db.create_view( + view_name, + "select * from {}".format(quote_identifier(file_table)), + ) + finally: + if should_close_fp and fp: + fp.close() + + if analyze: + _analyze(db, tables=None, columns=None, save=False) + return if dump: - for line in db.conn.iterdump(): + for line in db.iterdump(): click.echo(line) return @@ -1310,7 +2224,8 @@ def memory( if save: db2 = sqlite_utils.Database(save) - for line in db.conn.iterdump(): + _register_db_for_cleanup(db2) + for line in db.iterdump(): db2.execute(line) return @@ -1319,18 +2234,49 @@ def memory( _load_extensions(db, load_extension) db.register_fts4_bm25() + _maybe_register_functions(db, functions) + + if return_db: + return db + _execute_query( - db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + db, + sql, + param, + raw, + raw_lines, + table, + csv, + tsv, + no_headers, + fmt, + nl, + arrays, + json_cols, + ascii_, ) def _execute_query( - db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + db, + sql, + param, + raw, + raw_lines, + table, + csv, + tsv, + no_headers, + fmt, + nl, + arrays, + json_cols, + ascii_, ): with db.conn: try: cursor = db.execute(sql, dict(param)) - except sqlite3.OperationalError as e: + except OperationalError as e: raise click.ClickException(str(e)) if cursor.description is None: # This was an update/insert @@ -1338,14 +2284,29 @@ def _execute_query( cursor = [[cursor.rowcount]] else: headers = [c[0] for c in cursor.description] + cursor_or_rows: Any = cursor if raw: - data = cursor.fetchone()[0] + row = cursor_or_rows.fetchone() + data = row[0] if row else None if isinstance(data, bytes): sys.stdout.buffer.write(data) else: sys.stdout.write(str(data)) - elif table: - print(tabulate.tabulate(list(cursor), headers=headers, tablefmt=fmt)) + elif raw_lines: + for row in cursor: + data = row[0] + if isinstance(data, bytes): + sys.stdout.buffer.write(data + b"\n") + else: + sys.stdout.write(str(data) + "\n") + elif fmt or table: + print( + tabulate.tabulate( + list(cursor), + headers=() if no_headers else headers, + tablefmt=fmt or "simple", + ) + ) elif csv or tsv: writer = csv_std.writer(sys.stdout, dialect="excel-tab" if tsv else "excel") if not no_headers: @@ -1353,7 +2314,7 @@ def _execute_query( for row in cursor: writer.writerow(row) else: - for line in output_rows(cursor, headers, nl, arrays, json_cols): + for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_): click.echo(line) @@ -1375,6 +2336,7 @@ def _execute_query( @click.option( "--sql", "show_sql", is_flag=True, help="Show SQL query that would be run" ) +@click.option("--quote", is_flag=True, help="Apply FTS quoting rules to search term") @output_options @load_extension_option @click.pass_context @@ -1385,6 +2347,7 @@ def search( q, order, show_sql, + quote, column, limit, nl, @@ -1395,13 +2358,20 @@ def search( table, fmt, json_cols, + ascii_, load_extension, ): - "Execute a full-text search against this table" + """Execute a full-text search against this table + + Example: + + sqlite-utils search data.db chickens lila + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) # Check table exists - table_obj = db[dbtable] + table_obj = db.table(dbtable) if not table_obj.exists(): raise click.ClickException("Table '{}' does not exist".format(dbtable)) if not table_obj.detect_fts(): @@ -1420,6 +2390,104 @@ def search( if show_sql: click.echo(sql) return + if quote: + q = db.quote_fts(q) + try: + ctx.invoke( + query, + path=path, + sql=sql, + nl=nl, + arrays=arrays, + csv=csv, + tsv=tsv, + no_headers=no_headers, + table=table, + fmt=fmt, + json_cols=json_cols, + ascii_=ascii_, + param=[("query", q)], + load_extension=load_extension, + ) + except click.ClickException as e: + if "malformed MATCH expression" in str(e) or "unterminated string" in str(e): + raise click.ClickException( + "{}\n\nTry running this again with the --quote option".format(str(e)) + ) + else: + raise + + +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("dbtable") +@click.option("-c", "--column", type=str, multiple=True, help="Columns to return") +@click.option("--where", help="Optional where clause") +@click.option("-o", "--order", type=str, help="Order by ('column' or 'column desc')") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for where clause", +) +@click.option( + "--limit", + type=int, + help="Number of rows to return - defaults to everything", +) +@click.option( + "--offset", + type=int, + help="SQL offset to use", +) +@output_options +@load_extension_option +@click.pass_context +def rows( + ctx, + path, + dbtable, + column, + where, + order, + param, + limit, + offset, + nl, + arrays, + csv, + tsv, + no_headers, + table, + fmt, + json_cols, + ascii_, + load_extension, +): + """Output all rows in the specified table + + Example: + + \b + sqlite-utils rows trees.db Trees + """ + columns = "*" + if column: + columns = ", ".join(quote_identifier(c) for c in column) + sql = "select {} from {}".format(columns, quote_identifier(dbtable)) + if where: + sql += " where " + where + if order: + sql += " order by " + order + if limit: + sql += " limit {}".format(limit) + if offset: + sql += " offset {}".format(offset) ctx.invoke( query, path=path, @@ -1431,54 +2499,9 @@ def search( no_headers=no_headers, table=table, fmt=fmt, + param=param, json_cols=json_cols, - param=[("query", q)], - load_extension=load_extension, - ) - - -@cli.command() -@click.argument( - "path", - type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), - required=True, -) -@click.argument("dbtable") -@click.option("-c", "--column", type=str, multiple=True, help="Columns to return") -@output_options -@load_extension_option -@click.pass_context -def rows( - ctx, - path, - dbtable, - column, - nl, - arrays, - csv, - tsv, - no_headers, - table, - fmt, - json_cols, - load_extension, -): - "Output all rows in the specified table" - columns = "*" - if column: - columns = ", ".join("[{}]".format(c) for c in column) - ctx.invoke( - query, - path=path, - sql="select {} from [{}]".format(columns, dbtable), - nl=nl, - arrays=arrays, - csv=csv, - tsv=tsv, - no_headers=no_headers, - table=table, - fmt=fmt, - json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -1505,13 +2528,22 @@ def triggers( table, fmt, json_cols, + ascii_, load_extension, ): - "Show triggers configured in this database" - sql = "select name, tbl_name as [table], sql from sqlite_master where type = 'trigger'" + """Show triggers configured in this database + + Example: + + \b + sqlite-utils triggers trees.db + """ + sql = "select name, tbl_name as \"table\", sql from sqlite_master where type = 'trigger'" if tables: - quote = sqlite_utils.Database(memory=True).quote - sql += " and [table] in ({})".format( + _quote_db = sqlite_utils.Database(memory=True) + _register_db_for_cleanup(_quote_db) + quote = _quote_db.quote + sql += ' and "table" in ({})'.format( ", ".join(quote(table) for table in tables) ) ctx.invoke( @@ -1526,6 +2558,7 @@ def triggers( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -1554,9 +2587,16 @@ def indexes( table, fmt, json_cols, + ascii_, load_extension, ): - "Show indexes for this database" + """Show indexes for the whole database or specific tables + + Example: + + \b + sqlite-utils indexes trees.db Trees + """ sql = """ select sqlite_master.name as "table", @@ -1569,7 +2609,9 @@ def indexes( sqlite_master.type = 'table' """ if tables: - quote = sqlite_utils.Database(memory=True).quote + _quote_db = sqlite_utils.Database(memory=True) + _register_db_for_cleanup(_quote_db) + quote = _quote_db.quote sql += " and sqlite_master.name in ({})".format( ", ".join(quote(table) for table in tables) ) @@ -1587,6 +2629,7 @@ def indexes( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -1604,8 +2647,15 @@ def schema( tables, load_extension, ): - "Show full schema for this database or for specified tables" + """Show full schema for this database or for specified tables + + Example: + + \b + sqlite-utils schema trees.db + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) if tables: for table in tables: @@ -1625,10 +2675,12 @@ def schema( "--type", type=( str, - click.Choice(["INTEGER", "TEXT", "FLOAT", "BLOB"], case_sensitive=False), + click.Choice( + ["INTEGER", "TEXT", "FLOAT", "REAL", "BLOB"], case_sensitive=False + ), ), multiple=True, - help="Change column type to INTEGER, TEXT, FLOAT or BLOB", + help="Change column type to INTEGER, TEXT, FLOAT, REAL or BLOB", ) @click.option("--drop", type=str, multiple=True, help="Drop this column") @click.option( @@ -1653,10 +2705,23 @@ def schema( "--default-none", type=str, multiple=True, help="Remove default from this column" ) @click.option( + "add_foreign_keys", + "--add-foreign-key", + type=(str, str, str), + multiple=True, + help="Add a foreign key constraint from a column to another table with another column", +) +@click.option( + "drop_foreign_keys", "--drop-foreign-key", type=str, multiple=True, - help="Drop this foreign key constraint", + help="Drop foreign key constraint for this column", +) +@click.option( + "--strict/--no-strict", + default=None, + help="Enable or disable STRICT mode (default: preserve current mode)", ) @click.option("--sql", is_flag=True, help="Output SQL without executing it") @load_extension_option @@ -1673,15 +2738,25 @@ def transform( pk_none, default, default_none, - drop_foreign_key, + add_foreign_keys, + drop_foreign_keys, + strict, sql, load_extension, ): - "Transform a table beyond the capabilities of ALTER TABLE" + """Transform a table beyond the capabilities of ALTER TABLE + + Example: + + \b + sqlite-utils transform mydb.db mytable \\ + --drop column1 \\ + --rename column2 column_renamed + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) types = {} - kwargs = {} for column, ctype in type: if ctype.upper() not in VALID_COLUMN_TYPES: raise click.ClickException( @@ -1701,27 +2776,48 @@ def transform( for column in default_none: default_dict[column] = None - kwargs["types"] = types - kwargs["drop"] = set(drop) - kwargs["rename"] = dict(rename) - kwargs["column_order"] = column_order or None - kwargs["not_null"] = not_null_dict + drop_set = set(drop) + rename_dict = dict(rename) + column_order_list = list(column_order) or None + drop_foreign_keys_value = drop_foreign_keys or None + add_foreign_keys_value = add_foreign_keys or None + pk_value = DEFAULT if pk: if len(pk) == 1: - kwargs["pk"] = pk[0] + pk_value = pk[0] else: - kwargs["pk"] = pk + pk_value = pk elif pk_none: - kwargs["pk"] = None - kwargs["defaults"] = default_dict - if drop_foreign_key: - kwargs["drop_foreign_keys"] = drop_foreign_key + pk_value = None + table_obj = db.table(table) if sql: - for line in db[table].transform_sql(**kwargs): + for line in table_obj.transform_sql( + types=types, + drop=drop_set, + rename=rename_dict, + column_order=column_order_list, + not_null=not_null_dict, + pk=pk_value, + defaults=default_dict, + drop_foreign_keys=drop_foreign_keys_value, + add_foreign_keys=add_foreign_keys_value, + strict=strict, + ): click.echo(line) else: - db[table].transform(**kwargs) + table_obj.transform( + types=types, + drop=drop_set, + rename=rename_dict, + column_order=column_order_list, + not_null=not_null_dict, + pk=pk_value, + defaults=default_dict, + drop_foreign_keys=drop_foreign_keys_value, + add_foreign_keys=add_foreign_keys_value, + strict=strict, + ) @cli.command() @@ -1752,16 +2848,26 @@ def extract( rename, load_extension, ): - "Extract one or more columns into a separate table" + """Extract one or more columns into a separate table + + Example: + + \b + sqlite-utils extract trees.db Street_Trees species + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) - kwargs = dict( + kwargs: dict[str, Any] = dict( columns=columns, table=other_table, fk_column=fk_column, rename=dict(rename), ) - db[table].extract(**kwargs) + try: + db.table(table).extract(**kwargs) + except (NoTable, InvalidColumns) as e: + raise click.ClickException(str(e)) @cli.command(name="insert-files") @@ -1784,11 +2890,16 @@ def extract( multiple=True, help="Column definitions for the table", ) -@click.option("--pk", type=str, help="Column to use as primary key") +@click.option("pks", "--pk", help="Column to use as primary key", multiple=True) @click.option("--alter", is_flag=True, help="Alter table to add missing columns") @click.option("--replace", is_flag=True, help="Replace files with matching primary key") @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") @click.option("--name", type=str, help="File name to use") +@click.option("--text", is_flag=True, help="Store file content as TEXT, not BLOB") +@click.option( + "--encoding", + help="Character encoding for input, defaults to utf-8", +) @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") @load_extension_option def insert_files( @@ -1796,33 +2907,38 @@ def insert_files( table, file_or_dir, column, - pk, + pks, alter, replace, upsert, name, + text, + encoding, silent, load_extension, ): """ Insert one or more files using BLOB columns in the specified table - Example usage: + Example: \b - sqlite-utils insert-files pics.db images *.gif \\ - -c name:name \\ - -c content:content \\ - -c content_hash:sha256 \\ - -c created:ctime_iso \\ - -c modified:mtime_iso \\ - -c size:size \\ - --pk name + sqlite-utils insert-files pics.db images *.gif \\ + -c name:name \\ + -c content:content \\ + -c content_hash:sha256 \\ + -c created:ctime_iso \\ + -c modified:mtime_iso \\ + -c size:size \\ + --pk name """ if not column: - column = ["path:path", "content:content", "size:size"] - if not pk: - pk = "path" + if text: + column = ["path:path", "content_text:content_text", "size:size"] + else: + column = ["path:path", "content:content", "size:size"] + if not pks: + pks = ["path"] def yield_paths_and_relative_paths(): for f_or_d in file_or_dir: @@ -1844,7 +2960,16 @@ def insert_files( def to_insert(): for path, relative_path in bar: row = {} - lookups = FILE_COLUMNS + # content_text is special case as it considers 'encoding' + + def _content_text(p): + resolved = p.resolve() + try: + return resolved.read_text(encoding=encoding) + except UnicodeDecodeError as e: + raise UnicodeDecodeErrorForPath(e, resolved) + + lookups = dict(FILE_COLUMNS, content_text=_content_text) if path == "-": stdin_data = sys.stdin.buffer.read() # We only support a subset of columns for this case @@ -1852,6 +2977,9 @@ def insert_files( "name": lambda p: name or "-", "path": lambda p: name or "-", "content": lambda p: stdin_data, + "content_text": lambda p: stdin_data.decode( + encoding or "utf-8" + ), "sha256": lambda p: hashlib.sha256(stdin_data).hexdigest(), "md5": lambda p: hashlib.md5(stdin_data).hexdigest(), "size": lambda p: len(stdin_data), @@ -1876,10 +3004,22 @@ def insert_files( yield row db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) - with db.conn: - db[table].insert_all( - to_insert(), pk=pk, alter=alter, replace=replace, upsert=upsert + try: + with db.conn: + db.table(table).insert_all( + to_insert(), + pk=pks[0] if len(pks) == 1 else pks, + alter=alter, + replace=replace, + upsert=upsert, + ) + except UnicodeDecodeErrorForPath as e: + raise click.ClickException( + UNICODE_ERROR.format( + "Could not read file '{}' as text\n\n{}".format(e.path, e.exception) + ) ) @@ -1899,45 +3039,79 @@ def insert_files( help="Specific columns to analyze", ) @click.option("--save", is_flag=True, help="Save results to _analyze_tables table") +@click.option("--common-limit", type=int, default=10, help="How many common values") +@click.option("--no-most", is_flag=True, default=False, help="Skip most common values") +@click.option( + "--no-least", is_flag=True, default=False, help="Skip least common values" +) @load_extension_option def analyze_tables( path, tables, columns, save, + common_limit, + no_most, + no_least, load_extension, ): - "Analyze the columns in one or more tables" + """Analyze the columns in one or more tables + + Example: + + \b + sqlite-utils analyze-tables data.db trees + """ db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) _load_extensions(db, load_extension) + _analyze(db, tables, columns, save, common_limit, no_most, no_least) + + +def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least=False): if not tables: tables = db.table_names() todo = [] table_counts = {} + seen_columns = set() for table in tables: table_counts[table] = db[table].count for column in db[table].columns: if not columns or column.name in columns: todo.append((table, column.name)) + seen_columns.add(column.name) + # Check the user didn't specify a column that doesn't exist + if columns and (set(columns) - seen_columns): + raise click.ClickException( + "These columns were not found: {}".format( + ", ".join(sorted(set(columns) - seen_columns)) + ) + ) # Now we now how many we need to do for i, (table, column) in enumerate(todo): column_details = db[table].analyze_column( - column, total_rows=table_counts[table], value_truncate=80 + column, + common_limit=common_limit, + total_rows=table_counts[table], + value_truncate=80, + most_common=not no_most, + least_common=not no_least, ) if save: db["_analyze_tables_"].insert( column_details._asdict(), pk=("table", "column"), replace=True ) - most_common_rendered = _render_common( - "\n\n Most common:", column_details.most_common - ) + most_common_rendered = "" + if column_details.num_null != column_details.total_rows: + most_common_rendered = _render_common( + "\n\n Most common:", column_details.most_common + ) least_common_rendered = _render_common( "\n\n Least common:", column_details.least_common ) details = ( ( - textwrap.dedent( - """ + textwrap.dedent(""" {table}.{column}: ({i}/{total}) Total rows: {total_rows} @@ -1945,15 +3119,14 @@ def analyze_tables( Blank rows: {num_blank} Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered} - """ - ) + """) .strip() .format( i=i + 1, total=len(todo), most_common_rendered=most_common_rendered, least_common_rendered=least_common_rendered, - **column_details._asdict() + **column_details._asdict(), ) ) + "\n" @@ -1961,39 +3134,77 @@ def analyze_tables( click.echo(details) +@cli.command() +@click.argument("packages", nargs=-1, required=False) +@click.option( + "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version" +) +@click.option( + "-e", + "--editable", + help="Install a project in editable mode from this path", +) +def install(packages, upgrade, editable): + """Install packages from PyPI into the same environment as sqlite-utils""" + args = ["pip", "install"] + if upgrade: + args += ["--upgrade"] + if editable: + args += ["--editable", editable] + args += list(packages) + sys.argv = args + run_module("pip", run_name="__main__") + + +@cli.command() +@click.argument("packages", nargs=-1, required=True) +@click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") +def uninstall(packages, yes): + """Uninstall Python packages from the sqlite-utils environment""" + sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else []) + run_module("pip", run_name="__main__") + + def _generate_convert_help(): - help = textwrap.dedent( - """ + help = textwrap.dedent(""" Convert columns using Python code you supply. For example: \b - $ sqlite-utils convert my.db mytable mycolumn \\ + sqlite-utils convert my.db mytable mycolumn \\ '"\\n".join(textwrap.wrap(value, 10))' \\ --import=textwrap "value" is a variable with the column value to be converted. + CODE can also be a reference to a callable that takes the value, for example: + + \b + sqlite-utils convert my.db mytable date r.parsedate + sqlite-utils convert my.db mytable data json.loads --import json + + Use "-" for CODE to read Python code from standard input. + The following common operations are available as recipe functions: - """ - ).strip() + """).strip() recipe_names = [ - n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser") + n + for n in dir(recipes) + if not n.startswith("_") + and n not in ("json", "parser", "Callable", "Optional") + and callable(getattr(recipes, n)) ] for name in recipe_names: fn = getattr(recipes, name) - help += "\n\nr.{}{}\n\n {}".format( - name, str(inspect.signature(fn)), fn.__doc__ - ) + doc = textwrap.dedent(fn.__doc__.rstrip()).replace("\b\n", "") + help += "\n\nr.{}{}\n\n\b{}".format(name, str(inspect.signature(fn)), doc) help += "\n\n" - help += textwrap.dedent( - """ + help += textwrap.dedent(""" You can use these recipes like so: \b - $ sqlite-utils convert my.db mytable mycolumn \\ + sqlite-utils convert my.db mytable mycolumn \\ 'r.jsonsplit(value, delimiter=":")' - """ - ).strip() + """).strip() return help @@ -2032,6 +3243,7 @@ def _generate_convert_help(): ) @click.option("--drop", is_flag=True, help="Drop original column afterwards") @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") +@click.option("pdb_", "--pdb", is_flag=True, help="Open pdb debugger on first error") def convert( db_path, table, @@ -2046,33 +3258,39 @@ def convert( output_type, drop, silent, + pdb_, ): sqlite3.enable_callback_tracebacks(True) db = sqlite_utils.Database(db_path) + _register_db_for_cleanup(db) if output is not None and len(columns) > 1: raise click.ClickException("Cannot use --output with more than one column") if multi and len(columns) > 1: raise click.ClickException("Cannot use --multi with more than one column") if drop and not (output or multi): raise click.ClickException("--drop can only be used with --output or --multi") - # If single line and no 'return', add the return - if "\n" not in code and not code.strip().startswith("return "): - code = "return {}".format(code) + if code == "-": + # Read code from standard input + code = sys.stdin.read() where_args = dict(param) if param else [] # Compile the code into a function body called fn(value) - new_code = ["def fn(value):"] - for line in code.split("\n"): - new_code.append(" {}".format(line)) - code_o = compile("\n".join(new_code), "", "exec") - locals = {} - globals = {"r": recipes, "recipes": recipes} - for import_ in imports: - globals[import_] = __import__(import_) - exec(code_o, globals, locals) - fn = locals["fn"] + try: + fn = _compile_code(code, imports) + except SyntaxError as e: + raise click.ClickException(str(e)) if dry_run: # Pull first 20 values for first column and preview them - db.conn.create_function("preview_transform", 1, lambda v: fn(v) if v else v) + if multi: + + def preview(v): + return json.dumps(fn(v), default=repr, ensure_ascii=False) if v else v + + else: + + def preview(v): + return fn(v) if v else v + + db.conn.create_function("preview_transform", 1, preview) sql = """ select [{column}] as value, @@ -2094,8 +3312,21 @@ def convert( ) click.echo("Would affect {} row{}".format(count, "" if count == 1 else "s")) else: + # Wrap fn with a thing that will catch errors and optionally drop into pdb + if pdb_: + fn_ = fn + + def wrapped_fn(value): + try: + return fn_(value) + except Exception as ex: + print("\nException raised, dropping into pdb...:", ex) + pdb.post_mortem(ex.__traceback__) + sys.exit(1) + + fn = wrapped_fn try: - db[table].convert( + db.table(table).convert( columns, fn, where=where, @@ -2114,6 +3345,308 @@ def convert( ) +@cli.command("add-geometry-column") +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table", type=str) +@click.argument("column_name", type=str) +@click.option( + "-t", + "--type", + "geometry_type", + type=click.Choice( + [ + "point", + "linestring", + "polygon", + "multipoint", + "multilinestring", + "multipolygon", + "geometrycollection", + "geometry", + ], + case_sensitive=False, + ), + default="GEOMETRY", + help="Specify a geometry type for this column.", + show_default=True, +) +@click.option( + "--srid", + type=int, + default=4326, + show_default=True, + help="Spatial Reference ID. See https://spatialreference.org for details on specific projections.", +) +@click.option( + "--dimensions", + "coord_dimension", + type=str, + default="XY", + help="Coordinate dimensions. Use XYZ for three-dimensional geometries.", +) +@click.option("--not-null", "not_null", is_flag=True, help="Add a NOT NULL constraint.") +@load_extension_option +def add_geometry_column( + db_path, + table, + column_name, + geometry_type, + srid, + coord_dimension, + not_null, + load_extension, +): + """Add a SpatiaLite geometry column to an existing table. Requires SpatiaLite extension. + \n\n + By default, this command will try to load the SpatiaLite extension from usual paths. + To load it from a specific path, use --load-extension.""" + db = sqlite_utils.Database(db_path) + _register_db_for_cleanup(db) + if not db[table].exists(): + raise click.ClickException( + "You must create a table before adding a geometry column" + ) + + # load spatialite, one way or another + if load_extension: + _load_extensions(db, load_extension) + db.init_spatialite() + + if db.table(table).add_geometry_column( + column_name, geometry_type, srid, coord_dimension, not_null + ): + click.echo(f"Added {geometry_type} column {column_name} to {table}") + + +@cli.command("create-spatial-index") +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table", type=str) +@click.argument("column_name", type=str) +@load_extension_option +def create_spatial_index(db_path, table, column_name, load_extension): + """Create a spatial index on a SpatiaLite geometry column. + The table and geometry column must already exist before trying to add a spatial index. + \n\n + By default, this command will try to load the SpatiaLite extension from usual paths. + To load it from a specific path, use --load-extension.""" + db = sqlite_utils.Database(db_path) + _register_db_for_cleanup(db) + if not db[table].exists(): + raise click.ClickException( + "You must create a table and add a geometry column before creating a spatial index" + ) + + # load spatialite + if load_extension: + _load_extensions(db, load_extension) + db.init_spatialite() + + if column_name not in db[table].columns_dict: + raise click.ClickException( + "You must add a geometry column before creating a spatial index" + ) + + db.table(table).create_spatial_index(column_name) + + +def _find_migration_files(migrations): + if not migrations: + migrations = [pathlib.Path(".").resolve()] + files = set() + for path_str in migrations: + path = pathlib.Path(path_str) + if path.is_dir(): + files.update(path.rglob("migrations.py")) + else: + files.add(path) + return sorted(files) + + +def _compatible_migration_set(obj): + return isinstance(obj, sqlite_utils.Migrations) or all( + hasattr(obj, attr) for attr in ("name", "applied", "pending", "apply") + ) + + +def _load_migration_sets(files): + migration_sets = [] + for filepath in files: + code = filepath.read_text() + namespace = { + "__file__": str(filepath), + "__name__": "__sqlite_utils_migration__", + } + exec(code, namespace) + migration_sets.extend( + obj for obj in namespace.values() if _compatible_migration_set(obj) + ) + return migration_sets + + +def _display_migration_list(db, migration_sets): + for migration_set in migration_sets: + click.echo("Migrations for: {}".format(migration_set.name)) + click.echo() + click.echo(" Applied:") + for migration in migration_set.applied(db): + click.echo(" {} - {}".format(migration.name, migration.applied_at)) + click.echo() + click.echo(" Pending:") + output = False + for migration in migration_set.pending(db): + output = True + click.echo(" {}".format(migration.name)) + if not output: + click.echo(" (none)") + click.echo() + + +def _stop_before_for_migration_set(stop_before, migration_set_name): + matches = [] + for value in stop_before: + set_name, separator, migration_name = value.partition(":") + if separator: + if set_name == migration_set_name: + matches.append(migration_name) + else: + matches.append(value) + return matches + + +@click.command() +@click.argument( + "db_path", type=click.Path(dir_okay=False, readable=True, writable=True) +) +@click.argument("migrations", type=click.Path(dir_okay=True, exists=True), nargs=-1) +@click.option( + "--stop-before", + multiple=True, + help="Stop before applying this migration. Use set:name to target a migration set.", +) +@click.option( + "list_", "--list", is_flag=True, help="List migrations without running them" +) +@click.option("-v", "--verbose", is_flag=True, help="Show verbose output") +def migrate(db_path, migrations, stop_before, list_, verbose): + """ + Apply pending database migrations. + + Usage: + + sqlite-utils migrate database.db + + This will find the migrations.py file in the current directory + or subdirectories and apply any pending migrations. + + Or pass paths to one or more migrations.py files directly: + + sqlite-utils migrate database.db path/to/migrations.py + + Pass --list to see a list of applied and pending migrations + without applying them. + + Use --stop-before migration_set:name to stop before a + migration. This option can be used multiple times. + """ + files = _find_migration_files(migrations) + migration_sets = _load_migration_sets(files) + if not migration_sets: + raise click.ClickException("No migrations.py files found") + + if list_: + if pathlib.Path(db_path).exists(): + db = sqlite_utils.Database(db_path) + else: + # Listing is read-only - don't create the database file + db = sqlite_utils.Database(memory=True) + _register_db_for_cleanup(db) + # Legacy sqlite-migrate classes create the migrations table from + # their pending()/applied() methods - run the listing inside a + # transaction and roll it back so --list stays read-only + db.begin() + try: + _display_migration_list(db, migration_sets) + finally: + db.rollback() + return + + db = sqlite_utils.Database(db_path) + _register_db_for_cleanup(db) + + prev_schema = db.schema + if verbose: + click.echo("Migrating {}".format(db_path)) + click.echo("\nSchema before:\n") + click.echo(textwrap.indent(prev_schema, " ") or " (empty)") + click.echo() + if stop_before: + # Every --stop-before value must match at least one known migration + known_names = set() + for migration_set in migration_sets: + names = {m.name for m in migration_set.pending(db)} + names.update(m.name for m in migration_set.applied(db)) + known_names.update(names) + known_names.update( + "{}:{}".format(migration_set.name, name) for name in names + ) + unknown = [value for value in stop_before if value not in known_names] + if unknown: + raise click.ClickException( + "--stop-before did not match any migrations: {}".format( + ", ".join(unknown) + ) + ) + for migration_set in migration_sets: + matches = _stop_before_for_migration_set(stop_before, migration_set.name) + if isinstance(migration_set, sqlite_utils.Migrations): + try: + migration_set.apply(db, stop_before=matches) + except ValueError as e: + raise click.ClickException(str(e)) + else: + # Legacy sqlite-migrate Migrations objects take a single string + # for stop_before, not a list + distinct = list(dict.fromkeys(matches)) + if len(distinct) > 1: + raise click.ClickException( + "Migration set '{}' uses the older sqlite-migrate class, " + "which only supports a single --stop-before value - " + "got: {}".format(migration_set.name, ", ".join(distinct)) + ) + migration_set.apply(db, stop_before=distinct[0] if distinct else None) + if verbose: + click.echo("Schema after:\n") + post_schema = db.schema + if post_schema == prev_schema: + click.echo(" (unchanged)") + else: + click.echo(textwrap.indent(post_schema, " ")) + click.echo("\nSchema diff:\n") + diff = list( + difflib.unified_diff(prev_schema.splitlines(), post_schema.splitlines()) + ) + click.echo("\n".join(diff[3:])) + + +@cli.command(name="plugins") +def plugins_list(): + "List installed plugins" + click.echo(json.dumps(get_plugins(), indent=2, ensure_ascii=False)) + + +ensure_plugins_loaded() +pm.hook.register_commands(cli=cli) +cli.add_command(migrate) + + def _render_common(title, values): if values is None: return "" @@ -2123,6 +3656,12 @@ def _render_common(title, values): return "\n".join(lines) +class UnicodeDecodeErrorForPath(Exception): + def __init__(self, exception, path): + self.exception = exception + self.path = path + + FILE_COLUMNS = { "name": lambda p: p.name, "path": lambda p: str(p), @@ -2135,13 +3674,23 @@ FILE_COLUMNS = { "ctime": lambda p: p.stat().st_ctime, "mtime_int": lambda p: int(p.stat().st_mtime), "ctime_int": lambda p: int(p.stat().st_ctime), - "mtime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_mtime).isoformat(), - "ctime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_ctime).isoformat(), + "mtime_iso": lambda p: datetime.fromtimestamp(p.stat().st_mtime, timezone.utc) + .replace(tzinfo=None) + .isoformat(), + "ctime_iso": lambda p: datetime.fromtimestamp(p.stat().st_ctime, timezone.utc) + .replace(tzinfo=None) + .isoformat(), "size": lambda p: p.stat().st_size, + "stem": lambda p: p.stem, + "suffix": lambda p: p.suffix, } -def output_rows(iterator, headers, nl, arrays, json_cols): +def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False): + # Duplicate column names would collide as dictionary keys, so rename + # later occurrences id, id -> id, id_2 - CSV and table output keep + # the original duplicate headers since they never build dictionaries + headers = dedupe_keys(headers) # We have to iterate two-at-a-time so we can know if we # should output a trailing comma or if we have reached # the last row. @@ -2158,12 +3707,15 @@ def output_rows(iterator, headers, nl, arrays, json_cols): data = dict(zip(headers, data)) line = "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar=("[" if first else " ") if not nl else "", - serialized=json.dumps(data, default=json_binary), + serialized=json.dumps(data, default=json_binary, ensure_ascii=ascii_), maybecomma="," if (not nl and not is_last) else "", lastchar="]" if (is_last and not nl) else "", ) yield line first = False + if first: + # We didn't output any rows, so yield the empty list + yield "[]" def maybe_json(value): @@ -2185,10 +3737,83 @@ def json_binary(value): raise TypeError +def verify_is_dict(doc): + if not isinstance(doc, dict): + raise click.ClickException( + "Rows must all be dictionaries, got: {}".format(repr(doc)[:1000]) + ) + return doc + + def _load_extensions(db, load_extension): if load_extension: db.conn.enable_load_extension(True) for ext in load_extension: if ext == "spatialite" and not os.path.exists(ext): - ext = find_spatialite() - db.conn.load_extension(ext) + found = find_spatialite() + if found is None: + raise click.ClickException("Could not find SpatiaLite extension") + ext = found + if ":" in ext: + path, _, entrypoint = ext.partition(":") + db.conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + db.conn.load_extension(ext) + + +def _register_functions(db, functions): + # Register any Python functions as SQL functions: + # Check if this is a file path + if "\n" not in functions and functions.endswith(".py"): + try: + functions = pathlib.Path(functions).read_text() + except FileNotFoundError: + raise click.ClickException("File not found: {}".format(functions)) + + sqlite3.enable_callback_tracebacks(True) + globals = {} + try: + exec(functions, globals) + except SyntaxError as ex: + raise click.ClickException("Error in functions definition: {}".format(ex)) + # Register all callables in the locals dict: + for name, value in globals.items(): + if callable(value) and not name.startswith("_"): + db.register_function(value, name=name) + + +def _maybe_register_functions(db, functions_list): + if not functions_list: + return + for functions in functions_list: + if isinstance(functions, str) and functions.strip(): + _register_functions(db, functions) + + +def _rows_from_code(code): + # code may be a path to a .py file + if "\n" not in code and code.endswith(".py"): + try: + code = pathlib.Path(code).read_text() + except FileNotFoundError: + raise click.ClickException("File not found: {}".format(code)) + namespace = {} + try: + exec(code, namespace) + except SyntaxError as ex: + raise click.ClickException("Error in --code: {}".format(ex)) + rows = namespace.get("rows") + if callable(rows): + rows = rows() + if isinstance(rows, dict): + rows = [rows] + error = click.ClickException( + "--code must define a 'rows' function or iterable of rows to insert" + ) + if rows is None or isinstance(rows, (str, bytes)): + raise error + try: + iter(rows) + except TypeError: + raise error + return rows diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a21acc4..e97b7d9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,31 +1,65 @@ from .utils import ( + chunks, + dedupe_keys, + hash_record, sqlite3, OperationalError, suggest_column_types, types_for_column_types, column_affinity, progressbar, + find_spatialite, ) +import binascii from collections import namedtuple +from dataclasses import dataclass, field from collections.abc import Mapping import contextlib import datetime import decimal -import hashlib +import importlib import inspect import itertools import json import os import pathlib import re -from sqlite_fts4 import rank_bm25 # type: ignore -import sys +import secrets +from sqlite_fts4 import rank_bm25 import textwrap -from typing import Generator, Iterable, Union, Optional, List +from typing import ( + cast, + Any, + Callable, + Dict, + Generator, + Iterable, + Sequence, + Set, + Type, + Union, + Optional, + List, + Tuple, +) import uuid +from sqlite_utils.plugins import ensure_plugins_loaded, pm + +try: + iterdump = importlib.import_module("sqlite_dump").iterdump +except ImportError: + iterdump = None + SQLITE_MAX_VARS = 999 +# Names that refer to a rowid table's implicit integer primary key. These are +# valid primary key targets even though they are not listed among a table's +# columns. See https://www.sqlite.org/lang_createtable.html#rowid +ROWID_ALIASES = frozenset({"rowid", "_rowid_", "oid"}) + +_quote_fts_re = re.compile(r'\s+|(".*?")') + _virtual_table_using_re = re.compile( r""" ^ # Start of string @@ -41,24 +75,86 @@ _virtual_table_using_re = re.compile( ) ) \s+(IF\s+NOT\s+EXISTS\s+)? # IF NOT EXISTS (optional) -USING\s+(?P\w+) # e.g. USING FTS5 +USING\s+(?P\w+) # for example USING FTS5 """, re.VERBOSE | re.IGNORECASE, ) + +def quote_identifier(identifier: str) -> str: + """ + Quote an identifier (table name, column name, etc.) using double quotes. + + Double quotes inside the identifier are escaped by doubling them. + """ + return '"{}"'.format(identifier.replace('"', '""')) + + +_IDENTIFIER_CASEFOLD = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" +) + + +def fold_identifier_case(identifier: str) -> str: + """ + Lowercase an identifier using the same rules SQLite uses - only ASCII + characters are folded, other characters are left unchanged. + """ + return identifier.translate(_IDENTIFIER_CASEFOLD) + + +def resolve_casing(name: str, candidates: Iterable[str]) -> str: + """ + SQLite treats identifiers as case-insensitive. Return the entry in + ``candidates`` that matches ``name`` case-insensitively, preferring an + exact match. If nothing matches, return ``name`` unchanged. + """ + if name in candidates: + return name + folded = fold_identifier_case(name) + for candidate in candidates: + if fold_identifier_case(candidate) == folded: + return candidate + return name + + +pd: Any = None try: - import pandas as pd # type: ignore + pd = importlib.import_module("pandas") except ImportError: pd = None +np: Any = None try: - import numpy as np # type: ignore + np = importlib.import_module("numpy") except ImportError: - np = None # type: ignore + np = None Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) +Column.__doc__ = """ +Describes a SQLite column returned by the :attr:`.Table.columns` property. + +``cid`` + Column index + +``name`` + Column name + +``type`` + Column type + +``notnull`` + Does the column have a ``not null`` constraint + +``default_value`` + Default value for this column + +``is_pk`` + Is this column part of the primary key +""" + ColumnDetails = namedtuple( "ColumnDetails", ( @@ -72,9 +168,103 @@ ColumnDetails = namedtuple( "least_common", ), ) -ForeignKey = namedtuple( - "ForeignKey", ("table", "column", "other_table", "other_column") -) +ColumnDetails.__doc__ = """ +Summary information about a column, see :ref:`python_api_analyze_column`. + +``table`` + The name of the table + +``column`` + The name of the column + +``total_rows`` + The total number of rows in the table + +``num_null`` + The number of rows for which this column is null + +``num_blank`` + The number of rows for which this column is blank (the empty string) + +``num_distinct`` + The number of distinct values in this column + +``most_common`` + The ``N`` most common values as a list of ``(value, count)`` tuples, or ``None`` if the table consists entirely of distinct values + +``least_common`` + The ``N`` least common values as a list of ``(value, count)`` tuples, or ``None`` if the table is entirely distinct + or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) +""" + + +@dataclass(order=True, frozen=True) +class ForeignKey: + """ + A foreign key defined on a table. + + For single-column foreign keys ``column`` and ``other_column`` hold the + column names, and ``columns``/``other_columns`` are one-item tuples. + + For compound (multi-column) foreign keys ``column`` and ``other_column`` + are ``None`` - use ``columns`` and ``other_columns`` instead, and check + ``is_compound``. + + ``on_delete`` and ``on_update`` hold the foreign key actions, e.g. + ``"CASCADE"`` - ``"NO ACTION"`` if not set. + + Instances are immutable and hashable, so they can be collected into + sets and used as dictionary keys. Equality covers every compared field, + including ``on_delete`` and ``on_update`` - two foreign keys differing + only in their actions are different constraints. + + Prior to sqlite-utils 4.0 this was a ``namedtuple`` and could be unpacked + or indexed as ``(table, column, other_table, other_column)``. It is now a + dataclass - access its fields by name instead. + """ + + table: str + # column/other_column are None for compound keys, which would break + # ordering against str values - comparison uses columns/other_columns + column: Optional[str] = field(compare=False) + other_table: str + other_column: Optional[str] = field(compare=False) + columns: Tuple[str, ...] = () + other_columns: Tuple[str, ...] = () + is_compound: bool = False + on_delete: str = "NO ACTION" + on_update: str = "NO ACTION" + + def __post_init__(self): + # Populate columns/other_columns for single-column foreign keys, + # normalizing any lists to tuples. object.__setattr__ because the + # dataclass is frozen + if self.columns: + object.__setattr__(self, "columns", tuple(self.columns)) + else: + object.__setattr__( + self, "columns", (self.column,) if self.column is not None else () + ) + if self.other_columns: + object.__setattr__(self, "other_columns", tuple(self.other_columns)) + else: + object.__setattr__( + self, + "other_columns", + (self.other_column,) if self.other_column is not None else (), + ) + + +def _fk_actions_sql(fk: ForeignKey) -> str: + "ON UPDATE/ON DELETE clauses for a foreign key, or an empty string." + actions = "" + if fk.on_update and fk.on_update != "NO ACTION": + actions += " ON UPDATE {}".format(fk.on_update) + if fk.on_delete and fk.on_delete != "NO ACTION": + actions += " ON DELETE {}".format(fk.on_delete) + return actions + + Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) XIndex = namedtuple("XIndex", ("name", "columns")) XIndexColumn = namedtuple( @@ -83,77 +273,143 @@ XIndexColumn = namedtuple( Trigger = namedtuple("Trigger", ("name", "table", "sql")) -DEFAULT = object() +class TransformError(Exception): + pass -COLUMN_TYPE_MAPPING = { - float: "FLOAT", + +# A single column name, or a tuple of columns for a compound foreign key +ForeignKeyColumns = Union[str, Tuple[str, ...], List[str]] + +# (table, column(s), other_table, other_column(s)) +ForeignKeyTuple = Tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] + +ForeignKeyIndicator = Union[ + str, + ForeignKey, + Tuple[ForeignKeyColumns, str], + Tuple[ForeignKeyColumns, str, ForeignKeyColumns], + ForeignKeyTuple, +] + +ForeignKeysType = Union[Iterable[ForeignKeyIndicator], List[ForeignKeyIndicator]] + + +class Default: + pass + + +DEFAULT = Default() + +Tracer = Callable[[str, Optional[Union[Sequence[Any], Dict[str, Any]]]], None] + + +def _iter_complete_sql_statements(sql: str) -> Generator[str, None, None]: + statement = [] + for char in sql: + statement.append(char) + statement_sql = "".join(statement).strip() + if statement_sql and sqlite3.complete_statement(statement_sql): + yield statement_sql + statement = [] + statement_sql = "".join(statement).strip() + if statement_sql: + yield statement_sql + + +COLUMN_TYPE_MAPPING: Dict[Any, str] = { + float: "REAL", int: "INTEGER", bool: "INTEGER", str: "TEXT", + dict: "TEXT", + tuple: "TEXT", + list: "TEXT", bytes.__class__: "BLOB", bytes: "BLOB", memoryview: "BLOB", datetime.datetime: "TEXT", datetime.date: "TEXT", datetime.time: "TEXT", - decimal.Decimal: "FLOAT", + datetime.timedelta: "TEXT", + decimal.Decimal: "REAL", None.__class__: "TEXT", uuid.UUID: "TEXT", # SQLite explicit types "TEXT": "TEXT", "INTEGER": "INTEGER", "FLOAT": "FLOAT", + "REAL": "REAL", "BLOB": "BLOB", "text": "TEXT", + "str": "TEXT", "integer": "INTEGER", - "float": "FLOAT", + "int": "INTEGER", + "float": "REAL", + "real": "REAL", "blob": "BLOB", + "bytes": "BLOB", } # If numpy is available, add more types if np: - COLUMN_TYPE_MAPPING.update( - { - np.int8: "INTEGER", - np.int16: "INTEGER", - np.int32: "INTEGER", - np.int64: "INTEGER", - np.uint8: "INTEGER", - np.uint16: "INTEGER", - np.uint32: "INTEGER", - np.uint64: "INTEGER", - np.float16: "FLOAT", - np.float32: "FLOAT", - np.float64: "FLOAT", - } - ) + try: + COLUMN_TYPE_MAPPING.update( + { + np.int8: "INTEGER", + np.int16: "INTEGER", + np.int32: "INTEGER", + np.int64: "INTEGER", + np.uint8: "INTEGER", + np.uint16: "INTEGER", + np.uint32: "INTEGER", + np.uint64: "INTEGER", + np.float16: "REAL", + np.float32: "REAL", + np.float64: "REAL", + } + ) + except AttributeError: + # https://github.com/simonw/sqlite-utils/issues/632 + pass # If pandas is available, add more types if pd: - COLUMN_TYPE_MAPPING.update({pd.Timestamp: "TEXT"}) + COLUMN_TYPE_MAPPING.update({pd.Timestamp: "TEXT"}) # type: ignore class AlterError(Exception): - pass + "Error altering table" class NoObviousTable(Exception): - pass + "Could not tell which table this operation refers to" + + +class NoTable(Exception): + "Specified table does not exist" + + +class NoView(Exception): + "Specified view does not exist" class BadPrimaryKey(Exception): - pass + "Table does not have a single obvious primary key" class NotFoundError(Exception): - pass + "Record not found" class PrimaryKeyRequired(Exception): - pass + "Primary key needs to be specified" class InvalidColumns(Exception): - pass + "Specified columns do not exist" + + +class TransactionError(Exception): + "Operation cannot be performed while a transaction is open" class DescIndex(str): @@ -163,111 +419,573 @@ class DescIndex(str): class BadMultiValues(Exception): "With multi=True code must return a Python dictionary" - def __init__(self, values): + def __init__(self, values: object) -> None: self.values = values _COUNTS_TABLE_CREATE_SQL = """ -CREATE TABLE IF NOT EXISTS [{}]( - [table] TEXT PRIMARY KEY, +CREATE TABLE IF NOT EXISTS "{}"( + "table" TEXT PRIMARY KEY, count INTEGER DEFAULT 0 ); """.strip() +_TRANSACTION_CONTROL_KEYWORDS = { + "BEGIN", + "COMMIT", + "END", + "ROLLBACK", + "SAVEPOINT", + "RELEASE", +} + +# Statements that never return rows and cannot run inside (or would break +# out of) the savepoint guard used by query() +_QUERY_REJECTED_KEYWORDS = _TRANSACTION_CONTROL_KEYWORDS | { + "VACUUM", + "ATTACH", + "DETACH", +} + + +def _first_keyword(sql: str) -> str: + """ + Return the first keyword of a SQL statement, uppercased, skipping + everything the sqlite3 driver tolerates before the first real token: + whitespace, ``--`` or ``/* ... */`` comments, empty statements + (bare ``;``) and a UTF-8 byte order mark. Returns an empty string if + there is no leading keyword. + """ + i, n = 0, len(sql) + while i < n: + if sql[i].isspace() or sql[i] in (";", "\ufeff"): + i += 1 + elif sql.startswith("--", i): + newline = sql.find("\n", i) + if newline == -1: + return "" + i = newline + 1 + elif sql.startswith("/*", i): + end = sql.find("*/", i + 2) + if end == -1: + return "" + i = end + 2 + else: + break + j = i + while j < n and (sql[j].isalpha() or sql[j] == "_"): + j += 1 + return sql[i:j].upper() + + class Database: + """ + Wrapper for a SQLite database connection that adds a variety of useful utility methods. + + To create an instance:: + + # create data.db file, or open existing: + db = Database("data.db") + # Create an in-memory database: + dB = Database(memory=True) + + :param filename_or_conn: String path to a file, or a ``pathlib.Path`` object, or a + ``sqlite3`` connection + :param memory: set to ``True`` to create an in-memory database + :param memory_name: creates a named in-memory database that can be shared across multiple connections + :param recreate: set to ``True`` to delete and recreate a file database (**dangerous**) + :param recursive_triggers: defaults to ``True``, which sets ``PRAGMA recursive_triggers=on;`` - + set to ``False`` to avoid setting this pragma + :param tracer: set a tracer function (``print`` works for this) which will be called with + ``sql, parameters`` every time a SQL query is executed + :param use_counts_table: set to ``True`` to use a cached counts table, if available. See + :ref:`python_api_cached_table_counts` + :param use_old_upsert: set to ``True`` to force the older upsert implementation. See + :ref:`python_api_old_upsert` + :param strict: Apply STRICT mode to all created tables (unless overridden) + """ + _counts_table_name = "_counts" use_counts_table = False + conn: sqlite3.Connection def __init__( self, - filename_or_conn=None, - memory=False, - recreate=False, - recursive_triggers=True, - tracer=None, - use_counts_table=False, + filename_or_conn: Optional[Union[str, pathlib.Path, sqlite3.Connection]] = None, + memory: bool = False, + memory_name: Optional[str] = None, + recreate: bool = False, + recursive_triggers: bool = True, + tracer: Optional[Tracer] = None, + use_counts_table: bool = False, + execute_plugins: bool = True, + use_old_upsert: bool = False, + strict: bool = False, ): - assert (filename_or_conn is not None and not memory) or ( - filename_or_conn is None and memory - ), "Either specify a filename_or_conn or pass memory=True" - if memory or filename_or_conn == ":memory:": + self.memory_name = None + self.memory = False + self.use_old_upsert = use_old_upsert + if not ( + (filename_or_conn is not None and (not memory and not memory_name)) + or (filename_or_conn is None and (memory or memory_name)) + ): + raise ValueError("Either specify a filename_or_conn or pass memory=True") + if memory_name: + uri = "file:{}?mode=memory&cache=shared".format(memory_name) + self.conn = sqlite3.connect( + uri, + uri=True, + check_same_thread=False, + ) + self.memory = True + self.memory_name = memory_name + elif memory or filename_or_conn == ":memory:": self.conn = sqlite3.connect(":memory:") + self.memory = True elif isinstance(filename_or_conn, (str, pathlib.Path)): if recreate and os.path.exists(filename_or_conn): - os.remove(filename_or_conn) + try: + os.remove(filename_or_conn) + except OSError: + # Avoid mypy and __repr__ errors, see: + # https://github.com/simonw/sqlite-utils/issues/503 + self.conn = sqlite3.connect(":memory:") + raise self.conn = sqlite3.connect(str(filename_or_conn)) else: - assert not recreate, "recreate cannot be used with connections, only paths" - self.conn = filename_or_conn - self._tracer = tracer + if recreate: + raise ValueError("recreate cannot be used with connections, only paths") + self.conn = cast(sqlite3.Connection, filename_or_conn) + # Python 3.12+ autocommit=True/False connections make commit() + # and rollback() behave differently, silently breaking the + # transaction handling used by every write method + autocommit = getattr(self.conn, "autocommit", None) + if autocommit is not None and autocommit != getattr( + sqlite3, "LEGACY_TRANSACTION_CONTROL", -1 + ): + raise TransactionError( + "sqlite-utils requires a connection that uses the default " + "transaction handling - connections created with " + "autocommit=True or autocommit=False are not supported" + ) + self._tracer: Optional[Tracer] = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") - self._registered_functions = set() + self._registered_functions: set = set() self.use_counts_table = use_counts_table + if execute_plugins: + ensure_plugins_loaded() + pm.hook.prepare_connection(conn=self.conn) + self.strict = strict + + def __enter__(self) -> "Database": + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[object], + ) -> None: + self.close() + + def close(self) -> None: + "Close the SQLite connection, and the underlying database file" + self.conn.close() @contextlib.contextmanager - def tracer(self, tracer=None): + def atomic(self) -> Generator["Database", None, None]: + """ + Context manager for wrapping multiple database operations in a transaction. + + Nested blocks use SQLite savepoints. + """ + if self.conn.in_transaction: + savepoint = "sqlite_utils_{}".format(secrets.token_hex(16)) + self.conn.execute("SAVEPOINT {};".format(savepoint)) + try: + yield self + except BaseException: + # An error such as a RAISE(ROLLBACK) trigger can destroy + # the whole transaction, savepoints included - cleaning up + # anyway would mask the original exception with + # "no such savepoint" + if self.conn.in_transaction: + self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) + self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + raise + else: + self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + else: + self.conn.execute("BEGIN") + try: + yield self + except BaseException: + # rollback() is a no-op if the error already destroyed the + # transaction, so the original exception propagates + self.rollback() + raise + else: + try: + self.conn.execute("COMMIT") + except BaseException: + self.rollback() + raise + + def begin(self) -> None: + """ + Start a transaction with ``BEGIN``, taking manual control of transaction + handling. End it by calling :meth:`commit` or :meth:`rollback`. + + Raises ``sqlite3.OperationalError`` if a transaction is already open. + + Most code should use the :meth:`atomic` context manager instead, which + commits and rolls back automatically. See :ref:`python_api_transactions`. + """ + self.execute("BEGIN") + + def commit(self) -> None: + """ + Commit the current transaction. Does nothing if no transaction is open. + """ + if self.conn.in_transaction: + self.conn.execute("COMMIT") + + def rollback(self) -> None: + """ + Roll back the current transaction, discarding its changes. Does nothing + if no transaction is open. + """ + if self.conn.in_transaction: + self.conn.execute("ROLLBACK") + + @contextlib.contextmanager + def ensure_autocommit_on(self) -> Generator[None, None, None]: + """ + Ensure the connection is in driver-level autocommit mode for the + duration of a block of code. + + This temporarily sets ``isolation_level = None`` on the underlying + ``sqlite3`` connection, so the driver does not open implicit + transactions. This is useful for statements such as + ``PRAGMA journal_mode=wal`` which cannot run inside a transaction. + + Example usage:: + + with db.ensure_autocommit_on(): + # do stuff here + + The previous ``isolation_level`` is restored at the end of the block. + + :raises TransactionError: if a transaction is open - assigning + ``isolation_level`` would commit it as a side effect, silently + breaking the caller's ability to roll back + """ + if self.conn.in_transaction: + raise TransactionError( + "ensure_autocommit_on() cannot be used inside a transaction - " + "changing isolation_level would commit the open transaction" + ) + old_isolation_level = self.conn.isolation_level + try: + self.conn.isolation_level = None + yield + finally: + self.conn.isolation_level = old_isolation_level + + @contextlib.contextmanager + def tracer( + self, tracer: Optional[Tracer] = None + ) -> Generator["Database", None, None]: + """ + Context manager to temporarily set a tracer function - all executed SQL queries will + be passed to this. + + The tracer function should accept two arguments: ``sql`` and ``parameters`` + + Example usage:: + + with db.tracer(print): + db["creatures"].insert({"name": "Cleo"}) + + See :ref:`python_api_tracing`. + + :param tracer: Callable accepting ``sql`` and ``parameters`` arguments + """ prev_tracer = self._tracer - self._tracer = tracer or print + self._tracer = tracer or cast(Tracer, print) try: yield self finally: self._tracer = prev_tracer - def __getitem__(self, table_name): + def __getitem__(self, table_name: str) -> Union["Table", "View"]: + """ + ``db[name]`` returns a :class:`.Table` object for the table with the specified name, + or a :class:`.View` object if the name matches an existing SQL view. + If neither exists yet, a table is assumed - it will be created the first + time data is inserted into it. + + :param table_name: The name of the table or view + """ + if table_name in self.view_names(): + return self.view(table_name) return self.table(table_name) - def __repr__(self): + def __repr__(self) -> str: return "".format(self.conn) - def register_function(self, fn=None, deterministic=None, replace=False): - def register(fn): - name = fn.__name__ + def register_function( + self, + fn: Optional[Callable] = None, + deterministic: bool = False, + replace: bool = False, + name: Optional[str] = None, + ) -> Optional[Callable[[Callable], Callable]]: + """ + ``fn`` will be made available as a function within SQL, with the same name and number + of arguments. Can be used as a decorator:: + + @db.register_function + def upper(value): + return str(value).upper() + + The decorator can take arguments:: + + @db.register_function(deterministic=True, replace=True) + def upper(value): + return str(value).upper() + + See :ref:`python_api_register_function`. + + :param fn: Function to register + :param deterministic: set ``True`` for functions that always returns the same output for a given input + :param replace: set ``True`` to replace an existing function with the same name - otherwise throw an error + :param name: name of the SQLite function - if not specified, the Python function name will be used + """ + + def register(fn: Callable) -> Callable: + fn_name = name or fn.__name__ # type: ignore arity = len(inspect.signature(fn).parameters) - if not replace and (name, arity) in self._registered_functions: + if not replace and (fn_name, arity) in self._registered_functions: return fn - kwargs = {} - if deterministic and sys.version_info >= (3, 8): - kwargs["deterministic"] = True - self.conn.create_function(name, arity, fn, **kwargs) - self._registered_functions.add((name, arity)) + kwargs: Dict[str, bool] = {} + registered = False + if deterministic: + # Try this, but fall back if sqlite3.NotSupportedError + try: + self.conn.create_function( + fn_name, arity, fn, **dict(kwargs, deterministic=True) + ) + registered = True + except sqlite3.NotSupportedError: + pass + if not registered: + self.conn.create_function(fn_name, arity, fn, **kwargs) + self._registered_functions.add((fn_name, arity)) return fn if fn is None: return register else: register(fn) + return None - def register_fts4_bm25(self): - self.register_function(rank_bm25, deterministic=True) + def register_fts4_bm25(self) -> None: + "Register the ``rank_bm25(match_info)`` function used for calculating relevance with SQLite FTS4." + self.register_function(rank_bm25, deterministic=True, replace=True) - def attach(self, alias, filepath): + def attach(self, alias: str, filepath: Union[str, pathlib.Path]) -> None: + """ + Attach another SQLite database file to this connection with the specified alias, equivalent to:: + + ATTACH DATABASE 'filepath.db' AS alias + + :param alias: Alias name to use + :param filepath: Path to SQLite database file on disk + """ attach_sql = """ - ATTACH DATABASE '{}' AS [{}]; + ATTACH DATABASE '{}' AS {}; """.format( - str(pathlib.Path(filepath).resolve()), alias + str(pathlib.Path(filepath).resolve()), quote_identifier(alias) ).strip() self.execute(attach_sql) - def execute(self, sql, parameters=None): + def query( + self, sql: str, params: Optional[Union[Sequence, Dict[str, Any]]] = None + ) -> Generator[dict, None, None]: + """ + Execute ``sql`` and return an iterable of dictionaries representing each row. + + The SQL is executed as soon as this method is called - the resulting rows + are then fetched lazily as the returned iterable is iterated over. A + row-returning write such as ``INSERT ... RETURNING`` takes effect + immediately, even if the results are never iterated. + + :param sql: SQL query to execute + :param params: Parameters to use in that query - an iterable for ``where id = ?`` + parameters, or a dictionary for ``where id = :id`` + :raises ValueError: if the SQL statement does not return rows - use + :meth:`execute` for those statements instead. The rejected statement + is rolled back, so it has no effect on the database. One exception: + a row-less ``PRAGMA`` statement takes effect despite the + ``ValueError``, because PRAGMAs run outside the savepoint guard - + some of them refuse to run inside a transaction + """ + message = ( + "query() can only be used with SQL that returns rows - " + "use execute() for other statements" + ) + keyword = _first_keyword(sql) + if keyword in _QUERY_REJECTED_KEYWORDS: + # None of these return rows - reject them without executing anything + raise ValueError(message) + if self._tracer: + self._tracer(sql, params) + args: tuple = (params,) if params is not None else () + if keyword == "PRAGMA": + # Some PRAGMA statements refuse to run inside a transaction, so + # execute these without the savepoint guard used below. Some + # adapters open an implicit transaction before comment-prefixed + # PRAGMAs, so temporarily use driver autocommit when it is safe. + if self.conn.in_transaction: + cursor = self.conn.execute(sql, *args) + else: + with self.ensure_autocommit_on(): + cursor = self.conn.execute(sql, *args) + if cursor.description is None: + raise ValueError(message) + keys = dedupe_keys(d[0] for d in cursor.description) + return (dict(zip(keys, row)) for row in cursor) + # Execute inside a savepoint, so a statement that turns out not to + # return rows can be rolled back before the ValueError is raised + self.conn.execute('SAVEPOINT "sqlite_utils_query"') + released = False + try: + cursor = self.conn.execute(sql, *args) + if cursor.description is None: + raise ValueError(message) + keys = dedupe_keys(d[0] for d in cursor.description) + try: + self.conn.execute('RELEASE "sqlite_utils_query"') + released = True + except sqlite3.OperationalError: + # The savepoint cannot be released while a write statement is + # still executing - this is INSERT ... RETURNING or similar, + # with unfetched rows. Fetch them so the write completes, then + # release again - committing the write immediately, unless an + # outer transaction is open + fetched = cursor.fetchall() + self.conn.execute('RELEASE "sqlite_utils_query"') + released = True + return (dict(zip(keys, row)) for row in fetched) + return (dict(zip(keys, row)) for row in cursor) + finally: + if not released and self.conn.in_transaction: + # An error occurred - undo anything the statement changed. + # If the error itself destroyed the transaction (such as a + # RAISE(ROLLBACK) trigger) the savepoint is already gone + # and there is nothing left to undo + self.conn.execute('ROLLBACK TO "sqlite_utils_query"') + self.conn.execute('RELEASE "sqlite_utils_query"') + + def execute( + self, sql: str, parameters: Optional[Union[Sequence, Dict[str, Any]]] = None + ) -> sqlite3.Cursor: + """ + Execute SQL query and return a ``sqlite3.Cursor``. + + A write statement - ``INSERT``, ``UPDATE``, ``CREATE TABLE`` and so on - + is committed automatically, unless a transaction is already open, in + which case it becomes part of that transaction. See + :ref:`python_api_transactions`. + + :param sql: SQL query to execute + :param parameters: Parameters to use in that query - an iterable for ``where id = ?`` + parameters, or a dictionary for ``where id = :id`` + """ if self._tracer: self._tracer(sql, parameters) - if parameters is not None: - return self.conn.execute(sql, parameters) - else: - return self.conn.execute(sql) + was_in_transaction = self.conn.in_transaction + try: + if parameters is not None: + cursor = self.conn.execute(sql, parameters) + else: + cursor = self.conn.execute(sql) + except Exception: + if not was_in_transaction and self.conn.in_transaction: + # The failed statement opened an implicit transaction that + # nothing would ever commit - roll it back, otherwise it + # would capture every subsequent write + self.conn.execute("ROLLBACK") + raise + if ( + not was_in_transaction + and self.conn.in_transaction + and cursor.description is None + and _first_keyword(sql) not in _TRANSACTION_CONTROL_KEYWORDS + ): + # The statement opened an implicit transaction - commit it, so + # that execute() behaves consistently with the rest of the + # library and identically across connection modes + self.conn.execute("COMMIT") + return cursor - def executescript(self, sql): + def executescript(self, sql: str) -> sqlite3.Cursor: + """ + Execute multiple SQL statements separated by ; and return the ``sqlite3.Cursor``. + + :param sql: SQL to execute + """ if self._tracer: self._tracer(sql, None) + return self._executescript(sql) + + def _executescript(self, sql: str) -> sqlite3.Cursor: + if self.conn.in_transaction: + cursor = self.conn.cursor() + # avoid sqlite3.executescript()'s implicit commit: + for statement in _iter_complete_sql_statements(sql): + cursor.execute(statement) + return cursor return self.conn.executescript(sql) - def table(self, table_name, **kwargs): - klass = View if table_name in self.view_names() else Table - return klass(self, table_name, **kwargs) + def table(self, table_name: str, **kwargs: Any) -> "Table": + """ + Return a table object, optionally configured with default options. - def quote(self, value): + See :ref:`reference_db_table` for option details. + + :param table_name: Name of the table + """ + if table_name in self.view_names(): + raise NoTable("Table {} is actually a view".format(table_name)) + kwargs.setdefault("strict", self.strict) + return Table(self, table_name, **kwargs) + + def view(self, view_name: str) -> "View": + """ + Return a view object. + + :param view_name: Name of the view + """ + if view_name not in self.view_names(): + if view_name in self.table_names(): + raise NoView( + "View {name} does not exist - {name} is a table".format( + name=view_name + ) + ) + raise NoView("View {} does not exist".format(view_name)) + return View(self, view_name) + + def quote(self, value: str) -> str: + """ + Apply SQLite string quoting to a value, including wrapping it in single quotes. + + :param value: String to quote + """ # Normally we would use .execute(sql, [params]) for escaping, but # occasionally that isn't available - most notable when we need # to include a "... DEFAULT 'value'" in a column definition. @@ -277,7 +995,58 @@ class Database: {"value": value}, ).fetchone()[0] - def table_names(self, fts4=False, fts5=False): + def quote_fts(self, query: str) -> str: + """ + Escape special characters in a SQLite full-text search query. + + This works by surrounding each token within the query with double + quotes, in order to avoid words like ``NOT`` and ``OR`` having + special meaning as defined by the FTS query syntax here: + + https://www.sqlite.org/fts5.html#full_text_query_syntax + + If the query has unbalanced ``"`` characters, adds one at end. + + :param query: String to escape + """ + if query.count('"') % 2: + query += '"' + bits = _quote_fts_re.split(query) + bits = [b for b in bits if b and b != '""'] + return " ".join( + '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits + ) + + def quote_default_value(self, value: str) -> str: + if any( + [ + str(value).startswith("'") and str(value).endswith("'"), + str(value).startswith('"') and str(value).endswith('"'), + ] + ): + return value + + if str(value).upper() in ("CURRENT_TIME", "CURRENT_DATE", "CURRENT_TIMESTAMP"): + return value + + if isinstance(value, str) and value.upper() in ("TRUE", "FALSE", "NULL"): + # Keyword literals must stay unquoted; quoting them would turn the + # default into a string ('TRUE' instead of 1, 'NULL' instead of null). + return value + + if str(value).endswith(")"): + # Expr + return "({})".format(value) + + return self.quote(value) + + def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: + """ + List of string table names in this database. + + :param fts4: Only return tables that are part of FTS4 indexes + :param fts5: Only return tables that are part of FTS5 indexes + """ where = ["type = 'table'"] if fts4: where.append("sql like '%USING FTS4%'") @@ -286,7 +1055,8 @@ class Database: sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.execute(sql).fetchall()] - def view_names(self): + def view_names(self) -> List[str]: + "List of string view names in this database." return [ r[0] for r in self.execute( @@ -295,15 +1065,18 @@ class Database: ] @property - def tables(self): - return [self[name] for name in self.table_names()] + def tables(self) -> List["Table"]: + "List of Table objects in this database." + return [self.table(name) for name in self.table_names()] @property - def views(self): - return [self[name] for name in self.view_names()] + def views(self) -> List["View"]: + "List of View objects in this database." + return [self.view(name) for name in self.view_names()] @property - def triggers(self): + def triggers(self) -> List[Trigger]: + "List of ``(name, table_name, sql)`` tuples representing triggers in this database." return [ Trigger(*r) for r in self.execute( @@ -312,12 +1085,13 @@ class Database: ] @property - def triggers_dict(self): - "Returns {trigger_name: sql} dictionary" + def triggers_dict(self) -> Dict[str, str]: + "A ``{trigger_name: sql}`` dictionary of triggers in this database." return {trigger.name: trigger.sql for trigger in self.triggers} @property - def schema(self): + def schema(self) -> str: + "SQL schema for this database." sqls = [] for row in self.execute( "select sql from sqlite_master where sql is not null" @@ -329,22 +1103,106 @@ class Database: return "\n".join(sqls) @property - def journal_mode(self): + def supports_strict(self) -> bool: + "Does this database support STRICT mode?" + if not hasattr(self, "_supports_strict"): + try: + table_name = "t{}".format(secrets.token_hex(16)) + with self.atomic(): + self.conn.execute( + "create table {} (name text) strict".format(table_name) + ) + self.conn.execute("drop table {}".format(table_name)) + self._supports_strict = True + except Exception: + self._supports_strict = False + return self._supports_strict + + @property + def supports_on_conflict(self) -> bool: + # SQLite's upsert is implemented as INSERT INTO ... ON CONFLICT DO ... + if not hasattr(self, "_supports_on_conflict"): + table_name = "t{}".format(secrets.token_hex(16)) + try: + with self.atomic(): + self.conn.execute( + "create table {} (id integer primary key, name text)".format( + table_name + ) + ) + self.conn.execute( + "insert into {} (id, name) values (1, 'one')".format(table_name) + ) + self.conn.execute( + ( + "insert into {} (id, name) values (1, 'two') " + "on conflict do update set name = 'two'" + ).format(table_name) + ) + self._supports_on_conflict = True + except Exception: + self._supports_on_conflict = False + finally: + self.conn.execute("drop table if exists {}".format(table_name)) + return self._supports_on_conflict + + @property + def sqlite_version(self) -> Tuple[int, ...]: + "Version of SQLite, as a tuple of integers for example ``(3, 36, 0)``." + row = self.execute("select sqlite_version()").fetchall()[0] + return tuple(map(int, row[0].split("."))) + + @property + def journal_mode(self) -> str: + """ + Current ``journal_mode`` of this database. + + https://www.sqlite.org/pragma.html#pragma_journal_mode + """ return self.execute("PRAGMA journal_mode;").fetchone()[0] - def enable_wal(self): + def enable_wal(self) -> None: + """ + Sets ``journal_mode`` to ``'wal'`` to enable Write-Ahead Log mode. + + :raises TransactionError: if called while a transaction is open - the + journal mode can only be changed outside of a transaction + """ if self.journal_mode != "wal": - self.execute("PRAGMA journal_mode=wal;") + self._ensure_no_open_transaction("enable_wal()") + with self.ensure_autocommit_on(): + self.execute("PRAGMA journal_mode=wal;") - def disable_wal(self): + def disable_wal(self) -> None: + """ + Sets ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode. + + :raises TransactionError: if called while a transaction is open - the + journal mode can only be changed outside of a transaction + """ if self.journal_mode != "delete": - self.execute("PRAGMA journal_mode=delete;") + self._ensure_no_open_transaction("disable_wal()") + with self.ensure_autocommit_on(): + self.execute("PRAGMA journal_mode=delete;") - def _ensure_counts_table(self): - with self.conn: + def _ensure_no_open_transaction(self, operation: str) -> None: + # Changing journal mode assigns conn.isolation_level, which commits + # any open transaction as a side effect - breaking the rollback + # guarantee of atomic() and of user-managed transactions + if self.conn.in_transaction: + raise TransactionError( + "{} cannot be used while a transaction is open".format(operation) + ) + + def _ensure_counts_table(self) -> None: + with self.atomic(): self.execute(_COUNTS_TABLE_CREATE_SQL.format(self._counts_table_name)) - def enable_counts(self): + def enable_counts(self) -> None: + """ + Enable trigger-based count caching for every table in the database, see + :ref:`python_api_cached_table_counts`. + """ self._ensure_counts_table() for table in self.tables: if ( @@ -354,94 +1212,222 @@ class Database: table.enable_counts() self.use_counts_table = True - def cached_counts(self, tables=None): - sql = "select [table], count from {}".format(self._counts_table_name) - if tables: - sql += " where [table] in ({})".format(", ".join("?" for table in tables)) + def cached_counts(self, tables: Optional[Iterable[str]] = None) -> Dict[str, int]: + """ + Return ``{table_name: count}`` dictionary of cached counts for specified tables, or + all tables if ``tables`` not provided. + + :param tables: Subset list of tables to return counts for. + """ + sql = 'select "table", count from {}'.format(self._counts_table_name) + tables_list = list(tables) if tables else None + if tables_list: + sql += ' where "table" in ({})'.format(", ".join("?" for _ in tables_list)) try: - return {r[0]: r[1] for r in self.execute(sql, tables).fetchall()} + return {r[0]: r[1] for r in self.execute(sql, tables_list).fetchall()} except OperationalError: return {} - def reset_counts(self): + def reset_counts(self) -> None: + "Re-calculate cached counts for tables." tables = [table for table in self.tables if table.has_counts_triggers] - with self.conn: + with self.atomic(): self._ensure_counts_table() - counts_table = self[self._counts_table_name] + counts_table = self.table(self._counts_table_name) counts_table.delete_where() counts_table.insert_all( {"table": table.name, "count": table.execute_count()} for table in tables ) - def query( - self, sql: str, params: Optional[Union[Iterable, dict]] = None - ) -> Generator[dict, None, None]: - cursor = self.execute(sql, params or tuple()) - keys = [d[0] for d in cursor.description] - for row in cursor: - yield dict(zip(keys, row)) - def execute_returning_dicts( - self, sql: str, params: Optional[Union[Iterable, dict]] = None + self, sql: str, params: Optional[Union[Sequence, Dict[str, Any]]] = None ) -> List[dict]: return list(self.query(sql, params)) - def resolve_foreign_keys(self, name, foreign_keys): - # foreign_keys may be a list of strcolumn names, a list of ForeignKey tuples, - # a list of tuple-pairs or a list of tuple-triples. We want to turn - # it into a list of ForeignKey tuples - if all(isinstance(fk, ForeignKey) for fk in foreign_keys): - return foreign_keys - if all(isinstance(fk, str) for fk in foreign_keys): - # It's a list of columns - fks = [] - for column in foreign_keys: - other_table = self[name].guess_foreign_table(column) - other_column = self[name].guess_foreign_column(other_table) - fks.append(ForeignKey(name, column, other_table, other_column)) - return fks - assert all( - isinstance(fk, (tuple, list)) for fk in foreign_keys - ), "foreign_keys= should be a list of tuples" + def resolve_foreign_keys( + self, name: str, foreign_keys: ForeignKeysType + ) -> List[ForeignKey]: + """ + Given a list of differing foreign_keys definitions, return a list of + fully resolved ForeignKey() named tuples. + + :param name: Name of table that foreign keys are being defined for + :param foreign_keys: List of foreign keys, each of which can be a + string, a ForeignKey() object, a tuple of (column, other_table), + or a tuple of (column, other_table, other_column), or a tuple of + (table, column, other_table, other_column). For compound foreign + keys the column elements can be tuples of column names, e.g. + (("campus_name", "dept_code"), "departments") or + (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) + """ + table = self.table(name) fks = [] - for tuple_or_list in foreign_keys: - assert len(tuple_or_list) in ( - 2, - 3, - ), "foreign_keys= should be a list of tuple pairs or triples" - if len(tuple_or_list) == 3: - fks.append( - ForeignKey( - name, tuple_or_list[0], tuple_or_list[1], tuple_or_list[2] + for fk in foreign_keys: + if isinstance(fk, ForeignKey): + fks.append(fk) + continue + if isinstance(fk, str): + # A bare column name - guess the other table and column + other_table = table.guess_foreign_table(fk) + other_column = table.guess_foreign_column(other_table) + fks.append(ForeignKey(name, fk, other_table, other_column)) + continue + if not isinstance(fk, (tuple, list)): + raise ValueError( + "foreign_keys= should be a list of tuples, " + "ForeignKey objects or column name strings" + ) + tuple_or_list = cast(Sequence[Any], fk) + if len(tuple_or_list) == 4: + if tuple_or_list[0] != name: + raise ValueError( + "First item in {} should have been {}".format( + tuple_or_list, name + ) ) + tuple_or_list = tuple_or_list[1:] + if len(tuple_or_list) not in (2, 3): + raise ValueError( + "foreign_keys= should be a list of tuple pairs or triples" + ) + column_or_columns = tuple_or_list[0] + other_table = tuple_or_list[1] + if isinstance(column_or_columns, (list, tuple)): + # Compound foreign key + columns = tuple(column_or_columns) + if len(tuple_or_list) == 3: + if not isinstance(tuple_or_list[2], (list, tuple)): + raise ValueError( + "Compound foreign key {} should reference a tuple " + "of other columns".format(tuple(tuple_or_list)) + ) + other_columns = tuple(tuple_or_list[2]) + else: + # Guess the compound primary key of the other table + other_columns = tuple(self.table(other_table).pks) + if len(columns) != len(other_columns): + raise ValueError( + "Compound foreign key {} should have the same number " + "of columns on both sides".format(tuple(tuple_or_list)) + ) + if len(columns) == 1: + # Single-column key passed as a one-item list + fks.append( + ForeignKey(name, columns[0], other_table, other_columns[0]) + ) + else: + fks.append( + ForeignKey( + name, + None, + other_table, + None, + columns=columns, + other_columns=other_columns, + is_compound=True, + ) + ) + elif len(tuple_or_list) == 3: + fks.append( + ForeignKey(name, column_or_columns, other_table, tuple_or_list[2]) ) else: # Guess the primary key fks.append( ForeignKey( name, - tuple_or_list[0], - tuple_or_list[1], - self[name].guess_foreign_column(tuple_or_list[1]), + column_or_columns, + other_table, + table.guess_foreign_column(other_table), ) ) return fks + def _resolve_foreign_key_casing( + self, fk: ForeignKey, columns: Iterable[str] + ) -> ForeignKey: + """ + Return ``fk`` with its column references resolved to match the casing + of the actual columns. ``columns`` provides the column names of + ``fk.table``, which may be a table that is still being created. + """ + resolved_columns = tuple(resolve_casing(c, columns) for c in fk.columns) + if fk.other_table == fk.table: + other_candidates: Iterable[str] = columns + else: + other_candidates = self[fk.other_table].columns_dict + resolved_other_columns = tuple( + resolve_casing(c, other_candidates) for c in fk.other_columns + ) + if ( + resolved_columns == fk.columns + and resolved_other_columns == fk.other_columns + ): + return fk + if fk.is_compound: + return ForeignKey( + fk.table, + None, + fk.other_table, + None, + columns=resolved_columns, + other_columns=resolved_other_columns, + is_compound=True, + on_delete=fk.on_delete, + on_update=fk.on_update, + ) + return ForeignKey( + fk.table, + resolved_columns[0], + fk.other_table, + resolved_other_columns[0], + on_delete=fk.on_delete, + on_update=fk.on_update, + ) + def create_table_sql( self, - name, - columns, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - hash_id=None, - extracts=None, - ): - foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) - foreign_keys_by_column = {fk.column: fk for fk in foreign_keys} + name: str, + columns: Dict[str, Any], + pk: Optional[Any] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, + if_not_exists: bool = False, + strict: bool = False, + ) -> str: + """ + Returns the SQL ``CREATE TABLE`` statement for creating the specified table. + + :param name: Name of table + :param columns: Dictionary mapping column names to their types, for example ``{"name": str, "age": int}`` + :param pk: String name of column to use as a primary key, or a tuple of strings for a compound primary key covering multiple columns + :param foreign_keys: List of foreign key definitions for this table + :param column_order: List specifying which columns should come first + :param not_null: List of columns that should be created as ``NOT NULL`` + :param defaults: Dictionary specifying default values for columns + :param hash_id: Name of column to be used as a primary key containing a hash of the other columns + :param hash_id_columns: List of columns to be used when calculating the hash ID for a row + :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts` + :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS`` + :param strict: Apply STRICT mode to table + """ + if hash_id_columns and (hash_id is None): + hash_id = "id" + resolved_fks: List[ForeignKey] = [ + self._resolve_foreign_key_casing(fk, columns) + for fk in self.resolve_foreign_keys(name, foreign_keys or []) + ] + # Compound foreign keys are rendered as table-level constraints; + # single-column ones as inline REFERENCES on their column + foreign_keys_by_column = { + fk.column: fk for fk in resolved_fks if not fk.is_compound + } # any extracts will be treated as integer columns with a foreign key extracts = resolve_extracts(extracts) for extract_column, extract_table in extracts.items(): @@ -455,45 +1441,57 @@ class Database: name, extract_column, extract_table, "id" ) # Soundness check not_null, and defaults if provided - not_null = not_null or set() - defaults = defaults or {} - assert all( - n in columns for n in not_null - ), "not_null set {} includes items not in columns {}".format( - repr(not_null), repr(set(columns.keys())) - ) - assert all( - n in columns for n in defaults - ), "defaults set {} includes items not in columns {}".format( - repr(set(defaults)), repr(set(columns.keys())) - ) - validate_column_names(columns.keys()) + not_null = {resolve_casing(n, columns) for n in not_null or set()} + defaults = {resolve_casing(n, columns): v for n, v in (defaults or {}).items()} + if column_order is not None: + column_order = [resolve_casing(c, columns) for c in column_order] + if not columns: + raise ValueError("Tables must have at least one column") + if not all(n in columns for n in not_null): + raise ValueError( + "not_null set {} includes items not in columns {}".format( + repr(not_null), repr(set(columns.keys())) + ) + ) + if not all(n in columns for n in defaults): + raise ValueError( + "defaults set {} includes items not in columns {}".format( + repr(set(defaults)), repr(set(columns.keys())) + ) + ) column_items = list(columns.items()) if column_order is not None: - column_items.sort( - key=lambda p: column_order.index(p[0]) if p[0] in column_order else 999 - ) + + def sort_key(p): + return column_order.index(p[0]) if p[0] in column_order else 999 + + column_items.sort(key=sort_key) if hash_id: column_items.insert(0, (hash_id, str)) pk = hash_id # Soundness check foreign_keys point to existing tables - for fk in foreign_keys: - if not any( - c for c in self[fk.other_table].columns if c.name == fk.other_column - ): - raise AlterError( - "No such column: {}.{}".format(fk.other_table, fk.other_column) - ) + for fk in resolved_fks: + for other_column in fk.other_columns: + if fk.other_table == name and columns.get(other_column): + continue + if other_column != "rowid" and not any( + c for c in self[fk.other_table].columns if c.name == other_column + ): + raise AlterError( + "No such column: {}.{}".format(fk.other_table, other_column) + ) column_defs = [] # ensure pk is a tuple single_pk = None - if isinstance(pk, list) and len(pk) == 1 and isinstance(pk[0], str): + if isinstance(pk, (list, tuple)) and len(pk) == 1 and isinstance(pk[0], str): pk = pk[0] if isinstance(pk, str): - single_pk = pk + single_pk = pk = resolve_casing(pk, [c[0] for c in column_items]) if pk not in [c[0] for c in column_items]: column_items.insert(0, (pk, int)) + elif pk: + pk = [resolve_casing(p, [c[0] for c in column_items]) for p in pk] for column_name, column_type in column_items: column_extras = [] if column_name == single_pk: @@ -502,50 +1500,181 @@ class Database: column_extras.append("NOT NULL") if column_name in defaults and defaults[column_name] is not None: column_extras.append( - "DEFAULT {}".format(self.quote(defaults[column_name])) + "DEFAULT {}".format(self.quote_default_value(defaults[column_name])) ) if column_name in foreign_keys_by_column: + fk = foreign_keys_by_column[column_name] column_extras.append( - "REFERENCES [{other_table}]([{other_column}])".format( - other_table=foreign_keys_by_column[column_name].other_table, - other_column=foreign_keys_by_column[column_name].other_column, + "REFERENCES {}({}){}".format( + quote_identifier(fk.other_table), + quote_identifier(cast(str, fk.other_column)), + _fk_actions_sql(fk), ) ) + column_type_str = COLUMN_TYPE_MAPPING[column_type] + # Special case for strict tables to map FLOAT to REAL + # Refs https://github.com/simonw/sqlite-utils/issues/644 + if strict and column_type_str == "FLOAT": + column_type_str = "REAL" column_defs.append( - " [{column_name}] {column_type}{column_extras}".format( - column_name=column_name, - column_type=COLUMN_TYPE_MAPPING[column_type], - column_extras=(" " + " ".join(column_extras)) - if column_extras - else "", + " {} {column_type}{column_extras}".format( + quote_identifier(column_name), + column_type=column_type_str, + column_extras=( + (" " + " ".join(column_extras)) if column_extras else "" + ), ) ) extra_pk = "" if single_pk is None and pk and len(pk) > 1: extra_pk = ",\n PRIMARY KEY ({pks})".format( - pks=", ".join(["[{}]".format(p) for p in pk]) + pks=", ".join([quote_identifier(p) for p in pk]) + ) + # Compound foreign keys become table-level FOREIGN KEY constraints + column_names = [c[0] for c in column_items] + for fk in resolved_fks: + if not fk.is_compound: + continue + missing = [c for c in fk.columns if c not in column_names] + if missing: + raise AlterError( + "No such column: {}".format(", ".join(sorted(missing))) + ) + column_defs.append( + " FOREIGN KEY ({columns}) REFERENCES {other_table}({other_columns}){actions}".format( + columns=", ".join(quote_identifier(c) for c in fk.columns), + other_table=quote_identifier(fk.other_table), + other_columns=", ".join( + quote_identifier(c) for c in fk.other_columns + ), + actions=_fk_actions_sql(fk), + ) ) columns_sql = ",\n".join(column_defs) - sql = """CREATE TABLE [{table}] ( + sql = """CREATE TABLE {if_not_exists}{table} ( {columns_sql}{extra_pk} -); +){strict}; """.format( - table=name, columns_sql=columns_sql, extra_pk=extra_pk + if_not_exists="IF NOT EXISTS " if if_not_exists else "", + table=quote_identifier(name), + columns_sql=columns_sql, + extra_pk=extra_pk, + strict=" STRICT" if strict and self.supports_strict else "", ) return sql def create_table( self, - name, - columns, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - hash_id=None, - extracts=None, - ): + name: str, + columns: Dict[str, Any], + pk: Optional[Any] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, + if_not_exists: bool = False, + replace: bool = False, + ignore: bool = False, + transform: bool = False, + strict: bool = False, + ) -> "Table": + """ + Create a table with the specified name and the specified ``{column_name: type}`` columns. + + See :ref:`python_api_explicit_create`. + + :param name: Name of table + :param columns: Dictionary mapping column names to their types, for example ``{"name": str, "age": int}`` + :param pk: String name of column to use as a primary key, or a tuple of strings for a compound primary key covering multiple columns + :param foreign_keys: List of foreign key definitions for this table + :param column_order: List specifying which columns should come first + :param not_null: List of columns that should be created as ``NOT NULL`` + :param defaults: Dictionary specifying default values for columns + :param hash_id: Name of column to be used as a primary key containing a hash of the other columns + :param hash_id_columns: List of columns to be used when calculating the hash ID for a row + :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts` + :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS`` + :param replace: Drop and replace table if it already exists + :param ignore: Silently do nothing if table already exists + :param transform: If table already exists transform it to fit the specified schema + :param strict: Apply STRICT mode to table + """ + # Transform table to match the new definition if table already exists: + if self[name].exists(): + if ignore: + return self.table(name) + elif replace: + self[name].drop() + if transform and self[name].exists(): + table = self.table(name) + should_transform = False + # First add missing columns and figure out columns to drop + existing_columns = table.columns_dict + # Match existing columns case-insensitively, the way SQLite does + columns = { + resolve_casing(col_name, existing_columns): col_type + for col_name, col_type in columns.items() + } + missing_columns = dict( + (col_name, col_type) + for col_name, col_type in columns.items() + if col_name not in existing_columns + ) + columns_to_drop = [ + column for column in existing_columns if column not in columns + ] + if missing_columns: + for col_name, col_type in missing_columns.items(): + table.add_column(col_name, col_type) + if missing_columns or columns_to_drop or columns != existing_columns: + should_transform = True + # Do we need to change the column order? + if ( + column_order + and list(existing_columns)[: len(column_order)] != column_order + ): + should_transform = True + # Has the primary key changed? + current_pks = table.pks + desired_pk = None + if isinstance(pk, str): + desired_pk = [resolve_casing(pk, existing_columns)] + elif pk: + desired_pk = [resolve_casing(p, existing_columns) for p in pk] + if desired_pk and current_pks != desired_pk: + should_transform = True + # Any not-null changes? + current_not_null = {c.name for c in table.columns if c.notnull} + desired_not_null = ( + {resolve_casing(n, existing_columns) for n in not_null} + if not_null + else set() + ) + if current_not_null != desired_not_null: + should_transform = True + # How about defaults? + if ( + defaults + and { + resolve_casing(c, existing_columns): v for c, v in defaults.items() + } + != table.default_values + ): + should_transform = True + # Only run .transform() if there is something to do + if should_transform: + table.transform( + types=columns, + drop=columns_to_drop, + column_order=column_order, + not_null=not_null, + defaults=defaults, + pk=pk, + ) + return table sql = self.create_table_sql( name=name, columns=columns, @@ -555,7 +1684,10 @@ class Database: not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, extracts=extracts, + if_not_exists=if_not_exists, + strict=strict, ) self.execute(sql) return self.table( @@ -566,13 +1698,38 @@ class Database: not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, ) - def create_view(self, name, sql, ignore=False, replace=False): - assert not ( - ignore and replace - ), "Use one or the other of ignore/replace, not both" - create_sql = "CREATE VIEW {name} AS {sql}".format(name=name, sql=sql) + def rename_table(self, name: str, new_name: str) -> None: + """ + Rename a table. + + :param name: Current table name + :param new_name: Name to rename it to + """ + self.execute( + "ALTER TABLE {} RENAME TO {}".format( + quote_identifier(name), quote_identifier(new_name) + ) + ) + + def create_view( + self, name: str, sql: str, ignore: bool = False, replace: bool = False + ) -> "Database": + """ + Create a new SQL view with the specified name - ``sql`` should start with ``SELECT ...``. + + :param name: Name of the view + :param sql: SQL ``SELECT`` query to use for this view. + :param ignore: Set to ``True`` to do nothing if a view with this name already exists + :param replace: Set to ``True`` to replace the view if one with this name already exists + """ + if ignore and replace: + raise ValueError("Use one or the other of ignore/replace, not both") + create_sql = "CREATE VIEW {name} AS {sql}".format( + name=quote_identifier(name), sql=sql + ) if ignore or replace: # Does view exist already? if name in self.view_names(): @@ -586,136 +1743,297 @@ class Database: self.execute(create_sql) return self - def m2m_table_candidates(self, table, other_table): - "Returns potential m2m tables for arguments, based on FKs" + def m2m_table_candidates(self, table: str, other_table: str) -> List[str]: + """ + Given two table names returns the name of tables that could define a + many-to-many relationship between those two tables, based on having + foreign keys to both of the provided tables. + + :param table: Table name + :param other_table: Other table name + """ candidates = [] tables = {table, other_table} - for table in self.tables: + for table_obj in self.tables: # Does it have foreign keys to both table and other_table? - has_fks_to = {fk.other_table for fk in table.foreign_keys} + has_fks_to = {fk.other_table for fk in table_obj.foreign_keys} if has_fks_to.issuperset(tables): - candidates.append(table.name) + candidates.append(table_obj.name) return candidates - def add_foreign_keys(self, foreign_keys): - # foreign_keys is a list of explicit 4-tuples - assert all( - len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys - ), "foreign_keys must be a list of 4-tuples, (table, column, other_table, other_column)" + def add_foreign_keys( + self, foreign_keys: Iterable[Union[ForeignKey, ForeignKeyTuple]] + ) -> None: + """ + See :ref:`python_api_add_foreign_keys`. - foreign_keys_to_create = [] + :param foreign_keys: A list of ``(table, column, other_table, other_column)`` + tuples - for compound foreign keys, ``column`` and ``other_column`` can + be tuples of column names + """ + # foreign_keys is a list of explicit 4-tuples + if not all( + isinstance(fk, ForeignKey) + or (isinstance(fk, (list, tuple)) and len(fk) == 4) + for fk in foreign_keys + ): + raise ValueError( + "foreign_keys must be a list of 4-tuples, " + "(table, column, other_table, other_column)" + ) + + foreign_keys_to_create: List[ForeignKey] = [] # Verify that all tables and columns exist - for table, column, other_table, other_column in foreign_keys: - if not self[table].exists(): + for fk in foreign_keys: + if isinstance(fk, ForeignKey): + fk_object = fk + else: + table, column_or_columns, other_table, other_column_or_columns = fk + # Compound foreign keys use tuples of columns + columns = ( + (column_or_columns,) + if isinstance(column_or_columns, str) + else tuple(column_or_columns) + ) + other_columns = ( + (other_column_or_columns,) + if isinstance(other_column_or_columns, str) + else tuple(other_column_or_columns) + ) + if len(columns) != len(other_columns): + raise ValueError( + "Compound foreign key must have the same number of " + "columns on both sides" + ) + if len(columns) == 1: + fk_object = ForeignKey( + table, columns[0], other_table, other_columns[0] + ) + else: + fk_object = ForeignKey( + table, + None, + other_table, + None, + columns=columns, + other_columns=other_columns, + is_compound=True, + ) + table = fk_object.table + other_table = fk_object.other_table + if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) - if column not in self[table].columns_dict: - raise AlterError("No such column: {} in {}".format(column, table)) + table_obj = self.table(table) + fk_object = self._resolve_foreign_key_casing( + fk_object, table_obj.columns_dict + ) + columns = fk_object.columns + other_columns = fk_object.other_columns + for column in columns: + if column not in table_obj.columns_dict: + raise AlterError("No such column: {} in {}".format(column, table)) if not self[other_table].exists(): raise AlterError("No such other_table: {}".format(other_table)) - if ( - other_column != "rowid" - and other_column not in self[other_table].columns_dict + for other_column in other_columns: + if ( + other_column != "rowid" + and other_column not in self[other_table].columns_dict + ): + raise AlterError( + "No such other_column: {} in {}".format( + other_column, other_table + ) + ) + # Silently skip foreign keys that exist already - but only if + # they match exactly, including ON DELETE/ON UPDATE actions + columns_folded = tuple(fold_identifier_case(c) for c in columns) + other_columns_folded = tuple(fold_identifier_case(c) for c in other_columns) + existing = [ + fk + for fk in table_obj.foreign_keys + if tuple(fold_identifier_case(c) for c in fk.columns) == columns_folded + and fold_identifier_case(fk.other_table) + == fold_identifier_case(other_table) + and tuple(fold_identifier_case(c) for c in fk.other_columns) + == other_columns_folded + ] + if not existing: + foreign_keys_to_create.append(fk_object) + elif any( + fk.on_delete != fk_object.on_delete + or fk.on_update != fk_object.on_update + for fk in existing ): raise AlterError( - "No such other_column: {} in {}".format(other_column, other_table) - ) - # We will silently skip foreign keys that exist already - if not any( - fk - for fk in self[table].foreign_keys - if fk.column == column - and fk.other_table == other_table - and fk.other_column == other_column - ): - foreign_keys_to_create.append( - (table, column, other_table, other_column) + "Foreign key already exists for {} => {}.{} but with " + "different ON DELETE/ON UPDATE actions - use " + "table.transform() to change them".format( + ", ".join(columns), other_table, ", ".join(other_columns) + ) ) - # Construct SQL for use with "UPDATE sqlite_master SET sql = ? WHERE name = ?" - table_sql = {} - for table, column, other_table, other_column in foreign_keys_to_create: - old_sql = table_sql.get(table, self[table].schema) - extra_sql = ",\n FOREIGN KEY([{column}]) REFERENCES [{other_table}]([{other_column}])\n".format( - column=column, other_table=other_table, other_column=other_column - ) - # Stick that bit in at the very end just before the closing ')' - last_paren = old_sql.rindex(")") - new_sql = old_sql[:last_paren].strip() + extra_sql + old_sql[last_paren:] - table_sql[table] = new_sql + # Group them by table + by_table: Dict[str, List[ForeignKey]] = {} + for fk_object in foreign_keys_to_create: + by_table.setdefault(fk_object.table, []).append(fk_object) - # And execute it all within a single transaction - with self.conn: - cursor = self.conn.cursor() - schema_version = cursor.execute("PRAGMA schema_version").fetchone()[0] - cursor.execute("PRAGMA writable_schema = 1") - for table_name, new_sql in table_sql.items(): - cursor.execute( - "UPDATE sqlite_master SET sql = ? WHERE name = ?", - (new_sql, table_name), - ) - cursor.execute("PRAGMA schema_version = %d" % (schema_version + 1)) - cursor.execute("PRAGMA writable_schema = 0") - # Have to VACUUM outside the transaction to ensure .foreign_keys property - # can see the newly created foreign key. - self.vacuum() + for table, fks in by_table.items(): + self.table(table).transform(add_foreign_keys=fks) - def index_foreign_keys(self): + if not self.conn.in_transaction: + self.vacuum() + + def index_foreign_keys(self) -> None: + "Create indexes for every foreign key column on every table in the database." for table_name in self.table_names(): - table = self[table_name] - existing_indexes = { - i.columns[0] for i in table.indexes if len(i.columns) == 1 - } + table = self.table(table_name) + existing_indexes = {tuple(i.columns) for i in table.indexes} for fk in table.foreign_keys: - if fk.column not in existing_indexes: - table.create_index([fk.column]) + # A compound foreign key gets a single composite index + if fk.columns not in existing_indexes: + table.create_index(fk.columns, find_unique_name=True) + existing_indexes.add(fk.columns) - def vacuum(self): + def vacuum(self) -> None: + "Run a SQLite ``VACUUM`` against the database." self.execute("VACUUM;") + def analyze(self, name: Optional[str] = None) -> None: + """ + Run ``ANALYZE`` against the entire database or a named table or index. + + :param name: Run ``ANALYZE`` against this specific named table or index + """ + sql = "ANALYZE" + if name is not None: + sql += " {}".format(quote_identifier(name)) + self.execute(sql) + + def iterdump(self) -> Generator[str, None, None]: + "A sequence of strings representing a SQL dump of the database" + if iterdump: + yield from iterdump(self.conn) + else: + try: + yield from self.conn.iterdump() + except AttributeError: + raise AttributeError( + "conn.iterdump() not found - try pip install sqlite-dump" + ) + + def init_spatialite(self, path: Optional[str] = None) -> bool: + """ + The ``init_spatialite`` method will load and initialize the SpatiaLite extension. + The ``path`` argument should be an absolute path to the compiled extension, which + can be found using ``find_spatialite``. + + Returns ``True`` if SpatiaLite was successfully initialized. + + .. code-block:: python + + from sqlite_utils.db import Database + from sqlite_utils.utils import find_spatialite + + db = Database("mydb.db") + db.init_spatialite(find_spatialite()) + + If you've installed SpatiaLite somewhere unexpected (for testing an alternate version, for example) + you can pass in an absolute path: + + .. code-block:: python + + from sqlite_utils.db import Database + from sqlite_utils.utils import find_spatialite + + db = Database("mydb.db") + db.init_spatialite("./local/mod_spatialite.dylib") + + :param path: Path to SpatiaLite module on disk + """ + if path is None: + path = find_spatialite() + if path is None: + raise OSError("Could not find SpatiaLite extension") + + self.conn.enable_load_extension(True) + self.conn.load_extension(path) + # Initialize SpatiaLite if not yet initialized + if "spatial_ref_sys" in self.table_names(): + return False + cursor = self.execute("select InitSpatialMetadata(1)") + result = cursor.fetchone() + return result and bool(result[0]) + class Queryable: - def exists(self): + db: "Database" + name: str + + def exists(self) -> bool: + "Does this table or view exist yet?" return False - def __init__(self, db, name): + def __init__(self, db: "Database", name: str) -> None: self.db = db self.name = name def count_where( self, - where=None, - where_args=None, - ): - sql = "select count(*) from [{}]".format(self.name) + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + ) -> int: + """ + Executes ``SELECT count(*) FROM table WHERE ...`` and returns a count. + + :param where: SQL where fragment to use, for example ``id > ?`` + :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` + parameters, or a dictionary for ``id > :id`` + """ + sql = "select count(*) from {}".format(quote_identifier(self.name)) if where is not None: sql += " where " + where return self.db.execute(sql, where_args or []).fetchone()[0] - def execute_count(self): + def execute_count(self) -> int: # Backwards compatibility, see https://github.com/simonw/sqlite-utils/issues/305#issuecomment-890713185 return self.count_where() @property - def count(self): + def count(self) -> int: + "A count of the rows in this table or view." return self.count_where() @property - def rows(self): + def rows(self) -> Generator[Dict[str, Any], None, None]: + "Iterate over every dictionaries for each row in this table or view." return self.rows_where() def rows_where( self, - where=None, - where_args=None, - order_by=None, - select="*", - limit=None, - offset=None, - ): + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + order_by: Optional[str] = None, + select: str = "*", + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Generator[Dict[str, Any], None, None]: + """ + Iterate over every row in this table or view that matches the specified where clause. + + Returns each row as a dictionary. See :ref:`python_api_rows` for more details. + + :param where: SQL where fragment to use, for example ``id > ?`` + :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` + parameters, or a dictionary for ``id > :id`` + :param order_by: Column or fragment of SQL to order by + :param select: Comma-separated list of columns to select - defaults to ``*`` + :param limit: Integer number of rows to limit to + :param offset: Integer for SQL offset + """ if not self.exists(): - return [] - sql = "select {} from [{}]".format(select, self.name) + return + sql = "select {} from {}".format(select, quote_identifier(self.name)) if where is not None: sql += " where " + where if order_by is not None: @@ -725,25 +2043,45 @@ class Queryable: if offset is not None: sql += " offset {}".format(offset) cursor = self.db.execute(sql, where_args or []) - columns = [c[0] for c in cursor.description] + columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: yield dict(zip(columns, row)) def pks_and_rows_where( self, - where=None, - where_args=None, - order_by=None, - limit=None, - offset=None, - ): - "Like .rows_where() but returns (pk, row) pairs - pk can be a single value or tuple" - column_names = [column.name for column in self.columns] - pks = [column.name for column in self.columns if column.is_pk] + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + order_by: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Generator[Tuple[Any, Dict[str, Any]], None, None]: + """ + Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple. + + :param where: SQL where fragment to use, for example ``id > ?`` + :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` + parameters, or a dictionary for ``id > :id`` + :param order_by: Column or fragment of SQL to order by + :param select: Comma-separated list of columns to select - defaults to ``*`` + :param limit: Integer number of rows to limit to + :param offset: Integer for SQL offset + """ + # This method is defined on Queryable so it serves views too, which + # have no pks property - sort pk columns into declaration order here + pk_columns = sorted( + (column for column in self.columns if column.is_pk), + key=lambda column: column.is_pk, + ) + pks = [column.name for column in pk_columns] + select_parts = [quote_identifier(column.name) for column in self.columns] if not pks: - column_names.insert(0, "rowid") + # rowid is left unquoted: it is not a real column, and SQLite + # turns a double-quoted identifier that does not resolve into a + # string literal - on a view that would silently select the + # string 'rowid' instead of raising an error + select_parts.insert(0, "rowid") pks = ["rowid"] - select = ",".join("[{}]".format(column_name) for column_name in column_names) + select = ",".join(select_parts) for row in self.rows_where( select=select, where=where, @@ -758,45 +2096,79 @@ class Queryable: yield row_pk, row @property - def columns(self): + def columns(self) -> List["Column"]: + "List of :ref:`Columns ` representing the columns in this table or view." if not self.exists(): return [] - rows = self.db.execute("PRAGMA table_info([{}])".format(self.name)).fetchall() + rows = self.db.execute( + "PRAGMA table_info({})".format(quote_identifier(self.name)) + ).fetchall() return [Column(*row) for row in rows] @property - def columns_dict(self): - "Returns {column: python-type} dictionary" + def columns_dict(self) -> Dict[str, Any]: + "``{column_name: python-type}`` dictionary representing columns in this table or view." return {column.name: column_affinity(column.type) for column in self.columns} @property - def schema(self): + def schema(self) -> str: + "SQL schema for this table or view." return self.db.execute( "select sql from sqlite_master where name = ?", (self.name,) ).fetchone()[0] class Table(Queryable): - last_rowid = None - last_pk = None + """ + Tables should usually be initialized using the ``db.table(table_name)`` or + ``db[table_name]`` methods. + + The following optional parameters can be passed to ``db.table(table_name, ...)``: + + :param db: Provided by ``db.table(table_name)`` + :param name: Provided by ``db.table(table_name)`` + :param pk: Name of the primary key column, or tuple of columns + :param foreign_keys: List of foreign key definitions + :param column_order: List of column names in the order they should be in the table + :param not_null: List of columns that cannot be null + :param defaults: Dictionary of column names and default values + :param batch_size: Integer number of rows to insert at a time + :param hash_id: Name of a column to create and use as a primary key, where the + value of that primary key is derived from a hash of the row values + :param hash_id_columns: List of columns to use for the hash_id + :param alter: If True, automatically alter the table if it doesn't match the schema + :param ignore: If True, ignore rows that already exist when inserting + :param replace: If True, replace rows that already exist when inserting + :param extracts: Dictionary or list of column names to extract into a separate table on inserts + :param conversions: Dictionary of column names and conversion functions + :param columns: Dictionary of column names to column types + :param strict: If True, apply STRICT mode to table + """ + + #: The ``rowid`` of the last inserted, updated or selected row. + last_rowid: Optional[int] = None + #: The primary key of the last inserted, updated or selected row. + last_pk: Optional[Any] = None def __init__( self, - db, - name, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - batch_size=100, - hash_id=None, - alter=False, - ignore=False, - replace=False, - extracts=None, - conversions=None, - columns=None, + db: Database, + name: str, + pk: Optional[Any] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + batch_size: int = 100, + hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, + alter: bool = False, + ignore: bool = False, + replace: bool = False, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, + conversions: Optional[dict] = None, + columns: Optional[Dict[str, Any]] = None, + strict: bool = False, ): super().__init__(db, name) self._defaults = dict( @@ -807,45 +2179,69 @@ class Table(Queryable): defaults=defaults, batch_size=batch_size, hash_id=hash_id, + hash_id_columns=hash_id_columns, alter=alter, ignore=ignore, replace=replace, extracts=extracts, conversions=conversions or {}, columns=columns, + strict=strict, ) - def __repr__(self): + def __repr__(self) -> str: return "
".format( self.name, - " (does not exist yet)" - if not self.exists() - else " ({})".format(", ".join(c.name for c in self.columns)), + ( + " (does not exist yet)" + if not self.exists() + else " ({})".format(", ".join(c.name for c in self.columns)) + ), ) @property - def count(self): + def count(self) -> int: + "Count of the rows in this table - optionally from the table count cache, if configured." if self.db.use_counts_table: counts = self.db.cached_counts([self.name]) if counts: return next(iter(counts.values())) return self.count_where() - def exists(self): + def exists(self) -> bool: return self.name in self.db.table_names() @property - def pks(self): - names = [column.name for column in self.columns if column.is_pk] + def pks(self) -> List[str]: + """ + Primary key columns for this table, in PRIMARY KEY declaration order - + ``PRAGMA table_info`` sets ``is_pk`` to the 1-based position of each + column within the primary key, which can differ from the order of the + columns in the table. SQLite uses the declaration order to resolve + implicit foreign key references, so this order matters. + """ + pk_columns = sorted( + (column for column in self.columns if column.is_pk), + key=lambda column: column.is_pk, + ) + names = [column.name for column in pk_columns] if not names: names = ["rowid"] return names @property - def use_rowid(self): + def use_rowid(self) -> bool: + "Does this table use ``rowid`` for its primary key (no other primary keys are specified)?" return not any(column for column in self.columns if column.is_pk) - def get(self, pk_values): + def get(self, pk_values: Union[list, tuple, str, int]) -> dict: + """ + Return row (as dictionary) for the specified primary key. + + Raises ``sqlite_utils.db.NotFoundError`` if a matching row cannot be found. + + :param pk_values: A single value, or a tuple of values for tables that have a compound primary key + """ if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] pks = self.pks @@ -857,7 +2253,7 @@ class Table(Queryable): ) ) - wheres = ["[{}] = ?".format(pk_name) for pk_name in pks] + wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in pks] rows = self.rows_where(" and ".join(wheres), pk_values) try: row = list(rows)[0] @@ -867,33 +2263,64 @@ class Table(Queryable): raise NotFoundError @property - def foreign_keys(self): - fks = [] + def foreign_keys(self) -> List["ForeignKey"]: + """ + List of foreign keys defined on this table. + + Compound (multi-column) foreign keys are returned as a single + ``ForeignKey`` with ``is_compound=True`` and populated + ``columns``/``other_columns`` lists. + """ + # PRAGMA foreign_key_list returns one row per column, grouped by "id" + # with "seq" giving the column order within a compound foreign key. + by_id: Dict[int, list] = {} for row in self.db.execute( - "PRAGMA foreign_key_list([{}])".format(self.name) + "PRAGMA foreign_key_list({})".format(quote_identifier(self.name)) ).fetchall(): if row is not None: id, seq, table_name, from_, to_, on_update, on_delete, match = row - fks.append( - ForeignKey( - table=self.name, - column=from_, - other_table=table_name, - other_column=to_, - ) + by_id.setdefault(id, []).append( + (seq, table_name, from_, to_, on_update, on_delete) ) + fks = [] + for id in sorted(by_id): + rows = sorted(by_id[id]) # order columns by seq + other_table = rows[0][1] + columns = tuple(row[2] for row in rows) + other_columns = tuple(row[3] for row in rows) + if all(c is None for c in other_columns): + # "REFERENCES other_table" with no columns - the pragma + # returns None, meaning the other table's primary key + other_table_pks = tuple(self.db.table(other_table).pks) + if len(other_table_pks) == len(columns): + other_columns = other_table_pks + is_compound = len(rows) > 1 + fks.append( + ForeignKey( + table=self.name, + column=None if is_compound else columns[0], + other_table=other_table, + other_column=None if is_compound else other_columns[0], + columns=columns, + other_columns=other_columns, + is_compound=is_compound, + on_update=rows[0][4], + on_delete=rows[0][5], + ) + ) return fks @property - def virtual_table_using(self): - "Returns type of virtual table or None if this is not a virtual table" + def virtual_table_using(self) -> Optional[str]: + "Type of virtual table, or ``None`` if this is not a virtual table." match = _virtual_table_using_re.match(self.schema) if match is None: return None return match.groupdict()["using"].upper() @property - def indexes(self): + def indexes(self) -> List[Index]: + "List of indexes defined on this table." sql = 'PRAGMA index_list("{}")'.format(self.name) indexes = [] for row in self.db.execute_returning_dicts(sql): @@ -916,7 +2343,8 @@ class Table(Queryable): return indexes @property - def xindexes(self): + def xindexes(self) -> List[XIndex]: + "List of indexes defined on this table using the more detailed ``XIndex`` format." sql = 'PRAGMA index_list("{}")'.format(self.name) indexes = [] for row in self.db.execute_returning_dicts(sql): @@ -934,7 +2362,8 @@ class Table(Queryable): return indexes @property - def triggers(self): + def triggers(self) -> List[Trigger]: + "List of triggers defined on this table." return [ Trigger(*r) for r in self.db.execute( @@ -945,49 +2374,179 @@ class Table(Queryable): ] @property - def triggers_dict(self): - "Returns {trigger_name: sql} dictionary" + def triggers_dict(self) -> Dict[str, str]: + "``{trigger_name: sql}`` dictionary of triggers defined on this table." return {trigger.name: trigger.sql for trigger in self.triggers} + @property + def default_values(self) -> Dict[str, Any]: + "``{column_name: default_value}`` dictionary of default values for columns in this table." + return { + column.name: _decode_default_value(column.default_value) + for column in self.columns + if column.default_value is not None + } + + @property + def strict(self) -> bool: + "Is this a STRICT table?" + table_suffix = self.schema.split(")")[-1].strip().upper() + table_options = [bit.strip() for bit in table_suffix.split(",")] + return "STRICT" in table_options + def create( self, - columns, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - hash_id=None, - extracts=None, - ): + columns: Dict[str, Any], + pk: Optional[Any] = DEFAULT, + foreign_keys: Union[Optional[ForeignKeysType], Default] = DEFAULT, + column_order: Union[Optional[List[str]], Default] = DEFAULT, + not_null: Union[Optional[Iterable[str]], Default] = DEFAULT, + defaults: Union[Optional[Dict[str, Any]], Default] = DEFAULT, + hash_id: Union[Optional[str], Default] = DEFAULT, + hash_id_columns: Union[Optional[Iterable[str]], Default] = DEFAULT, + extracts: Union[Optional[Union[Dict[str, str], List[str]]], Default] = DEFAULT, + if_not_exists: bool = False, + replace: bool = False, + ignore: bool = False, + transform: bool = False, + strict: Union[bool, Default] = DEFAULT, + ) -> "Table": + """ + Create a table with the specified columns. + + See :ref:`python_api_explicit_create` for full details. + + :param columns: Dictionary mapping column names to their types, for example ``{"name": str, "age": int}`` + :param pk: String name of column to use as a primary key, or a tuple of strings for a compound primary key covering multiple columns + :param foreign_keys: List of foreign key definitions for this table + :param column_order: List specifying which columns should come first + :param not_null: List of columns that should be created as ``NOT NULL`` + :param defaults: Dictionary specifying default values for columns + :param hash_id: Name of column to be used as a primary key containing a hash of the other columns + :param hash_id_columns: List of columns to be used when calculating the hash ID for a row + :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts` + :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS`` + :param replace: Drop and replace table if it already exists + :param ignore: Silently do nothing if table already exists + :param transform: If table already exists transform it to fit the specified schema + :param strict: Apply STRICT mode to table + """ + # Resolve defaults from _defaults (issue #655) + pk = self.value_or_default("pk", pk) + foreign_keys = self.value_or_default("foreign_keys", foreign_keys) + column_order = self.value_or_default("column_order", column_order) + not_null = self.value_or_default("not_null", not_null) + defaults = self.value_or_default("defaults", defaults) + hash_id = self.value_or_default("hash_id", hash_id) + hash_id_columns = self.value_or_default("hash_id_columns", hash_id_columns) + extracts = self.value_or_default("extracts", extracts) + strict = self.value_or_default("strict", strict) + + # Store configuration in _defaults for subsequent operations (issue #655) + # Don't store pk if hash_id is set, since pk is derived from hash_id in that case + if pk is not None and hash_id is None: + self._defaults["pk"] = pk + if foreign_keys is not None: + self._defaults["foreign_keys"] = foreign_keys + if column_order is not None: + self._defaults["column_order"] = column_order + if not_null is not None: + self._defaults["not_null"] = not_null + if defaults is not None: + self._defaults["defaults"] = defaults + if hash_id is not None: + self._defaults["hash_id"] = hash_id + if hash_id_columns is not None: + self._defaults["hash_id_columns"] = hash_id_columns + if extracts is not None: + self._defaults["extracts"] = extracts + if strict: + self._defaults["strict"] = strict + columns = {name: value for (name, value) in columns.items()} - with self.db.conn: + with self.db.atomic(): self.db.create_table( self.name, columns, pk=pk, - foreign_keys=foreign_keys, - column_order=column_order, - not_null=not_null, - defaults=defaults, - hash_id=hash_id, - extracts=extracts, + foreign_keys=foreign_keys, # type: ignore[arg-type] + column_order=column_order, # type: ignore[arg-type] + not_null=not_null, # type: ignore[arg-type] + defaults=defaults, # type: ignore[arg-type] + hash_id=hash_id, # type: ignore[arg-type] + hash_id_columns=hash_id_columns, # type: ignore[arg-type] + extracts=extracts, # type: ignore[arg-type] + if_not_exists=if_not_exists, + replace=replace, + ignore=ignore, + transform=transform, + strict=strict, # type: ignore[arg-type] ) return self + def duplicate(self, new_name: str) -> "Table": + """ + Create a duplicate of this table, copying across the schema and all row data. + + :param new_name: Name of the new table + """ + if not self.exists(): + raise NoTable(f"Table {self.name} does not exist") + with self.db.atomic(): + sql = "CREATE TABLE {} AS SELECT * FROM {};".format( + quote_identifier(new_name), + quote_identifier(self.name), + ) + self.db.execute(sql) + return self.db.table(new_name) + def transform( self, *, - types=None, - rename=None, - drop=None, - pk=DEFAULT, - not_null=None, - defaults=None, - drop_foreign_keys=None, - column_order=None, - ): - assert self.exists(), "Cannot transform a table that doesn't exist yet" + types: Optional[dict] = None, + rename: Optional[dict] = None, + drop: Optional[Iterable] = None, + pk: Optional[Any] = DEFAULT, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + drop_foreign_keys: Optional[Iterable[str]] = None, + add_foreign_keys: Optional[ForeignKeysType] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + keep_table: Optional[str] = None, + strict: Optional[bool] = None, + ) -> "Table": + """ + Apply an advanced alter table, including operations that are not supported by + ``ALTER TABLE`` in SQLite itself. + + See :ref:`python_api_transform` for full details. + + Raises :py:class:`sqlite_utils.db.TransactionError` if called while a + transaction is open with ``PRAGMA foreign_keys`` enabled and the table + is referenced by foreign keys with destructive ``ON DELETE`` actions - + see :ref:`python_api_transform_foreign_keys_transactions`. + + :param types: Columns that should have their type changed, for example ``{"weight": float}`` + :param rename: Columns to rename, for example ``{"headline": "title"}`` + :param drop: Columns to drop + :param pk: New primary key for the table + :param not_null: Columns to set as ``NOT NULL`` + :param defaults: Default values for columns + :param drop_foreign_keys: Foreign key constraints to remove - a column name + drops any foreign key that column participates in, a tuple of column names + drops the compound foreign key with exactly those columns + :param add_foreign_keys: List of foreign keys to add to the table + :param foreign_keys: List of foreign keys to set for the table, replacing any existing foreign keys + :param column_order: List of strings specifying a full or partial column order + to use when creating the table + :param keep_table: If specified, the existing table will be renamed to this and will not be + dropped + :param strict: Set to ``True`` to make the table strict or ``False`` to make it + non-strict. Defaults to ``None``, which preserves the existing strict mode. + """ + if not self.exists(): + raise ValueError("Cannot transform a table that doesn't exist yet") sqls = self.transform_sql( types=types, rename=rename, @@ -996,41 +2555,228 @@ class Table(Queryable): not_null=not_null, defaults=defaults, drop_foreign_keys=drop_foreign_keys, + add_foreign_keys=add_foreign_keys, + foreign_keys=foreign_keys, column_order=column_order, + keep_table=keep_table, + strict=strict, ) - pragma_foreign_keys_was_on = self.db.execute("PRAGMA foreign_keys").fetchone()[ - 0 - ] + pragma_foreign_keys_was_on = bool( + self.db.execute("PRAGMA foreign_keys").fetchone()[0] + ) + already_in_transaction = self.db.conn.in_transaction + should_disable_foreign_keys = ( + pragma_foreign_keys_was_on and not already_in_transaction + ) + should_defer_foreign_keys = ( + pragma_foreign_keys_was_on and already_in_transaction + ) + if should_defer_foreign_keys: + # PRAGMA foreign_keys is a no-op inside a transaction, and + # defer_foreign_keys only defers violation checks, not ON DELETE + # actions - so dropping the old table would still fire destructive + # actions on any tables that reference it. Refuse rather than + # silently modify or delete those rows. + destructive_fks = [ + (table.name, fk) + for table in self.db.tables + for fk in table.foreign_keys + if fk.other_table == self.name + and fk.on_delete in ("CASCADE", "SET NULL", "SET DEFAULT") + ] + if destructive_fks: + raise TransactionError( + "Cannot transform table {table} while a transaction is open: " + "PRAGMA foreign_keys cannot be changed inside a transaction, " + "and the table is referenced by foreign keys with ON DELETE " + "actions that would fire when the old table is dropped: " + "{fks}. Call transform() outside of the transaction, or " + 'execute "PRAGMA foreign_keys = off" before opening it.'.format( + table=self.name, + fks=", ".join( + "{}.{} (ON DELETE {})".format( + table_name, ", ".join(fk.columns), fk.on_delete + ) + for table_name, fk in destructive_fks + ), + ) + ) + defer_foreign_keys_was_on = False try: - if pragma_foreign_keys_was_on: + if should_disable_foreign_keys: self.db.execute("PRAGMA foreign_keys=0;") - with self.db.conn: + elif should_defer_foreign_keys: + defer_foreign_keys_was_on = bool( + self.db.execute("PRAGMA defer_foreign_keys").fetchone()[0] + ) + if not defer_foreign_keys_was_on: + self.db.execute("PRAGMA defer_foreign_keys=ON;") + with self.db.atomic(): for sql in sqls: self.db.execute(sql) # Run the foreign_key_check before we commit if pragma_foreign_keys_was_on: - self.db.execute("PRAGMA foreign_key_check;") + foreign_key_violations = self.db.execute( + "PRAGMA foreign_key_check;" + ).fetchall() + if foreign_key_violations: + raise sqlite3.IntegrityError("FOREIGN KEY constraint failed") finally: - if pragma_foreign_keys_was_on: + if should_defer_foreign_keys and not defer_foreign_keys_was_on: + self.db.execute("PRAGMA defer_foreign_keys=OFF;") + if should_disable_foreign_keys: self.db.execute("PRAGMA foreign_keys=1;") + if strict is not None: + self._defaults["strict"] = strict return self def transform_sql( self, *, - types=None, - rename=None, - drop=None, - pk=DEFAULT, - not_null=None, - defaults=None, - drop_foreign_keys=None, - column_order=None, - tmp_suffix=None, - ): + types: Optional[dict] = None, + rename: Optional[dict] = None, + drop: Optional[Iterable] = None, + pk: Optional[Any] = DEFAULT, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + drop_foreign_keys: Optional[Iterable] = None, + add_foreign_keys: Optional[ForeignKeysType] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + tmp_suffix: Optional[str] = None, + keep_table: Optional[str] = None, + strict: Optional[bool] = None, + ) -> List[str]: + """ + Return a list of SQL statements that should be executed in order to apply this transformation. + + :param types: Columns that should have their type changed, for example ``{"weight": float}`` + :param rename: Columns to rename, for example ``{"headline": "title"}`` + :param drop: Columns to drop + :param pk: New primary key for the table + :param not_null: Columns to set as ``NOT NULL`` + :param defaults: Default values for columns + :param drop_foreign_keys: Foreign key constraints to remove - a column name + drops any foreign key that column participates in, a tuple of column names + drops the compound foreign key with exactly those columns + :param add_foreign_keys: List of foreign keys to add to the table + :param foreign_keys: List of foreign keys to set for the table, replacing any existing foreign keys + :param column_order: List of strings specifying a full or partial column order + to use when creating the table + :param tmp_suffix: Suffix to use for the temporary table name + :param keep_table: If specified, the existing table will be renamed to this and will not be + dropped + :param strict: Set to ``True`` to make the table strict or ``False`` to make it + non-strict. Defaults to ``None``, which preserves the existing strict mode. + """ + if strict is True and not self.db.supports_strict: + raise TransformError("SQLite does not support STRICT tables") types = types or {} rename = rename or {} drop = drop or set() + + # Resolve column references against the existing schema, matching + # case-insensitively the way SQLite does + existing_columns = self.columns_dict + types = {resolve_casing(c, existing_columns): t for c, t in types.items()} + rename = {resolve_casing(c, existing_columns): v for c, v in rename.items()} + drop = {resolve_casing(c, existing_columns) for c in drop} + if pk is not DEFAULT and pk is not None: + if isinstance(pk, str): + pk = resolve_casing(pk, existing_columns) + else: + pk = [resolve_casing(p, existing_columns) for p in pk] + if isinstance(not_null, dict): + not_null = { + resolve_casing(c, existing_columns): v + for c, v in cast(Dict[str, Any], not_null).items() + } + elif isinstance(not_null, set): + not_null = {resolve_casing(c, existing_columns) for c in not_null} + if defaults is not None: + defaults = { + resolve_casing(c, existing_columns): v for c, v in defaults.items() + } + if column_order is not None: + column_order = [resolve_casing(c, existing_columns) for c in column_order] + + create_table_foreign_keys: List[ForeignKeyIndicator] = [] + + if foreign_keys is not None: + if add_foreign_keys is not None: + raise ValueError( + "Cannot specify both foreign_keys and add_foreign_keys" + ) + if drop_foreign_keys is not None: + raise ValueError( + "Cannot specify both foreign_keys and drop_foreign_keys" + ) + create_table_foreign_keys.extend(foreign_keys) + else: + # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys + # The casing of columns in a foreign key definition can differ + # from the casing of the columns themselves, so these comparisons + # are all case-folded + dropped_columns_folded = {fold_identifier_case(c) for c in drop} + renamed_columns_folded = { + fold_identifier_case(k): v for k, v in rename.items() + } + + def fk_should_be_dropped(fk: ForeignKey) -> bool: + fk_columns_folded = tuple(fold_identifier_case(c) for c in fk.columns) + if drop_foreign_keys is not None: + for spec in drop_foreign_keys: + if isinstance(spec, str): + # A column name matches any foreign key it participates in + if fold_identifier_case(spec) in fk_columns_folded: + return True + elif ( + tuple(fold_identifier_case(s) for s in spec) + == fk_columns_folded + ): + # A tuple/list must match a compound key's columns exactly + return True + # Dropping any of a foreign key's columns drops the whole key + return any( + column in dropped_columns_folded for column in fk_columns_folded + ) + + def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: + columns = tuple( + renamed_columns_folded.get(fold_identifier_case(column)) or column + for column in fk.columns + ) + if fk.is_compound: + return ForeignKey( + self.name, + None, + fk.other_table, + None, + columns=columns, + other_columns=fk.other_columns, + is_compound=True, + on_delete=fk.on_delete, + on_update=fk.on_update, + ) + return ForeignKey( + self.name, + columns[0], + fk.other_table, + fk.other_columns[0], + on_delete=fk.on_delete, + on_update=fk.on_update, + ) + + create_table_foreign_keys = [] + # Copy over old foreign keys, unless we are dropping them + for fk in self.foreign_keys: + if not fk_should_be_dropped(fk): + create_table_foreign_keys.append(fk_with_renamed_columns(fk)) + # Add new foreign keys + if add_foreign_keys is not None: + for fk in self.db.resolve_foreign_keys(self.name, add_foreign_keys): + create_table_foreign_keys.append(fk_with_renamed_columns(fk)) + new_table_name = "{}_new_{}".format( self.name, tmp_suffix or os.urandom(6).hex() ) @@ -1046,10 +2792,10 @@ class Table(Queryable): new_column_pairs.append((new_name, type_)) copy_from_to[name] = new_name - sqls = [] if pk is DEFAULT: pks_renamed = tuple( - rename.get(p.name) or p.name for p in self.columns if p.is_pk + rename.get(pk_name) or pk_name + for pk_name in (self.pks if not self.use_rowid else []) ) if len(pks_renamed) == 1: pk = pks_renamed[0] @@ -1074,10 +2820,14 @@ class Table(Queryable): create_table_not_null.add(key) elif isinstance(not_null, set): create_table_not_null.update((rename.get(k) or k) for k in not_null) - elif not_null is None: + elif not not_null: pass else: - assert False, "not_null must be a dict or a set or None" + raise ValueError( + "not_null must be a dict or a set or None, it was {}".format( + repr(not_null) + ) + ) # defaults= create_table_defaults = { (rename.get(c.name) or c.name): c.default_value @@ -1089,17 +2839,10 @@ class Table(Queryable): {rename.get(c) or c: v for c, v in defaults.items()} ) - # foreign_keys - create_table_foreign_keys = [] - for table, column, other_table, other_column in self.foreign_keys: - if (drop_foreign_keys is None) or (column not in drop_foreign_keys): - create_table_foreign_keys.append( - (rename.get(column) or column, other_table, other_column) - ) - if column_order is not None: column_order = [rename.get(col) or col for col in column_order] + sqls = [] sqls.append( self.db.create_table_sql( new_table_name, @@ -1109,6 +2852,7 @@ class Table(Queryable): defaults=create_table_defaults, foreign_keys=create_table_foreign_keys, column_order=column_order, + strict=self.strict if strict is None else strict, ).strip() ) @@ -1118,114 +2862,217 @@ class Table(Queryable): for from_, to_ in copy_from_to.items(): old_cols.append(from_) new_cols.append(to_) - copy_sql = "INSERT INTO [{new_table}] ({new_cols})\n SELECT {old_cols} FROM [{old_table}];".format( - new_table=new_table_name, - old_table=self.name, - old_cols=", ".join("[{}]".format(col) for col in old_cols), - new_cols=", ".join("[{}]".format(col) for col in new_cols), + # Ensure rowid is copied too + if "rowid" not in new_cols: + new_cols.insert(0, "rowid") + old_cols.insert(0, "rowid") + copy_sql = "INSERT INTO {} ({new_cols})\n SELECT {old_cols} FROM {};".format( + quote_identifier(new_table_name), + quote_identifier(self.name), + old_cols=", ".join(quote_identifier(col) for col in old_cols), + new_cols=", ".join(quote_identifier(col) for col in new_cols), ) sqls.append(copy_sql) - # Drop the old table - sqls.append("DROP TABLE [{}];".format(self.name)) + # Drop (or keep) the old table + if keep_table: + sqls.append( + "ALTER TABLE {} RENAME TO {};".format( + quote_identifier(self.name), quote_identifier(keep_table) + ) + ) + else: + sqls.append("DROP TABLE {};".format(quote_identifier(self.name))) # Rename the new one sqls.append( - "ALTER TABLE [{}] RENAME TO [{}];".format(new_table_name, self.name) + "ALTER TABLE {} RENAME TO {};".format( + quote_identifier(new_table_name), quote_identifier(self.name) + ) ) + # Re-add existing indexes + for index in self.indexes: + if index.origin != "pk": + index_sql = self.db.execute( + """SELECT sql FROM sqlite_master WHERE type = 'index' AND name = :index_name;""", + {"index_name": index.name}, + ).fetchall()[0][0] + if index_sql is None: + raise TransformError( + f"Index '{index.name}' on table '{self.name}' does not have a " + "CREATE INDEX statement. You must manually drop this index prior to running this " + "transformation and manually recreate the new index after running this transformation." + ) + if keep_table: + sqls.append(f"DROP INDEX IF EXISTS {quote_identifier(index.name)};") + for col in index.columns: + if col in rename.keys() or col in drop: + raise TransformError( + f"Index '{index.name}' column '{col}' is not in updated table '{self.name}'. " + f"You must manually drop this index prior to running this transformation " + f"and manually recreate the new index after running this transformation. " + f"The original index sql statement is: `{index_sql}`. No changes have been applied to this table." + ) + sqls.append(index_sql) return sqls - def extract(self, columns, table=None, fk_column=None, rename=None): + def extract( + self, + columns: Union[str, Iterable[str]], + table: Optional[str] = None, + fk_column: Optional[str] = None, + rename: Optional[Dict[str, str]] = None, + ) -> "Table": + """ + Extract specified columns into a separate table. + + See :ref:`python_api_extract` for details. + + :param columns: Single column or list of columns that should be extracted + :param table: Name of table in which the new records should be created + :param fk_column: Name of the foreign key column to populate in the original table + :param rename: Dictionary of columns that should be renamed when populating the new table + """ rename = rename or {} if isinstance(columns, str): columns = [columns] + columns = [resolve_casing(c, self.columns_dict) for c in columns] + rename = {resolve_casing(k, self.columns_dict): v for k, v in rename.items()} if not set(columns).issubset(self.columns_dict.keys()): raise InvalidColumns( "Invalid columns {} for table with columns {}".format( columns, list(self.columns_dict.keys()) ) ) - table = table or "_".join(columns) - lookup_table = self.db[table] - fk_column = fk_column or "{}_id".format(table) - magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) + with self.db.atomic(): + table = table or "_".join(columns) + lookup_table = self.db.table(table) + fk_column = fk_column or "{}_id".format(table) + magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) - # Populate the lookup table with all of the extracted unique values - lookup_columns_definition = { - (rename.get(col) or col): typ - for col, typ in self.columns_dict.items() - if col in columns - } - if lookup_table.exists(): - if not set(lookup_columns_definition.items()).issubset( - lookup_table.columns_dict.items() - ): - raise InvalidColumns( - "Lookup table {} already exists but does not have columns {}".format( - table, lookup_columns_definition + # Populate the lookup table with all of the extracted unique values + lookup_columns_definition = { + (rename.get(col) or col): typ + for col, typ in self.columns_dict.items() + if col in columns + } + if lookup_table.exists(): + if not set(lookup_columns_definition.items()).issubset( + lookup_table.columns_dict.items() + ): + raise InvalidColumns( + "Lookup table {} already exists but does not have columns {}".format( + table, lookup_columns_definition + ) ) - ) - else: - lookup_table.create( - { - **{ - "id": int, - }, - **lookup_columns_definition, - }, - pk="id", - ) - lookup_columns = [(rename.get(col) or col) for col in columns] - lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) - self.db.execute( - "INSERT OR IGNORE INTO [{lookup_table}] ({lookup_columns}) SELECT DISTINCT {table_cols} FROM [{table}]".format( - lookup_table=table, - lookup_columns=", ".join("[{}]".format(c) for c in lookup_columns), - table_cols=", ".join("[{}]".format(c) for c in columns), - table=self.name, - ) - ) - - # Now add the new fk_column - self.add_column(magic_lookup_column, int) - - # And populate it - self.db.execute( - "UPDATE [{table}] SET [{magic_lookup_column}] = (SELECT id FROM [{lookup_table}] WHERE {where})".format( - table=self.name, - magic_lookup_column=magic_lookup_column, - lookup_table=table, - where=" AND ".join( - "[{table}].[{column}] = [{lookup_table}].[{lookup_column}]".format( - table=self.name, - lookup_table=table, - column=column, - lookup_column=rename.get(column) or column, - ) - for column in columns - ), - ) - ) - # Figure out the right column order - column_order = [] - for c in self.columns: - if c.name in columns and magic_lookup_column not in column_order: - column_order.append(magic_lookup_column) - elif c.name == magic_lookup_column: - continue else: - column_order.append(c.name) + lookup_table.create( + { + **{ + "id": int, + }, + **lookup_columns_definition, + }, + pk="id", + ) + lookup_columns = [(rename.get(col) or col) for col in columns] + lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) + # Rows where every extracted column is null are left alone - they + # get a null foreign key and no lookup table record, see #186 + all_columns_are_null = " AND ".join( + "{} IS NULL".format(quote_identifier(c)) for c in columns + ) + # INSERT OR IGNORE dedupes against the unique index, but unique + # indexes treat NULLs as distinct - the NOT EXISTS guard uses IS + # comparison so NULL-containing rows match existing lookup rows + # instead of being inserted again + already_in_lookup = " AND ".join( + "{lookup}.{lookup_col} IS {source}.{source_col}".format( + lookup=quote_identifier(table), + lookup_col=quote_identifier(rename.get(column) or column), + source=quote_identifier(self.name), + source_col=quote_identifier(column), + ) + for column in columns + ) + self.db.execute( + "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {} " + "WHERE NOT ({all_null}) AND NOT EXISTS (SELECT 1 FROM {lookup} WHERE {already_in_lookup})".format( + quote_identifier(table), + quote_identifier(self.name), + lookup_columns=", ".join( + quote_identifier(c) for c in lookup_columns + ), + table_cols=", ".join(quote_identifier(c) for c in columns), + all_null=all_columns_are_null, + lookup=quote_identifier(table), + already_in_lookup=already_in_lookup, + ) + ) - # Drop the unnecessary columns and rename lookup column - self.transform( - drop=set(columns), - rename={magic_lookup_column: fk_column}, - column_order=column_order, - ) + # Now add the new fk_column + self.add_column(magic_lookup_column, int) - # And add the foreign key constraint - self.add_foreign_key(fk_column, table, "id") + # And populate it + self.db.execute( + "UPDATE {} SET {} = (SELECT id FROM {} WHERE {where}) WHERE NOT ({all_null})".format( + quote_identifier(self.name), + quote_identifier(magic_lookup_column), + quote_identifier(table), + where=" AND ".join( + "{}.{} IS {}.{}".format( + quote_identifier(self.name), + quote_identifier(column), + quote_identifier(table), + quote_identifier(rename.get(column) or column), + ) + for column in columns + ), + all_null=all_columns_are_null, + ) + ) + # Figure out the right column order + column_order = [] + for c in self.columns: + if c.name in columns and magic_lookup_column not in column_order: + column_order.append(magic_lookup_column) + elif c.name == magic_lookup_column: + continue + else: + column_order.append(c.name) + + # Drop the unnecessary columns and rename lookup column + self.transform( + drop=set(columns), + rename={magic_lookup_column: fk_column}, + column_order=column_order, + ) + + # And add the foreign key constraint + self.add_foreign_key(fk_column, table, "id") return self - def create_index(self, columns, index_name=None, unique=False, if_not_exists=False): + def create_index( + self, + columns: Iterable[Union[str, DescIndex]], + index_name: Optional[str] = None, + unique: bool = False, + if_not_exists: bool = False, + find_unique_name: bool = False, + analyze: bool = False, + ): + """ + Create an index on this table. + + :param columns: A single columns or list of columns to index. These can be strings or, + to create an index using the column in descending order, ``db.DescIndex(column_name)`` objects. + :param index_name: The name to use for the new index. Defaults to the column names joined on ``_``. + :param unique: Should the index be marked as unique, forcing unique values? + :param if_not_exists: Only create the index if one with that name does not already exist. + :param find_unique_name: If ``index_name`` is not provided and the automatically derived name + already exists, keep incrementing a suffix number to find an available name. + :param analyze: Run ``ANALYZE`` against this index after creating it. + + See :ref:`python_api_create_index`. + """ if index_name is None: index_name = "idx_{}_{}".format( self.name.replace(" ", "_"), "_".join(columns) @@ -1233,32 +3080,85 @@ class Table(Queryable): columns_sql = [] for column in columns: if isinstance(column, DescIndex): - fmt = "[{}] desc" + columns_sql.append("{} desc".format(quote_identifier(column))) else: - fmt = "[{}]" - columns_sql.append(fmt.format(column)) - sql = ( - textwrap.dedent( - """ - CREATE {unique}INDEX {if_not_exists}[{index_name}] - ON [{table_name}] ({columns}); + columns_sql.append(quote_identifier(column)) + + suffix = None + created_index_name = None + while True: + created_index_name = ( + "{}_{}".format(index_name, suffix) if suffix else index_name + ) + sql = ( + textwrap.dedent(""" + CREATE {unique}INDEX {if_not_exists}{index_name} + ON {table_name} ({columns}); + """) + .strip() + .format( + index_name=quote_identifier(created_index_name), + table_name=quote_identifier(self.name), + columns=", ".join(columns_sql), + unique="UNIQUE " if unique else "", + if_not_exists="IF NOT EXISTS " if if_not_exists else "", + ) + ) + try: + self.db.execute(sql) + break + except OperationalError as e: + # find_unique_name=True - try again if 'index ... already exists' + arg = e.args[0] + if ( + find_unique_name + and arg.startswith("index ") + and arg.endswith(" already exists") + ): + if suffix is None: + suffix = 2 + else: + suffix += 1 + continue + else: + raise e + if analyze: + self.db.analyze(created_index_name) + return self + + def drop_index(self, index_name: str, ignore: bool = False): """ + Drop an index on this table. + + :param index_name: Name of the index to drop + :param ignore: Set to ``True`` to ignore the error if the index does not exist + """ + if index_name not in {index.name for index in self.indexes}: + if ignore: + return self + raise OperationalError( + "No index named {} on table {}".format(index_name, self.name) ) - .strip() - .format( - index_name=index_name, - table_name=self.name, - columns=", ".join(columns_sql), - unique="UNIQUE " if unique else "", - if_not_exists="IF NOT EXISTS " if if_not_exists else "", - ) - ) - self.db.execute(sql) + self.db.execute("DROP INDEX {}".format(quote_identifier(index_name))) return self def add_column( - self, col_name, col_type=None, fk=None, fk_col=None, not_null_default=None + self, + col_name: str, + col_type: Optional[Any] = None, + fk: Optional[str] = None, + fk_col: Optional[str] = None, + not_null_default: Optional[Any] = None, ): + """ + Add a column to this table. See :ref:`python_api_add_column`. + + :param col_name: Name of the new column + :param col_type: Column type - a Python type such as ``str`` or a SQLite type string such as ``"BLOB"`` + :param fk: Name of a table that this column should be a foreign key reference to + :param fk_col: Column in the foreign key table that this should reference + :param not_null_default: Set this column to ``not null`` and give it this default value + """ fk_col_type = None if fk is not None: # fk must be a valid table @@ -1266,11 +3166,15 @@ class Table(Queryable): raise AlterError("table '{}' does not exist".format(fk)) # if fk_col specified, must be a valid column if fk_col is not None: + fk_col = resolve_casing(fk_col, self.db[fk].columns_dict) if fk_col not in self.db[fk].columns_dict: raise AlterError("table '{}' has no column {}".format(fk, fk_col)) else: # automatically set fk_col to first primary_key of fk table - pks = [c for c in self.db[fk].columns if c.is_pk] + pks = sorted( + (c for c in self.db[fk].columns if c.is_pk), + key=lambda c: c.is_pk, + ) if pks: fk_col = pks[0].name fk_col_type = pks[0].type @@ -1281,10 +3185,12 @@ class Table(Queryable): col_type = str not_null_sql = None if not_null_default is not None: - not_null_sql = "NOT NULL DEFAULT {}".format(self.db.quote(not_null_default)) - sql = "ALTER TABLE [{table}] ADD COLUMN [{col_name}] {col_type}{not_null_default};".format( - table=self.name, - col_name=col_name, + not_null_sql = "NOT NULL DEFAULT {}".format( + self.db.quote_default_value(not_null_default) + ) + sql = "ALTER TABLE {} ADD COLUMN {} {col_type}{not_null_default};".format( + quote_identifier(self.name), + quote_identifier(col_name), col_type=fk_col_type or COLUMN_TYPE_MAPPING[col_type], not_null_default=(" " + not_null_sql) if not_null_sql else "", ) @@ -1293,14 +3199,30 @@ class Table(Queryable): self.add_foreign_key(col_name, fk, fk_col) return self - def drop(self, ignore=False): + def drop(self, ignore: bool = False) -> None: + """ + Drop this table. + + :param ignore: Set to ``True`` to ignore the error if the table does not exist + """ try: - self.db.execute("DROP TABLE [{}]".format(self.name)) + self.db.execute("DROP TABLE {}".format(quote_identifier(self.name))) except sqlite3.OperationalError: if not ignore: raise - def guess_foreign_table(self, column): + def guess_foreign_table(self, column: str) -> str: + """ + For a given column, suggest another table that might be referenced by this + column should it be used as a foreign key. + + For example, a column called ``tag_id`` or ``tag`` or ``tags`` might suggest + a ``tag`` table, if one exists. + + If no candidates can be found, raises a ``NoObviousTable`` exception. + + :param column: Name of column + """ column = column.lower() possibilities = [column] if column.endswith("_id"): @@ -1321,7 +3243,7 @@ class Table(Queryable): ) ) - def guess_foreign_column(self, other_table): + def guess_foreign_column(self, other_table: str) -> str: pks = [c for c in self.db[other_table].columns if c.is_pk] if len(pks) != 1: raise BadPrimaryKey( @@ -1331,89 +3253,165 @@ class Table(Queryable): return pks[0].name def add_foreign_key( - self, column, other_table=None, other_column=None, ignore=False + self, + column: ForeignKeyColumns, + other_table: Optional[str] = None, + other_column: Optional[ForeignKeyColumns] = None, + ignore: bool = False, + on_delete: str = "NO ACTION", + on_update: str = "NO ACTION", ): - # Ensure column exists - if column not in self.columns_dict: - raise AlterError("No such column: {}".format(column)) + """ + Alter the schema to mark the specified column as a foreign key to another table. + + :param column: The column to mark as a foreign key - use a tuple of columns + for a compound foreign key. + :param other_table: The table it refers to - if omitted, will be guessed based on the column name. + :param other_column: The column on the other table it - if omitted, will be guessed. + Use a tuple of columns for a compound foreign key. + :param ignore: Set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. + :param on_delete: ``ON DELETE`` action for the foreign key, e.g. ``"CASCADE"`` + or ``"SET NULL"``. + :param on_update: ``ON UPDATE`` action for the foreign key. + """ + columns = (column,) if isinstance(column, str) else tuple(column) + columns = tuple(resolve_casing(c, self.columns_dict) for c in columns) + # Ensure columns exist + for col in columns: + if col not in self.columns_dict: + raise AlterError("No such column: {}".format(col)) # If other_table is not specified, attempt to guess it from the column if other_table is None: - other_table = self.guess_foreign_table(column) + if len(columns) > 1: + raise ValueError( + "other_table must be specified for a compound foreign key" + ) + other_table = self.guess_foreign_table(columns[0]) # If other_column is not specified, detect the primary key on other_table if other_column is None: - other_column = self.guess_foreign_column(other_table) + if len(columns) > 1: + other_columns = tuple(self.db.table(other_table).pks) + else: + other_columns = (self.guess_foreign_column(other_table),) + elif isinstance(other_column, str): + other_columns = (other_column,) + else: + other_columns = tuple(other_column) + other_columns = tuple( + resolve_casing(c, self.db[other_table].columns_dict) for c in other_columns + ) + if len(columns) != len(other_columns): + raise ValueError( + "Compound foreign key must have the same number of columns " + "on both sides" + ) - # Soundness check that the other column exists - if ( - not [c for c in self.db[other_table].columns if c.name == other_column] - and other_column != "rowid" - ): - raise AlterError("No such column: {}.{}".format(other_table, other_column)) + # Soundness check that the other columns exist + for other_col in other_columns: + if ( + not [c for c in self.db[other_table].columns if c.name == other_col] + and other_col != "rowid" + ): + raise AlterError("No such column: {}.{}".format(other_table, other_col)) # Check we do not already have an existing foreign key if any( fk for fk in self.foreign_keys - if fk.column == column - and fk.other_table == other_table - and fk.other_column == other_column + if tuple(fold_identifier_case(c) for c in fk.columns) + == tuple(fold_identifier_case(c) for c in columns) + and fold_identifier_case(fk.other_table) + == fold_identifier_case(other_table) + and tuple(fold_identifier_case(c) for c in fk.other_columns) + == tuple(fold_identifier_case(c) for c in other_columns) ): if ignore: return self else: raise AlterError( "Foreign key already exists for {} => {}.{}".format( - column, other_table, other_column + ", ".join(columns), other_table, ", ".join(other_columns) ) ) - self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) + if len(columns) == 1: + fk_object = ForeignKey( + self.name, + columns[0], + other_table, + other_columns[0], + on_delete=on_delete, + on_update=on_update, + ) + else: + fk_object = ForeignKey( + self.name, + None, + other_table, + None, + columns=columns, + other_columns=other_columns, + is_compound=True, + on_delete=on_delete, + on_update=on_update, + ) + self.db.add_foreign_keys([fk_object]) return self - def enable_counts(self): + def enable_counts(self) -> None: + """ + Set up triggers to update a cache of the count of rows in this table. + + See :ref:`python_api_cached_table_counts` for details. + """ sql = ( - textwrap.dedent( - """ + textwrap.dedent(""" {create_counts_table} - CREATE TRIGGER IF NOT EXISTS [{table}{counts_table}_insert] AFTER INSERT ON [{table}] + CREATE TRIGGER IF NOT EXISTS {trigger_insert} AFTER INSERT ON {table} BEGIN - INSERT OR REPLACE INTO [{counts_table}] + INSERT OR REPLACE INTO {counts_table} VALUES ( {table_quoted}, COALESCE( - (SELECT count FROM [{counts_table}] WHERE [table] = {table_quoted}), + (SELECT count FROM {counts_table} WHERE "table" = {table_quoted}), 0 ) + 1 ); END; - CREATE TRIGGER IF NOT EXISTS [{table}{counts_table}_delete] AFTER DELETE ON [{table}] + CREATE TRIGGER IF NOT EXISTS {trigger_delete} AFTER DELETE ON {table} BEGIN - INSERT OR REPLACE INTO [{counts_table}] + INSERT OR REPLACE INTO {counts_table} VALUES ( {table_quoted}, COALESCE( - (SELECT count FROM [{counts_table}] WHERE [table] = {table_quoted}), + (SELECT count FROM {counts_table} WHERE "table" = {table_quoted}), 0 ) - 1 ); END; - INSERT OR REPLACE INTO _counts VALUES ({table_quoted}, (select count(*) from [{table}])); - """ - ) + INSERT OR REPLACE INTO _counts VALUES ({table_quoted}, (select count(*) from {table})); + """) .strip() .format( create_counts_table=_COUNTS_TABLE_CREATE_SQL.format( self.db._counts_table_name ), - counts_table=self.db._counts_table_name, - table=self.name, + counts_table=quote_identifier(self.db._counts_table_name), + table=quote_identifier(self.name), table_quoted=self.db.quote(self.name), + trigger_insert=quote_identifier( + self.name + self.db._counts_table_name + "_insert" + ), + trigger_delete=quote_identifier( + self.name + self.db._counts_table_name + "_delete" + ), ) ) - with self.db.conn: - self.db.conn.executescript(sql) + with self.db.atomic(): + self.db._executescript(sql) self.db.use_counts_table = True @property - def has_counts_triggers(self): + def has_counts_triggers(self) -> bool: + "Does this table have triggers setup to update cached counts?" trigger_names = { "{table}{counts_table}_{suffix}".format( counts_table=self.db._counts_table_name, table=self.name, suffix=suffix @@ -1424,26 +3422,35 @@ class Table(Queryable): def enable_fts( self, - columns, - fts_version="FTS5", - create_triggers=False, - tokenize=None, - replace=False, + columns: Iterable[str], + fts_version: str = "FTS5", + create_triggers: bool = False, + tokenize: Optional[str] = None, + replace: bool = False, ): - "Enables FTS on the specified columns." - create_fts_sql = ( - textwrap.dedent( - """ - CREATE VIRTUAL TABLE [{table}_fts] USING {fts_version} ( - {columns},{tokenize} - content=[{table}] - ) """ + Enable SQLite full-text search against the specified columns. + + See :ref:`python_api_fts` for more details. + + :param columns: List of column names to include in the search index. + :param fts_version: FTS version to use - defaults to ``FTS5`` but you may want ``FTS4`` for older SQLite versions. + :param create_triggers: Should triggers be created to keep the search index up-to-date? Defaults to ``False``. + :param tokenize: Custom SQLite tokenizer to use, for example ``"porter"`` to enable Porter stemming. + :param replace: Should any existing FTS index for this table be replaced by the new one? + """ + create_fts_sql = ( + textwrap.dedent(""" + CREATE VIRTUAL TABLE {table_fts} USING {fts_version} ( + {columns},{tokenize} + content={table} ) + """) .strip() .format( - table=self.name, - columns=", ".join("[{}]".format(c) for c in columns), + table=quote_identifier(self.name), + table_fts=quote_identifier(self.name + "_fts"), + columns=", ".join(quote_identifier(c) for c in columns), fts_version=fts_version, tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", ) @@ -1470,27 +3477,32 @@ class Table(Queryable): self.populate_fts(columns) if create_triggers: - old_cols = ", ".join("old.[{}]".format(c) for c in columns) - new_cols = ", ".join("new.[{}]".format(c) for c in columns) + old_cols = ", ".join("old.{}".format(quote_identifier(c)) for c in columns) + new_cols = ", ".join("new.{}".format(quote_identifier(c)) for c in columns) + columns_quoted = ", ".join(quote_identifier(c) for c in columns) + table = quote_identifier(self.name) + table_fts = quote_identifier(self.name + "_fts") triggers = ( - textwrap.dedent( - """ - CREATE TRIGGER [{table}_ai] AFTER INSERT ON [{table}] BEGIN - INSERT INTO [{table}_fts] (rowid, {columns}) VALUES (new.rowid, {new_cols}); + textwrap.dedent(""" + CREATE TRIGGER {table_ai} AFTER INSERT ON {table} BEGIN + INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; - CREATE TRIGGER [{table}_ad] AFTER DELETE ON [{table}] BEGIN - INSERT INTO [{table}_fts] ([{table}_fts], rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); + CREATE TRIGGER {table_ad} AFTER DELETE ON {table} BEGIN + INSERT INTO {table_fts} ({table_fts}, rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); END; - CREATE TRIGGER [{table}_au] AFTER UPDATE ON [{table}] BEGIN - INSERT INTO [{table}_fts] ([{table}_fts], rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); - INSERT INTO [{table}_fts] (rowid, {columns}) VALUES (new.rowid, {new_cols}); + CREATE TRIGGER {table_au} AFTER UPDATE ON {table} BEGIN + INSERT INTO {table_fts} ({table_fts}, rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); + INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; - """ - ) + """) .strip() .format( - table=self.name, - columns=", ".join("[{}]".format(c) for c in columns), + table=table, + table_fts=table_fts, + table_ai=quote_identifier(self.name + "_ai"), + table_ad=quote_identifier(self.name + "_ad"), + table_au=quote_identifier(self.name + "_au"), + columns=columns_quoted, old_cols=old_cols, new_cols=new_cols, ) @@ -1498,191 +3510,295 @@ class Table(Queryable): self.db.executescript(triggers) return self - def populate_fts(self, columns): - sql = ( - textwrap.dedent( - """ - INSERT INTO [{table}_fts] (rowid, {columns}) - SELECT rowid, {columns} FROM [{table}]; + def populate_fts(self, columns: Iterable[str]) -> "Table": """ - ) + Update the associated SQLite full-text search index with the latest data from the + table for the specified columns. + + :param columns: Columns to populate the data for + """ + columns_quoted = ", ".join(quote_identifier(c) for c in columns) + sql = ( + textwrap.dedent(""" + INSERT INTO {table_fts} (rowid, {columns}) + SELECT rowid, {columns} FROM {table}; + """) .strip() .format( - table=self.name, columns=", ".join("[{}]".format(c) for c in columns) + table=quote_identifier(self.name), + table_fts=quote_identifier(self.name + "_fts"), + columns=columns_quoted, ) ) self.db.executescript(sql) return self - def disable_fts(self): + def disable_fts(self) -> "Table": + "Remove any full-text search index and related triggers configured for this table." fts_table = self.detect_fts() if fts_table: self.db[fts_table].drop() # Now delete the triggers that related to that table - sql = ( - textwrap.dedent( - """ + sql = textwrap.dedent(""" SELECT name FROM sqlite_master WHERE type = 'trigger' - AND sql LIKE '% INSERT INTO [{}]%' - """ - ) - .strip() - .format(fts_table) - ) + AND (sql LIKE '% INSERT INTO [{}]%' OR sql LIKE '% INSERT INTO "{}"%') + """).strip().format(fts_table, fts_table) trigger_names = [] for row in self.db.execute(sql).fetchall(): trigger_names.append(row[0]) - with self.db.conn: + with self.db.atomic(): for trigger_name in trigger_names: - self.db.execute("DROP TRIGGER IF EXISTS [{}]".format(trigger_name)) + self.db.execute( + "DROP TRIGGER IF EXISTS {}".format(quote_identifier(trigger_name)) + ) return self - def rebuild_fts(self): + def rebuild_fts(self) -> "Table": + "Run the ``rebuild`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is None: # Assume this is itself an FTS table fts_table = self.name - self.db.execute( - "INSERT INTO [{table}]([{table}]) VALUES('rebuild');".format( - table=fts_table + with self.db.atomic(): + self.db.execute( + "INSERT INTO {table}({table}) VALUES('rebuild');".format( + table=quote_identifier(fts_table) + ) ) - ) return self - def detect_fts(self): + def detect_fts(self) -> Optional[str]: "Detect if table has a corresponding FTS virtual table and return it" - sql = ( - textwrap.dedent( - """ + sql = textwrap.dedent(""" SELECT name FROM sqlite_master WHERE rootpage = 0 AND ( - sql LIKE '%VIRTUAL TABLE%USING FTS%content=%{table}%' + sql LIKE :like + OR sql LIKE :like2 OR ( - tbl_name = "{table}" + tbl_name = :table AND sql LIKE '%VIRTUAL TABLE%USING FTS%' ) ) - """ - ) - .strip() - .format(table=self.name) - ) - rows = self.db.execute(sql).fetchall() + """).strip() + args = { + "like": "%VIRTUAL TABLE%USING FTS%content=[{}]%".format(self.name), + "like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), + "table": self.name, + } + rows = self.db.execute(sql, args).fetchall() if len(rows) == 0: return None else: return rows[0][0] - def optimize(self): + def optimize(self) -> "Table": + "Run the ``optimize`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is not None: - self.db.execute( - """ - INSERT INTO [{table}] ([{table}]) VALUES ("optimize"); - """.strip().format( - table=fts_table - ) - ) + with self.db.atomic(): + self.db.execute(""" + INSERT INTO {table} ({table}) VALUES ("optimize"); + """.strip().format(table=quote_identifier(fts_table))) return self - def search_sql(self, columns=None, order_by=None, limit=None, offset=None): + def search_sql( + self, + columns: Optional[Iterable[str]] = None, + order_by: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + where: Optional[str] = None, + include_rank: bool = False, + ) -> str: + """ " + Return SQL string that can be used to execute searches against this table. + + :param columns: Columns to search against + :param order_by: Column or SQL expression to sort by + :param limit: SQL limit + :param offset: SQL offset + :param where: Extra SQL fragment for the WHERE clause + :param include_rank: Select the search rank column in the final query + """ # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" + original_quoted = quote_identifier(original) columns_sql = "*" - columns_with_prefix_sql = "[{}].*".format(original) + columns_with_prefix_sql = "{}.*".format(original_quoted) if columns: - columns_sql = ",\n ".join("[{}]".format(c) for c in columns) + columns_sql = ",\n ".join(quote_identifier(c) for c in columns) columns_with_prefix_sql = ",\n ".join( - "[{}].[{}]".format(original, c) for c in columns + "{}.{}".format(original_quoted, quote_identifier(c)) for c in columns ) fts_table = self.detect_fts() - assert fts_table, "Full-text search is not configured for table '{}'".format( - self.name - ) - virtual_table_using = self.db[fts_table].virtual_table_using - sql = textwrap.dedent( - """ + if not fts_table: + raise ValueError( + "Full-text search is not configured for table '{}'".format(self.name) + ) + fts_table_quoted = quote_identifier(fts_table) + virtual_table_using = self.db.table(fts_table).virtual_table_using + sql = textwrap.dedent(""" with {original} as ( select rowid, {columns} - from [{dbtable}] + from {dbtable}{where_clause} ) select {columns_with_prefix} from - [{original}] - join [{fts_table}] on [{original}].rowid = [{fts_table}].rowid + {original} + join {fts_table} on {original}.rowid = {fts_table}.rowid where - [{fts_table}] match :query + {fts_table} match :query order by {order_by} {limit_offset} - """ - ).strip() + """).strip() if virtual_table_using == "FTS5": - rank_implementation = "[{}].rank".format(fts_table) + rank_implementation = "{}.rank".format(fts_table_quoted) else: self.db.register_fts4_bm25() - rank_implementation = "rank_bm25(matchinfo([{}], 'pcnalx'))".format( - fts_table + rank_implementation = "rank_bm25(matchinfo({}, 'pcnalx'))".format( + fts_table_quoted ) + if include_rank: + columns_with_prefix_sql += ",\n " + rank_implementation + " rank" limit_offset = "" if limit is not None: limit_offset += " limit {}".format(limit) if offset is not None: limit_offset += " offset {}".format(offset) return sql.format( - dbtable=self.name, - original=original, + dbtable=quote_identifier(self.name), + where_clause="\n where {}".format(where) if where else "", + original=original_quoted, columns=columns_sql, columns_with_prefix=columns_with_prefix_sql, - fts_table=fts_table, + fts_table=fts_table_quoted, order_by=order_by or rank_implementation, limit_offset=limit_offset.strip(), ).strip() - def search(self, q, order_by=None, columns=None, limit=None, offset=None): + def search( + self, + q: str, + order_by: Optional[str] = None, + columns: Optional[Iterable[str]] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + where: Optional[str] = None, + where_args: Optional[Union[Iterable, dict]] = None, + include_rank: bool = False, + quote: bool = False, + ) -> Generator[dict, None, None]: + """ + Execute a search against this table using SQLite full-text search, returning a sequence of + dictionaries for each row. + + :param q: Terms to search for + :param order_by: Defaults to order by rank, or specify a column here. + :param columns: List of columns to return, defaults to all columns. + :param limit: Optional integer limit for returned rows. + :param offset: Optional integer SQL offset. + :param where: Extra SQL fragment for the WHERE clause + :param where_args: Arguments to use for :param placeholders in the extra WHERE clause + :param include_rank: Select the search rank column in the final query + :param quote: Apply quoting to disable any special characters in the search query + + See :ref:`python_api_fts_search`. + """ + args = {"query": self.db.quote_fts(q) if quote else q} + if where_args and "query" in where_args: + raise ValueError( + "'query' is a reserved key and cannot be passed to where_args for .search()" + ) + if where_args: + args.update(where_args) + cursor = self.db.execute( self.search_sql( order_by=order_by, columns=columns, limit=limit, offset=offset, + where=where, + include_rank=include_rank, ), - {"query": q}, + args, ) - columns = [c[0] for c in cursor.description] + columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: yield dict(zip(columns, row)) - def value_or_default(self, key, value): + def value_or_default(self, key: str, value: Any) -> Any: return self._defaults[key] if value is DEFAULT else value - def delete(self, pk_values): + def delete(self, pk_values: Union[list, tuple, str, int, float]) -> "Table": + """ + Delete row matching the specified primary key. + + :param pk_values: A single value, or a tuple of values for tables that have a compound primary key + """ if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] self.get(pk_values) - wheres = ["[{}] = ?".format(pk_name) for pk_name in self.pks] - sql = "delete from [{table}] where {wheres}".format( - table=self.name, wheres=" and ".join(wheres) + wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in self.pks] + sql = "delete from {} where {wheres}".format( + quote_identifier(self.name), wheres=" and ".join(wheres) ) - with self.db.conn: + with self.db.atomic(): self.db.execute(sql, pk_values) return self - def delete_where(self, where=None, where_args=None): + def delete_where( + self, + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + analyze: bool = False, + ) -> "Table": + """ + Delete rows matching the specified where clause, or delete all rows in the table. + + See :ref:`python_api_delete_where`. + + :param where: SQL where fragment to use, for example ``id > ?`` + :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` + parameters, or a dictionary for ``id > :id`` + :param analyze: Set to ``True`` to run ``ANALYZE`` after the rows have been deleted. + """ if not self.exists(): - return [] - sql = "delete from [{}]".format(self.name) + return self + sql = "delete from {}".format(quote_identifier(self.name)) if where is not None: sql += " where " + where - self.db.execute(sql, where_args or []) + with self.db.atomic(): + self.db.execute(sql, where_args or []) + if analyze: + self.analyze() return self - def update(self, pk_values, updates=None, alter=False, conversions=None): + def update( + self, + pk_values: Union[list, tuple, str, int, float], + updates: Optional[dict] = None, + alter: bool = False, + conversions: Optional[dict] = None, + ) -> "Table": + """ + Execute a SQL ``UPDATE`` against the specified row. + + See :ref:`python_api_update`. + + :param pk_values: The primary key of an individual record - can be a tuple if the + table has a compound primary key. + :param updates: A dictionary mapping columns to their updated values. + :param alter: Set to ``True`` to add any missing columns. + :param conversions: Optional dictionary of SQL functions to apply during the update, for example + ``{"mycolumn": "upper(?)"}``. + """ updates = updates or {} conversions = conversions or {} if not isinstance(pk_values, (list, tuple)): @@ -1695,16 +3811,19 @@ class Table(Queryable): sets = [] wheres = [] pks = self.pks - validate_column_names(updates.keys()) for key, value in updates.items(): - sets.append("[{}] = {}".format(key, conversions.get(key, "?"))) + sets.append( + "{} = {}".format(quote_identifier(key), conversions.get(key, "?")) + ) args.append(jsonify_if_needed(value)) - wheres = ["[{}] = ?".format(pk_name) for pk_name in pks] + wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in pks] args.extend(pk_values) - sql = "update [{table}] set {sets} where {wheres}".format( - table=self.name, sets=", ".join(sets), wheres=" and ".join(wheres) + sql = "update {} set {sets} where {wheres}".format( + quote_identifier(self.name), + sets=", ".join(sets), + wheres=" and ".join(wheres), ) - with self.db.conn: + with self.db.atomic(): try: rowcount = self.db.execute(sql, args).rowcount except OperationalError as e: @@ -1722,18 +3841,37 @@ class Table(Queryable): def convert( self, - columns, - fn, - output=None, - output_type=None, - drop=False, - multi=False, - where=None, - where_args=None, - show_progress=False, - ): + columns: Union[str, List[str]], + fn: Callable, + output: Optional[str] = None, + output_type: Optional[Any] = None, + drop: bool = False, + multi: bool = False, + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + show_progress: bool = False, + ) -> "Table": + """ + Apply conversion function ``fn`` to every value in the specified columns. + + :param columns: A single column or list of string column names to convert. + :param fn: A callable that takes a single argument, ``value``, and returns it converted. + :param output: Optional string column name to write the results to (defaults to the input column). + :param output_type: If the output column needs to be created, this is the type that will be used + for the new column. + :param drop: Should the original column be dropped once the conversion is complete? + :param multi: If ``True`` the return value of ``fn(value)`` will be expected to be a + dictionary, and new columns will be created for each key of that dictionary. + :param where: SQL fragment to use as a ``WHERE`` clause to limit the rows to which the conversion + is applied, for example ``age > ?`` or ``age > :age``. + :param where_args: List of arguments (if using ``?``) or a dictionary (if using ``:age``). + :param show_progress: Should a progress bar be displayed? + + See :ref:`python_api_convert`. + """ if isinstance(columns, str): columns = [columns] + columns = [resolve_casing(c, self.columns_dict) for c in columns] if multi: return self._convert_multi( @@ -1746,7 +3884,9 @@ class Table(Queryable): ) if output is not None: - assert len(columns) == 1, "output= can only be used with a single column" + if len(columns) != 1: + raise ValueError("output= can only be used with a single column") + output = resolve_casing(output, self.columns_dict) if output not in self.columns_dict: self.add_column(output, output_type or "text") @@ -1755,24 +3895,27 @@ class Table(Queryable): def convert_value(v): bar.update(1) - if not v: - return v - return fn(v) + return jsonify_if_needed(fn(v)) - self.db.register_function(convert_value) - sql = "update [{table}] set {sets}{where};".format( - table=self.name, + fn_name = getattr(fn, "__name__", "fn") + if fn_name == "": + fn_name = f"lambda_{abs(hash(fn))}" + self.db.register_function(convert_value, name=fn_name) + sql = "update {} set {sets}{where};".format( + quote_identifier(self.name), sets=", ".join( [ - "[{output_column}] = convert_value([{column}])".format( - output_column=output or column, column=column + "{} = {}({})".format( + quote_identifier(output or column), + fn_name, + quote_identifier(column), ) for column in columns ] ), where=" where {}".format(where) if where is not None else "", ) - with self.db.conn: + with self.db.atomic(): self.db.execute(sql, where_args or []) if drop: self.transform(drop=columns) @@ -1783,17 +3926,15 @@ class Table(Queryable): ): # First we execute the function pk_to_values = {} - new_column_types = {} - pks = [column.name for column in self.columns if column.is_pk] - if not pks: - pks = ["rowid"] + new_column_types: Dict[str, Set[type]] = {} + pks = self.pks with progressbar( length=self.count, silent=not show_progress, label="1: Evaluating" ) as bar: for row in self.rows_where( select=", ".join( - "[{}]".format(column_name) for column_name in (pks + [column]) + quote_identifier(column_name) for column_name in (pks + [column]) ), where=where, where_args=where_args, @@ -1820,7 +3961,7 @@ class Table(Queryable): with progressbar( length=self.count, silent=not show_progress, label="2: Updating" ) as bar: - with self.db.conn: + with self.db.atomic(): for pk, updates in pk_to_values.items(): self.update(pk, updates) bar.update(1) @@ -1833,94 +3974,215 @@ class Table(Queryable): chunk, all_columns, hash_id, + hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, ignore, + list_mode=False, ): - # values is the list of insert data that is passed to the - # .execute() method - but some of them may be replaced by - # new primary keys if we are extracting any columns. - values = [] - extracts = resolve_extracts(extracts) - for record in chunk: - record_values = [] - for key in all_columns: - value = jsonify_if_needed( - record.get(key, None if key != hash_id else _hash(record)) + """ + Given a list ``chunk`` of records that should be written to *this* table, + return a list of ``(sql, parameters)`` 2-tuples which, when executed in + order, perform the desired INSERT / UPSERT / REPLACE operation. + """ + # Dict-mode insert({}) has no explicit columns; SQLite spells that as + # DEFAULT VALUES. List mode with no columns is a different input shape. + if not list_mode and not all_columns: + if upsert: + raise PrimaryKeyRequired( + "upsert() requires a value for the primary key - " + "an empty record cannot be upserted" ) - if key in extracts: - extract_table = extracts[key] - value = self.db[extract_table].lookup({"value": value}) - record_values.append(value) - values.append(record_values) - - queries_and_params = [] - if upsert: - if isinstance(pk, str): - pks = [pk] - else: - pks = pk - self.last_pk = None - for record_values in values: - # TODO: make more efficient: - record = dict(zip(all_columns, record_values)) - sql = "INSERT OR IGNORE INTO [{table}]({pks}) VALUES({pk_placeholders});".format( - table=self.name, - pks=", ".join(["[{}]".format(p) for p in pks]), - pk_placeholders=", ".join(["?" for p in pks]), - ) - queries_and_params.append((sql, [record[col] for col in pks])) - # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001; - set_cols = [col for col in all_columns if col not in pks] - if set_cols: - sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( - table=self.name, - pairs=", ".join( - "[{}] = {}".format(col, conversions.get(col, "?")) - for col in set_cols - ), - wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), - ) - queries_and_params.append( - ( - sql2, - [record[col] for col in set_cols] - + [record[pk] for pk in pks], - ) - ) - # We can populate .last_pk right here - if num_records_processed == 1: - self.last_pk = tuple(record[pk] for pk in pks) - if len(self.last_pk) == 1: - self.last_pk = self.last_pk[0] - - else: - or_what = "" + or_clause = "" if replace: - or_what = "OR REPLACE " + or_clause = " OR REPLACE" elif ignore: - or_what = "OR IGNORE " - sql = """ - INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; - """.strip().format( - or_what=or_what, - table=self.name, - columns=", ".join("[{}]".format(c) for c in all_columns), - rows=", ".join( - "({placeholders})".format( - placeholders=", ".join( - [conversions.get(col, "?") for col in all_columns] + or_clause = " OR IGNORE" + sql = ( + f"INSERT{or_clause} INTO {quote_identifier(self.name)} " + "DEFAULT VALUES" + ) + return [(sql, []) for _ in chunk] + + if hash_id_columns and hash_id is None: + hash_id = "id" + + extracts = resolve_extracts(extracts) + + # Build a row-list ready for executemany-style flattening + values = [] + + if list_mode: + # In list mode, records are already lists of values + num_columns = len(all_columns) + has_extracts = bool(extracts) + for record in chunk: + # Pad short records with None, truncate long ones + record_len = len(record) + if record_len < num_columns: + record_values = [jsonify_if_needed(v) for v in record] + [None] * ( + num_columns - record_len + ) + else: + record_values = [jsonify_if_needed(v) for v in record[:num_columns]] + # Only process extracts if there are any + if has_extracts: + for i, key in enumerate(all_columns): + if key in extracts and record_values[i] is not None: + record_values[i] = self.db.table(extracts[key]).lookup( + {"value": record_values[i]} + ) + values.append(record_values) + else: + # Dict mode: original logic + for record in chunk: + record_values = [] + for key in all_columns: + value = jsonify_if_needed( + record.get( + key, + ( + None + if key != hash_id + else hash_record(record, hash_id_columns) + ), ) ) - for record in chunk - ), - ) - flat_values = list(itertools.chain(*values)) - queries_and_params = [(sql, flat_values)] + if key in extracts and value is not None: + extract_table = extracts[key] + value = self.db.table(extract_table).lookup({"value": value}) + record_values.append(value) + values.append(record_values) + columns_sql = ", ".join(quote_identifier(c) for c in all_columns) + placeholder_expr = ", ".join(conversions.get(c, "?") for c in all_columns) + row_placeholders_sql = ", ".join(f"({placeholder_expr})" for _ in values) + flat_params = list(itertools.chain.from_iterable(values)) + + # replace=True mean INSERT OR REPLACE INTO + if replace: + sql = ( + f"INSERT OR REPLACE INTO {quote_identifier(self.name)} " + f"({columns_sql}) VALUES {row_placeholders_sql}" + ) + return [(sql, flat_params)] + + # If not an upsert it's an INSERT, maybe with OR IGNORE + if not upsert: + or_ignore = "" + if ignore: + or_ignore = " OR IGNORE" + sql = ( + f"INSERT{or_ignore} INTO {quote_identifier(self.name)} " + f"({columns_sql}) VALUES {row_placeholders_sql}" + ) + return [(sql, flat_params)] + + # Everything from here on is for upsert=True + pk_cols = [pk] if isinstance(pk, str) else list(pk) + # The records may use different casing for the pk columns than pk= + pk_cols = [resolve_casing(c, all_columns) for c in pk_cols] + # Every record must provide a value for every primary key column - a + # NULL primary key never matches ON CONFLICT, so the record would be + # inserted as a brand new row instead of upserted + missing_pk_cols = [c for c in pk_cols if c not in all_columns] + if missing_pk_cols: + raise PrimaryKeyRequired( + "upsert() requires a value for the primary key column{}: {}".format( + "s" if len(missing_pk_cols) > 1 else "", + ", ".join(missing_pk_cols), + ) + ) + pk_indexes = [all_columns.index(c) for c in pk_cols] + for record_values in values: + if any(record_values[i] is None for i in pk_indexes): + raise PrimaryKeyRequired( + "upsert() requires a value for the primary key column{}: {}".format( + "s" if len(pk_cols) > 1 else "", + ", ".join(pk_cols), + ) + ) + non_pk_cols = [c for c in all_columns if c not in pk_cols] + conflict_sql = ", ".join(quote_identifier(c) for c in pk_cols) + + if self.db.supports_on_conflict and not self.db.use_old_upsert: + if non_pk_cols: + # DO UPDATE + assignments = [] + for c in non_pk_cols: + c_quoted = quote_identifier(c) + if c in conversions: + assignments.append( + f"{c_quoted} = {conversions[c].replace('?', f'excluded.{c_quoted}')}" + ) + else: + assignments.append(f"{c_quoted} = excluded.{c_quoted}") + do_clause = "DO UPDATE SET " + ", ".join(assignments) + else: + # All columns are in the PK – nothing to update. + do_clause = "DO NOTHING" + + sql = ( + f"INSERT INTO {quote_identifier(self.name)} ({columns_sql}) " + f"VALUES {row_placeholders_sql} " + f"ON CONFLICT({conflict_sql}) {do_clause}" + ) + return [(sql, flat_params)] + + # At this point we need compatibility UPSERT for SQLite < 3.24.0 + # (INSERT OR IGNORE + second UPDATE stage) + queries_and_params = [] + pks = pk_cols + self.last_pk = None + for record_values in values: + record = dict(zip(all_columns, record_values)) + placeholders = list(pks) + # Need to populate not-null columns too, or INSERT OR IGNORE ignores + # them since it ignores the resulting integrity errors + if not_null: + placeholders.extend(not_null) + sql = ( + "INSERT OR IGNORE INTO {table}({cols}) VALUES({placeholders});".format( + table=quote_identifier(self.name), + cols=", ".join([quote_identifier(p) for p in placeholders]), + placeholders=", ".join(["?" for p in placeholders]), + ) + ) + queries_and_params.append( + (sql, [record[col] for col in pks] + ["" for _ in (not_null or [])]) + ) + # UPDATE "book" SET "name" = 'Programming' WHERE "id" = 1001; + set_cols = [col for col in all_columns if col not in pks] + if set_cols: + sql2 = "UPDATE {} SET {pairs} WHERE {wheres}".format( + quote_identifier(self.name), + pairs=", ".join( + "{} = {}".format( + quote_identifier(col), conversions.get(col, "?") + ) + for col in set_cols + ), + wheres=" AND ".join( + "{} = ?".format(quote_identifier(pk)) for pk in pks + ), + ) + queries_and_params.append( + ( + sql2, + [record[col] for col in set_cols] + [record[pk] for pk in pks], + ) + ) + # We can populate .last_pk right here + if num_records_processed == 1: + pk_values = tuple(record[pk] for pk in pks) + if len(pk_values) == 1: + self.last_pk = pk_values[0] + else: + self.last_pk = pk_values return queries_and_params def insert_chunk( @@ -1930,28 +4192,33 @@ class Table(Queryable): chunk, all_columns, hash_id, + hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, ignore, - ): + list_mode=False, + ) -> Optional[sqlite3.Cursor]: queries_and_params = self.build_insert_queries_and_params( extracts, chunk, all_columns, hash_id, + hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, ignore, + list_mode, ) - - with self.db.conn: - result = None + result = None + with self.db.atomic(): for query, params in queries_and_params: try: result = self.db.execute(query, params) @@ -1961,7 +4228,6 @@ class Table(Queryable): self.add_missing_columns(chunk) result = self.db.execute(query, params) elif e.args[0] == "too many SQL variables": - first_half = chunk[: len(chunk) // 2] second_half = chunk[len(chunk) // 2 :] @@ -1971,61 +4237,89 @@ class Table(Queryable): first_half, all_columns, hash_id, + hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, ignore, + list_mode, ) - self.insert_chunk( + result = self.insert_chunk( alter, extracts, second_half, all_columns, hash_id, + hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, ignore, + list_mode, ) else: raise - if num_records_processed == 1 and not upsert: - self.last_rowid = result.lastrowid - self.last_pk = self.last_rowid - # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened - if (hash_id or pk) and self.last_rowid: - row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] - if hash_id: - self.last_pk = row[hash_id] - elif isinstance(pk, str): - self.last_pk = row[pk] - else: - self.last_pk = tuple(row[p] for p in pk) - - return + return result def insert( self, - record, + record: Dict[str, Any], pk=DEFAULT, foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - hash_id=DEFAULT, - alter=DEFAULT, - ignore=DEFAULT, - replace=DEFAULT, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - ): + column_order: Optional[Union[List[str], Default]] = DEFAULT, + not_null: Optional[Union[Iterable[str], Default]] = DEFAULT, + defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + hash_id: Optional[Union[str, Default]] = DEFAULT, + hash_id_columns: Optional[Union[Iterable[str], Default]] = DEFAULT, + alter: Optional[Union[bool, Default]] = DEFAULT, + ignore: Optional[Union[bool, Default]] = DEFAULT, + replace: Optional[Union[bool, Default]] = DEFAULT, + extracts: Optional[Union[Dict[str, str], List[str], Default]] = DEFAULT, + conversions: Optional[Union[Dict[str, str], Default]] = DEFAULT, + columns: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + strict: Optional[Union[bool, Default]] = DEFAULT, + ) -> "Table": + """ + Insert a single record into the table. The table will be created with a schema that matches + the inserted record if it does not already exist, see :ref:`python_api_creating_tables`. + + - ``record`` - required: a dictionary representing the record to be inserted. + + The other parameters are optional, and mostly influence how the new table will be created if + that table does not exist yet. + + Each of them defaults to ``DEFAULT``, which indicates that the default setting for the current + ``Table`` object (specified in the table constructor) should be used. + + :param record: Dictionary record to be inserted + :param pk: If creating the table, which column should be the primary key. + :param foreign_keys: See :ref:`python_api_foreign_keys`. + :param column_order: List of strings specifying a full or partial column order + to use when creating the table. + :param not_null: Set of strings specifying columns that should be ``NOT NULL``. + :param defaults: Dictionary specifying default values for specific columns. + :param hash_id: Name of a column to create and use as a primary key, where the + value of that primary key will be derived as a SHA1 hash of the other column values + in the record. ``hash_id="id"`` is a common column name used for this. + :param alter: Boolean, should any missing columns be added automatically? + :param ignore: Boolean, if a record already exists with this primary key, ignore this insert. + :param replace: Boolean, if a record already exists with this primary key, replace it with this new record. + :param extracts: A list of columns to extract to other tables, or a dictionary that maps + ``{column_name: other_table_name}``. See :ref:`python_api_extracts`. + :param conversions: Dictionary specifying SQL conversion functions to be applied to the data while it + is being inserted, for example ``{"name": "upper(?)"}``. See :ref:`python_api_conversions`. + :param columns: Dictionary over-riding the detected types used for the columns, for example + ``{"age": int, "weight": float}``. + :param strict: Boolean, apply STRICT mode if creating the table. + """ return self.insert_all( [record], pk=pk, @@ -2034,17 +4328,22 @@ class Table(Queryable): not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, alter=alter, ignore=ignore, replace=replace, extracts=extracts, conversions=conversions, columns=columns, + strict=strict, ) def insert_all( self, - records, + records: Union[ + Iterable[Dict[str, Any]], + Iterable[Sequence[Any]], + ], pk=DEFAULT, foreign_keys=DEFAULT, column_order=DEFAULT, @@ -2052,6 +4351,7 @@ class Table(Queryable): defaults=DEFAULT, batch_size=DEFAULT, hash_id=DEFAULT, + hash_id_columns=DEFAULT, alter=DEFAULT, ignore=DEFAULT, replace=DEFAULT, @@ -2060,11 +4360,14 @@ class Table(Queryable): conversions=DEFAULT, columns=DEFAULT, upsert=False, - ): + analyze=False, + strict=DEFAULT, + ) -> "Table": """ - Like .insert() but takes a list of records and ensures that the table - that it creates (if table does not exist) has columns for ALL of that - data + Like ``.insert()`` but takes a list of records and ensures that the table + that it creates (if table does not exist) has columns for ALL of that data. + + Use ``analyze=True`` to run ``ANALYZE`` after the insert has completed. """ pk = self.value_or_default("pk", pk) foreign_keys = self.value_or_default("foreign_keys", foreign_keys) @@ -2073,49 +4376,146 @@ class Table(Queryable): defaults = self.value_or_default("defaults", defaults) batch_size = self.value_or_default("batch_size", batch_size) hash_id = self.value_or_default("hash_id", hash_id) + hash_id_columns = self.value_or_default("hash_id_columns", hash_id_columns) alter = self.value_or_default("alter", alter) ignore = self.value_or_default("ignore", ignore) replace = self.value_or_default("replace", replace) extracts = self.value_or_default("extracts", extracts) - conversions = self.value_or_default("conversions", conversions) + conversions = self.value_or_default("conversions", conversions) or {} columns = self.value_or_default("columns", columns) + strict = self.value_or_default("strict", strict) + + if hash_id_columns and hash_id is None: + hash_id = "id" + + if upsert and not pk and not hash_id and self.exists(): + existing_pks = [column.name for column in self.columns if column.is_pk] + if existing_pks: + pk = existing_pks[0] if len(existing_pks) == 1 else tuple(existing_pks) if upsert and (not pk and not hash_id): raise PrimaryKeyRequired("upsert() requires a pk") - assert not (hash_id and pk), "Use either pk= or hash_id=" + + if hash_id and pk: + raise ValueError("Use either pk= or hash_id=") + if hash_id_columns and (hash_id is None): + hash_id = "id" if hash_id: pk = hash_id - assert not ( - ignore and replace - ), "Use either ignore=True or replace=True, not both" - all_columns = None + # pk columns missing from an existing table are an error - unless + # alter=True, where a pk column supplied by the records will be + # added, so validation waits until the record keys are known + deferred_invalid_pk_check = None + if pk and not hash_id and self.exists(): + pk_cols = [pk] if isinstance(pk, str) else list(pk) + existing_columns = self.columns_dict + # rowid and its aliases are valid primary keys for a rowid table + # even though they are not listed among the table's columns + rowid_aliases = ROWID_ALIASES if self.use_rowid else frozenset() + missing_pk_cols = [ + col + for col in pk_cols + if col.lower() not in rowid_aliases + and resolve_casing(col, existing_columns) not in existing_columns + ] + if missing_pk_cols: + invalid_pk_error = InvalidColumns( + "Invalid primary key column{} {} for table {} with columns {}".format( + "s" if len(missing_pk_cols) > 1 else "", + missing_pk_cols, + self.name, + list(existing_columns), + ) + ) + if not alter: + raise invalid_pk_error + deferred_invalid_pk_check = (missing_pk_cols, invalid_pk_error) + + if ignore and replace: + raise ValueError("Use either ignore=True or replace=True, not both") + all_columns = [] first = True num_records_processed = 0 - # We can only handle a max of 999 variables in a SQL insert, so - # we need to adjust the batch_size down if we have too many cols - records = iter(records) - # Peek at first record to count its columns: + + # Detect if we're using list-based iteration or dict-based iteration + list_mode = False + column_names: List[str] = [] + + # Fix up any records with square braces in the column names (only for dict mode) + # We'll handle this differently for list mode + records_iter = iter(records) + + # Peek at first record to determine mode: try: - first_record = next(records) + first_record = next(records_iter) except StopIteration: return self # It was an empty list - num_columns = len(first_record.keys()) - assert ( - num_columns <= SQLITE_MAX_VARS - ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) - batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) + + # Check if this is list mode or dict mode + if isinstance(first_record, (list, tuple)): + # List/tuple mode: first record should be column names + list_mode = True + if not all(isinstance(col, str) for col in first_record): + raise ValueError( + "When using list-based iteration, the first yielded value must be a list of column name strings" + ) + column_names = cast(List[str], list(first_record)) + all_columns = column_names + num_columns = len(column_names) + # Get the actual first data record + try: + first_record = next(records_iter) + except StopIteration: + return self # Only headers, no data + if not isinstance(first_record, (list, tuple)): + raise ValueError( + "After column names list, all subsequent records must also be lists" + ) + else: + # Dict mode: traditional behavior + records_iter = itertools.chain([first_record], records_iter) + try: + first_record = next(records_iter) + except StopIteration: + return self + first_record = cast(Dict[str, Any], first_record) + num_columns = len(first_record.keys()) + + if num_columns > SQLITE_MAX_VARS: + raise ValueError( + "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) + ) + batch_size = ( + 1 + if num_columns == 0 + else max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) + ) self.last_rowid = None self.last_pk = None if truncate and self.exists(): - self.db.execute("DELETE FROM [{}];".format(self.name)) - for chunk in chunks(itertools.chain([first_record], records), batch_size): + with self.db.atomic(): + self.db.execute("DELETE FROM {};".format(quote_identifier(self.name))) + result = None + for chunk in chunks(itertools.chain([first_record], records_iter), batch_size): chunk = list(chunk) num_records_processed += len(chunk) if first: if not self.exists(): # Use the first batch to derive the table names - column_types = suggest_column_types(chunk) + if list_mode: + # Convert list records to dicts for type detection + chunk_as_dicts = [dict(zip(column_names, row)) for row in chunk] + column_types = suggest_column_types(chunk_as_dicts) + else: + dict_chunk = cast(List[Dict[str, Any]], chunk) + column_types = suggest_column_types(dict_chunk) + if extracts: + for col in extracts: + if col in column_types: + column_types[col] = ( + int # This will be an integer foreign key + ) column_types.update(columns or {}) self.create( column_types, @@ -2125,37 +4525,177 @@ class Table(Queryable): not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, extracts=extracts, + strict=strict, ) - all_columns = set() - for record in chunk: - all_columns.update(record.keys()) - all_columns = list(sorted(all_columns)) - if hash_id: - all_columns.insert(0, hash_id) + if list_mode: + # In list mode, columns are already known + all_columns = list(column_names) + if hash_id: + all_columns.insert(0, hash_id) + else: + all_columns_set: Set[str] = set() + for record in cast(List[Dict[str, Any]], chunk): + all_columns_set.update(record.keys()) + all_columns = list(sorted(all_columns_set)) + if hash_id: + all_columns.insert(0, hash_id) + if deferred_invalid_pk_check is not None: + # alter=True - pk columns the table lacks are valid if + # the records supply them, otherwise raise the error + missing_pk_cols, invalid_pk_error = deferred_invalid_pk_check + record_columns = {column: True for column in all_columns} + if any( + resolve_casing(col, record_columns) not in record_columns + for col in missing_pk_cols + ): + raise invalid_pk_error else: - for record in chunk: - all_columns += [ - column for column in record if column not in all_columns - ] + if not list_mode: + for record in cast(List[Dict[str, Any]], chunk): + all_columns += [ + column for column in record if column not in all_columns + ] - validate_column_names(all_columns) first = False - self.insert_chunk( + result = self.insert_chunk( alter, extracts, chunk, all_columns, hash_id, + hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, ignore, + list_mode, ) + # If we only handled a single row populate self.last_pk + if num_records_processed == 1: + # For an insert we need to use result.lastrowid + if not upsert and result is not None: + ignored_insert = ignore and result.rowcount == 0 + if ignored_insert: + # The row was not inserted because it conflicts with an + # existing row. Point last_pk / last_rowid at that existing + # row when we can identify it from the record's primary key + # values, rather than leaving them stale or unset. + if list_mode: + first_record_dict = dict( + zip(column_names, cast(Sequence[Any], first_record)) + ) + else: + first_record_dict = cast(Dict[str, Any], first_record) + if hash_id: + self.last_pk = hash_record(first_record_dict, hash_id_columns) + elif isinstance(pk, str): + self.last_pk = first_record_dict[ + resolve_casing(pk, first_record_dict) + ] + elif pk: + self.last_pk = tuple( + first_record_dict[resolve_casing(p, first_record_dict)] + for p in pk + ) + # Locate the existing conflicting row using its primary key + # columns so we can report its rowid (and pk if not already + # known). Falls back to leaving them unset if the conflict + # cannot be resolved to a pk lookup (e.g. a UNIQUE column). + key_cols: Optional[List[str]] = None + if isinstance(pk, str): + key_cols = [pk] + elif pk: + key_cols = list(pk) + elif not hash_id and not self.use_rowid: + key_cols = self.pks + if key_cols: + try: + key_values = [ + first_record_dict[resolve_casing(c, first_record_dict)] + for c in key_cols + ] + except KeyError: + key_values = None + if key_values is not None: + where = " and ".join( + "{} = ?".format(quote_identifier(c)) for c in key_cols + ) + existing = self.db.execute( + "select rowid from {} where {} limit 1".format( + quote_identifier(self.name), where + ), + key_values, + ).fetchone() + if existing is not None: + self.last_rowid = existing[0] + # On a primary key conflict the record's pk + # values identify the existing row + if self.last_pk is None: + self.last_pk = ( + key_values[0] + if len(key_cols) == 1 + else tuple(key_values) + ) + else: + self.last_rowid = result.lastrowid + # A rowid-alias pk resolves directly to the rowid, so there + # is no separate pk column to look up + rowid_pk = isinstance(pk, str) and pk.lower() in ROWID_ALIASES + if (hash_id or (pk and not rowid_pk)) and self.last_rowid: + # Set self.last_pk to the pk(s) for that rowid + row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] + if hash_id: + self.last_pk = row[hash_id] + elif isinstance(pk, str): + self.last_pk = row[resolve_casing(pk, row)] + else: + self.last_pk = tuple( + row[resolve_casing(p, row)] for p in pk + ) + else: + self.last_pk = self.last_rowid + else: + # For an upsert use first_record from earlier + if list_mode: + # In list mode, look up pk value by column index + first_record_list = cast(Sequence[Any], first_record) + if hash_id: + # hash_id not supported in list mode for last_pk + pass + elif isinstance(pk, str): + pk_index = column_names.index(resolve_casing(pk, column_names)) + self.last_pk = first_record_list[pk_index] + else: + self.last_pk = tuple( + first_record_list[ + column_names.index(resolve_casing(p, column_names)) + ] + for p in pk + ) + else: + first_record_dict = cast(Dict[str, Any], first_record) + if hash_id: + self.last_pk = hash_record(first_record_dict, hash_id_columns) + else: + self.last_pk = ( + first_record_dict[resolve_casing(pk, first_record_dict)] + if isinstance(pk, str) + else tuple( + first_record_dict[resolve_casing(p, first_record_dict)] + for p in pk + ) + ) + + if analyze: + self.analyze() + return self def upsert( @@ -2167,11 +4707,19 @@ class Table(Queryable): not_null=DEFAULT, defaults=DEFAULT, hash_id=DEFAULT, + hash_id_columns=DEFAULT, alter=DEFAULT, extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, - ): + strict=DEFAULT, + ) -> "Table": + """ + Like ``.insert()`` but performs an ``UPSERT``, where records are inserted if they do + not exist and updated if they DO exist, based on matching against their primary key. + + See :ref:`python_api_upsert`. + """ return self.upsert_all( [record], pk=pk, @@ -2180,15 +4728,20 @@ class Table(Queryable): not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, alter=alter, extracts=extracts, conversions=conversions, columns=columns, + strict=strict, ) def upsert_all( self, - records, + records: Union[ + Iterable[Dict[str, Any]], + Iterable[Sequence[Any]], + ], pk=DEFAULT, foreign_keys=DEFAULT, column_order=DEFAULT, @@ -2196,11 +4749,17 @@ class Table(Queryable): defaults=DEFAULT, batch_size=DEFAULT, hash_id=DEFAULT, + hash_id_columns=DEFAULT, alter=DEFAULT, extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, - ): + analyze=False, + strict=DEFAULT, + ) -> "Table": + """ + Like ``.upsert()`` but can be applied to a list of records. + """ return self.insert_all( records, pk=pk, @@ -2210,14 +4769,17 @@ class Table(Queryable): defaults=defaults, batch_size=batch_size, hash_id=hash_id, + hash_id_columns=hash_id_columns, alter=alter, extracts=extracts, conversions=conversions, columns=columns, upsert=True, + analyze=analyze, + strict=strict, ) - def add_missing_columns(self, records): + def add_missing_columns(self, records: Iterable[Dict[str, Any]]) -> "Table": needed_columns = suggest_column_types(records) current_columns = {c.lower() for c in self.columns_dict} for col_name, col_type in needed_columns.items(): @@ -2225,45 +4787,142 @@ class Table(Queryable): self.add_column(col_name, col_type) return self - def lookup(self, column_values): - # lookups is a dictionary - all columns will be used for a unique index - assert isinstance(column_values, dict) + def lookup( + self, + lookup_values: Dict[str, Any], + extra_values: Optional[Dict[str, Any]] = None, + pk: Optional[str] = "id", + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, + conversions: Optional[Dict[str, str]] = None, + columns: Optional[Dict[str, Any]] = None, + strict: Optional[bool] = False, + ): + """ + Create or populate a lookup table with the specified values. + + ``db["Species"].lookup({"name": "Palm"})`` will create a table called ``Species`` + (if one does not already exist) with two columns: ``id`` and ``name``. It will + set up a unique constraint on the ``name`` column to guarantee it will not + contain duplicate rows. + + It will then insert a new row with the ``name`` set to ``Palm`` and return the + new integer primary key value. + + An optional second argument can be provided with more ``name: value`` pairs to + be included only if the record is being created for the first time. These will + be ignored on subsequent lookup calls for records that already exist. + + All other keyword arguments are passed through to ``.insert()``. + + See :ref:`python_api_lookup_tables` for more details. + + :param lookup_values: Dictionary specifying column names and values to use for the lookup + :param extra_values: Additional column values to be used only if creating a new record + :param strict: Boolean, apply STRICT mode if creating the table. + """ + if not isinstance(lookup_values, dict): + raise ValueError("lookup_values must be a dictionary") + if pk is None: + raise ValueError("pk cannot be None") + if extra_values is not None and not isinstance(extra_values, dict): + raise ValueError("extra_values must be a dictionary") + combined_values = dict(lookup_values) + if extra_values is not None: + combined_values.update(extra_values) if self.exists(): - self.add_missing_columns([column_values]) - unique_column_sets = [set(i.columns) for i in self.indexes] - if set(column_values.keys()) not in unique_column_sets: - self.create_index(column_values.keys(), unique=True) - wheres = ["[{}] = ?".format(column) for column in column_values] + self.add_missing_columns([combined_values]) + unique_column_sets = [ + {fold_identifier_case(c) for c in i.columns} for i in self.indexes + ] + if { + fold_identifier_case(c) for c in lookup_values + } not in unique_column_sets: + self.create_index(lookup_values.keys(), unique=True) + # IS rather than = so that null values are matched correctly + wheres = [ + "{} IS ?".format(quote_identifier(column)) for column in lookup_values + ] rows = list( self.rows_where( - " and ".join(wheres), [value for _, value in column_values.items()] + " and ".join(wheres), [value for _, value in lookup_values.items()] ) ) try: - return rows[0]["id"] + return rows[0][resolve_casing(pk, rows[0])] except IndexError: - return self.insert(column_values, pk="id").last_pk + return self.insert( + combined_values, + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + extracts=extracts, + conversions=conversions, + columns=columns, + strict=strict, + ).last_pk else: - pk = self.insert(column_values, pk="id").last_pk - self.create_index(column_values.keys(), unique=True) + pk = self.insert( + combined_values, + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + extracts=extracts, + conversions=conversions, + columns=columns, + strict=strict, + ).last_pk + self.create_index(lookup_values.keys(), unique=True) return pk def m2m( self, - other_table, - record_or_iterable=None, - pk=DEFAULT, - lookup=None, - m2m_table=None, - alter=False, + other_table: Union[str, "Table"], + record_or_iterable: Optional[ + Union[Iterable[Dict[str, Any]], Dict[str, Any]] + ] = None, + pk: Optional[Union[Any, Default]] = DEFAULT, + lookup: Optional[Dict[str, Any]] = None, + m2m_table: Optional[str] = None, + alter: bool = False, ): + """ + After inserting a record in a table, create one or more records in some other + table and then create many-to-many records linking the original record and the + newly created records together. + + For example:: + + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", {"id": 1, "name": "Natalie"}, pk="id" + ) + + See :ref:`python_api_m2m` for details. + + :param other_table: The name of the table to insert the new records into. + :param record_or_iterable: A single dictionary record to insert, or a list of records. + :param pk: The primary key to use if creating ``other_table``. + :param lookup: Same dictionary as for ``.lookup()``, to create a many-to-many lookup table. + :param m2m_table: The string name to use for the many-to-many table, defaults to creating + this automatically based on the names of the two tables. + :param alter: Set to ``True`` to add any missing columns on ``other_table`` if that table + already exists. + """ if isinstance(other_table, str): other_table = self.db.table(other_table, pk=pk) our_id = self.last_pk if lookup is not None: - assert record_or_iterable is None, "Provide lookup= or record, not both" - else: - assert record_or_iterable is not None, "Provide lookup= or record, not both" + if record_or_iterable is not None: + raise ValueError("Provide lookup= or record, not both") + elif record_or_iterable is None: + raise ValueError("Provide lookup= or record, not both") tables = list(sorted([self.name, other_table.name])) columns = ["{}_id".format(t) for t in tables] if m2m_table is not None: @@ -2282,20 +4941,19 @@ class Table(Queryable): else: # If not, create a new table m2m_table_name = m2m_table or "{}_{}".format(*tables) - m2m_table = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) + m2m_table_obj = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) if lookup is None: # if records is only one record, put the record in a list - records = ( - [record_or_iterable] - if isinstance(record_or_iterable, Mapping) - else record_or_iterable - ) + if isinstance(record_or_iterable, Mapping): + records = [record_or_iterable] + else: + records = cast(List, record_or_iterable) # Ensure each record exists in other table for record in records: id = other_table.insert( - record, pk=pk, replace=True, alter=alter + cast(dict, record), pk=pk, replace=True, alter=alter ).last_pk - m2m_table.insert( + m2m_table_obj.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, @@ -2304,7 +4962,7 @@ class Table(Queryable): ) else: id = other_table.lookup(lookup) - m2m_table.insert( + m2m_table_obj.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, @@ -2313,9 +4971,31 @@ class Table(Queryable): ) return self + def analyze(self) -> None: + "Run ANALYZE against this table" + self.db.analyze(self.name) + def analyze_column( - self, column, common_limit=10, value_truncate=None, total_rows=None - ): + self, + column: str, + common_limit: int = 10, + value_truncate=None, + total_rows=None, + most_common: bool = True, + least_common: bool = True, + ) -> "ColumnDetails": + """ + Return statistics about the specified column. + + See :ref:`python_api_analyze_column`. + + :param column: Column to analyze + :param common_limit: Show this many column values + :param value_truncate: Truncate display of common values to this many characters + :param total_rows: Optimization - pass the total number of rows in the table to save running a fresh ``count(*)`` query + :param most_common: If ``True``, calculate the most common values + :param least_common: If ``True``, calculate the least common values + """ db = self.db table = self.name if total_rows is None: @@ -2329,45 +5009,63 @@ class Table(Queryable): value = value[:value_truncate] + "..." return value + table_quoted = quote_identifier(table) + column_quoted = quote_identifier(column) num_null = db.execute( - "select count(*) from [{}] where [{}] is null".format(table, column) + "select count(*) from {} where {} is null".format( + table_quoted, column_quoted + ) ).fetchone()[0] num_blank = db.execute( - "select count(*) from [{}] where [{}] = ''".format(table, column) + "select count(*) from {} where {} = ''".format(table_quoted, column_quoted) ).fetchone()[0] num_distinct = db.execute( - "select count(distinct [{}]) from [{}]".format(column, table) + "select count(distinct {}) from {}".format(column_quoted, table_quoted) ).fetchone()[0] - most_common = None - least_common = None + most_common_results = None + least_common_results = None if num_distinct == 1: value = db.execute( - "select [{}] from [{}] limit 1".format(column, table) + "select {} from {} limit 1".format(column_quoted, table_quoted) ).fetchone()[0] - most_common = [(truncate(value), total_rows)] + most_common_results = [(truncate(value), total_rows)] elif num_distinct != total_rows: - most_common = [ - (truncate(r[0]), r[1]) - for r in db.execute( - "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( - column, table, column, column, common_limit - ) - ).fetchall() - ] - most_common.sort(key=lambda p: (p[1], p[0]), reverse=True) - if num_distinct <= common_limit: - # No need to run the query if it will just return the results in revers order - least_common = None - else: - least_common = [ - (truncate(r[0]), r[1]) - for r in db.execute( - "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format( - column, table, column, column, common_limit - ) - ).fetchall() - ] - least_common.sort(key=lambda p: (p[1], p[0])) + if most_common: + # Optimization - if all rows are null, don't run this query + if num_null == total_rows: + most_common_results = [(None, total_rows)] + else: + most_common_results = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select {}, count(*) from {} group by {} order by count(*) desc, {} limit {}".format( + column_quoted, + table_quoted, + column_quoted, + column_quoted, + common_limit, + ) + ).fetchall() + ] + most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True) + if least_common: + if num_distinct <= common_limit: + # No need to run the query if it will just return the results in reverse order + least_common_results = None + else: + least_common_results = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select {}, count(*) from {} group by {} order by count(*), {} desc limit {}".format( + column_quoted, + table_quoted, + column_quoted, + column_quoted, + common_limit, + ) + ).fetchall() + ] + least_common_results.sort(key=lambda p: (p[1], p[0])) return ColumnDetails( self.name, column, @@ -2375,61 +5073,136 @@ class Table(Queryable): num_null, num_blank, num_distinct, - most_common, - least_common, + most_common_results, + least_common_results, ) + def add_geometry_column( + self, + column_name: str, + geometry_type: str, + srid: int = 4326, + coord_dimension: str = "XY", + not_null: bool = False, + ) -> bool: + """ + In SpatiaLite, a geometry column can only be added to an existing table. + To do so, use ``table.add_geometry_column``, passing in a geometry type. + + By default, this will add a nullable column using + `SRID 4326 `__. This can + be customized using the ``column_name``, ``srid`` and ``not_null`` arguments. + + Returns ``True`` if the column was successfully added, ``False`` if not. + + .. code-block:: python + + from sqlite_utils.db import Database + from sqlite_utils.utils import find_spatialite + + db = Database("mydb.db") + db.init_spatialite(find_spatialite()) + + # the table must exist before adding a geometry column + table = db["locations"].create({"name": str}) + table.add_geometry_column("geometry", "POINT") + + :param column_name: Name of column to add + :param geometry_type: Type of geometry column, for example ``"GEOMETRY"`` or ``"POINT" or ``"POLYGON"`` + :param srid: Integer SRID, defaults to 4326 for WGS84 + :param coord_dimension: Dimensions to use, defaults to ``"XY"`` - set to ``"XYZ"`` to work in three dimensions + :param not_null: Should the column be ``NOT NULL`` + """ + cursor = self.db.execute( + "SELECT AddGeometryColumn(?, ?, ?, ?, ?, ?);", + [ + self.name, + column_name, + srid, + geometry_type, + coord_dimension, + int(not_null), + ], + ) + + result = cursor.fetchone() + return result and bool(result[0]) + + def create_spatial_index(self, column_name) -> bool: + """ + A spatial index allows for significantly faster bounding box queries. + To create one, use ``create_spatial_index`` with the name of an existing geometry column. + + Returns ``True`` if the index was successfully created, ``False`` if not. Calling this + function if an index already exists is a no-op. + + .. code-block:: python + + # assuming SpatiaLite is loaded, create the table, add the column + table = db["locations"].create({"name": str}) + table.add_geometry_column("geometry", "POINT") + + # now we can index it + table.create_spatial_index("geometry") + + # the spatial index is a virtual table, which we can inspect + print(db["idx_locations_geometry"].schema) + # outputs: + # CREATE VIRTUAL TABLE "idx_locations_geometry" USING rtree(pkid, xmin, xmax, ymin, ymax) + + :param column_name: Geometry column to create the spatial index against + """ + if f"idx_{self.name}_{column_name}" in self.db.table_names(): + return False + + cursor = self.db.execute( + "select CreateSpatialIndex(?, ?)", [self.name, column_name] + ) + result = cursor.fetchone() + return result and bool(result[0]) + class View(Queryable): - def exists(self): + def exists(self) -> bool: return True - def __repr__(self): + def __repr__(self) -> str: return "".format( self.name, ", ".join(c.name for c in self.columns) ) - def drop(self, ignore=False): + def drop(self, ignore: bool = False) -> None: + """ + Drop this view. + + :param ignore: Set to ``True`` to ignore the error if the view does not exist + """ + try: - self.db.execute("DROP VIEW [{}]".format(self.name)) + self.db.execute("DROP VIEW {}".format(quote_identifier(self.name))) except sqlite3.OperationalError: if not ignore: raise - def enable_fts(self, *args, **kwargs): - raise NotImplementedError( - "enable_fts() is supported on tables but not on views" - ) - -def chunks(sequence, size): - iterator = iter(sequence) - for item in iterator: - yield itertools.chain([item], itertools.islice(iterator, size - 1)) - - -def jsonify_if_needed(value): +def jsonify_if_needed(value: object) -> object: if isinstance(value, decimal.Decimal): return float(value) if isinstance(value, (dict, list, tuple)): return json.dumps(value, default=repr, ensure_ascii=False) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): return value.isoformat() + elif isinstance(value, datetime.timedelta): + return str(value) elif isinstance(value, uuid.UUID): return str(value) else: return value -def _hash(record): - return hashlib.sha1( - json.dumps(record, separators=(",", ":"), sort_keys=True, default=repr).encode( - "utf8" - ) - ).hexdigest() - - -def resolve_extracts(extracts): +def resolve_extracts( + extracts: Optional[Union[Dict[str, str], List[str], Tuple[str]]], +) -> dict: if extracts is None: extracts = {} if isinstance(extracts, (list, tuple)): @@ -2437,9 +5210,20 @@ def resolve_extracts(extracts): return extracts -def validate_column_names(columns): - # Validate no columns contain '[' or ']' - #86 - for column in columns: - assert ( - "[" not in column and "]" not in column - ), "'[' and ']' cannot be used in column names" +def _decode_default_value(value: str) -> object: + if value.startswith("'") and value.endswith("'"): + # It's a string + return value[1:-1] + if value.isdigit(): + # It's an integer + return int(value) + if value.startswith("X'") and value.endswith("'"): + # It's a binary string, stored as hex + to_decode = value[2:-1] + return binascii.unhexlify(to_decode) + # If it is a string containing a floating point number: + try: + return float(value) + except ValueError: + pass + return value diff --git a/sqlite_utils/hookspecs.py b/sqlite_utils/hookspecs.py new file mode 100644 index 0000000..a746619 --- /dev/null +++ b/sqlite_utils/hookspecs.py @@ -0,0 +1,18 @@ +import sqlite3 + +import click +from pluggy import HookimplMarker +from pluggy import HookspecMarker + +hookspec = HookspecMarker("sqlite_utils") +hookimpl = HookimplMarker("sqlite_utils") + + +@hookspec +def register_commands(cli: click.Group) -> None: + """Register additional CLI commands, e.g. 'sqlite-utils mycommand ...'""" + + +@hookspec +def prepare_connection(conn: sqlite3.Connection) -> None: + """Modify SQLite connection in some way e.g. register custom SQL functions""" diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py new file mode 100644 index 0000000..00d0fa5 --- /dev/null +++ b/sqlite_utils/migrations.py @@ -0,0 +1,177 @@ +from collections.abc import Iterable +from dataclasses import dataclass +import datetime +from typing import Callable, cast, TYPE_CHECKING + +if TYPE_CHECKING: + from sqlite_utils.db import Database, Table + + +class Migrations: + migrations_table = "_sqlite_migrations" + + @dataclass + class _Migration: + name: str + fn: Callable + transactional: bool = True + + @dataclass + class _AppliedMigration: + name: str + # A string timestamp such as "2026-07-04 12:00:00.000000+00:00" - + # stored as TEXT in the _sqlite_migrations table + applied_at: str + + def __init__(self, name: str): + """ + :param name: The name of the migration set. This should be unique. + """ + self.name = name + self._migrations: list[Migrations._Migration] = [] + + def __call__( + self, *, name: str | None = None, transactional: bool = True + ) -> Callable: + """ + :param name: The name to use for this migration - if not provided, + the name of the function will be used. + :param transactional: If ``True`` (the default) the migration and the + record of it having been applied are wrapped in a transaction, which + will be rolled back if the migration raises an exception. Pass + ``False`` for migrations that cannot run inside a transaction, for + example those that execute ``VACUUM``. + """ + + def inner(func: Callable) -> Callable: + migration_name = name or getattr(func, "__name__") + if any(m.name == migration_name for m in self._migrations): + raise ValueError( + "Migration '{}' is already registered in set '{}'".format( + migration_name, self.name + ) + ) + self._migrations.append( + self._Migration(migration_name, func, transactional) + ) + return func + + return inner + + def pending(self, db: "Database") -> list["Migrations._Migration"]: + """ + Return a list of pending migrations. + + This is a read-only operation - it does not write to the database. + """ + already_applied = {migration.name for migration in self.applied(db)} + return [ + migration + for migration in self._migrations + if migration.name not in already_applied + ] + + def applied(self, db: "Database") -> list["Migrations._AppliedMigration"]: + """ + Return a list of applied migrations, in the order they were applied. + + This is a read-only operation - it does not write to the database. + """ + table = _table(db, self.migrations_table) + if not table.exists(): + return [] + return [ + self._AppliedMigration(name=row["name"], applied_at=row["applied_at"]) + for row in table.rows_where( + "migration_set = ?", [self.name], order_by="rowid" + ) + ] + + def apply(self, db: "Database", *, stop_before: str | Iterable[str] | None = None): + """ + Apply any pending migrations to the database. + + Each migration runs inside a transaction, together with the record of + it having been applied - if the migration raises an exception its + changes are rolled back, no record is written and the migration stays + pending. Migrations registered with ``transactional=False`` run + outside of a transaction. + + :raises ValueError: if a ``stop_before`` name matches a migration in + this set that has already been applied - stopping before it is + impossible to honor, and no pending migrations are applied + """ + if stop_before is None: + stop_before_names = set() + elif isinstance(stop_before, str): + stop_before_names = {stop_before} + else: + stop_before_names = set(stop_before) + # A stop_before naming an already-applied migration cannot be + # honored - error rather than applying everything after it. Names + # not in this set at all are ignored, because unqualified CLI + # values are offered to every migration set + already_applied = stop_before_names.intersection( + migration.name for migration in self.applied(db) + ) + if already_applied: + raise ValueError( + "Cannot stop before migration{} {} in set '{}' - already " + "been applied".format( + "s" if len(already_applied) > 1 else "", + ", ".join(sorted(already_applied)), + self.name, + ) + ) + self.ensure_migrations_table(db) + for migration in self.pending(db): + name = migration.name + if name in stop_before_names: + return + if migration.transactional: + with db.atomic(): + migration.fn(db) + self._record_applied(db, name) + else: + migration.fn(db) + self._record_applied(db, name) + + def _record_applied(self, db: "Database", name: str): + _table(db, self.migrations_table).insert( + { + "migration_set": self.name, + "name": name, + "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), + } + ) + + def ensure_migrations_table(self, db: "Database"): + """ + Ensure the _sqlite_migrations table exists and has the correct schema. + """ + table = _table(db, self.migrations_table) + if not table.exists(): + table.create( + { + "id": int, + "migration_set": str, + "name": str, + "applied_at": str, + }, + pk="id", + ) + table.create_index(["migration_set", "name"], unique=True) + elif table.pks != ["id"]: + table.transform(pk="id") + unique_indexes = {tuple(index.columns) for index in table.indexes} + if ("migration_set", "name") not in unique_indexes: + table.create_index(["migration_set", "name"], unique=True) + + def __repr__(self): + return "".format( + self.name, ", ".join(m.name for m in self._migrations) + ) + + +def _table(db: "Database", name: str) -> "Table": + return cast("Table", db[name]) diff --git a/sqlite_utils/plugins.py b/sqlite_utils/plugins.py new file mode 100644 index 0000000..0aff7ff --- /dev/null +++ b/sqlite_utils/plugins.py @@ -0,0 +1,35 @@ +from typing import Dict, List, Union + +import pluggy +import sys +from . import hookspecs + +pm: pluggy.PluginManager = pluggy.PluginManager("sqlite_utils") +pm.add_hookspecs(hookspecs) +_plugins_loaded = False + + +def ensure_plugins_loaded() -> None: + global _plugins_loaded + if _plugins_loaded or getattr(sys, "_called_from_test", False): + return + pm.load_setuptools_entrypoints("sqlite_utils") + _plugins_loaded = True + + +def get_plugins() -> List[Dict[str, Union[str, List[str]]]]: + ensure_plugins_loaded() + plugins: List[Dict[str, Union[str, List[str]]]] = [] + plugin_to_distinfo = dict(pm.list_plugin_distinfo()) + for plugin in pm.get_plugins(): + hookcallers = pm.get_hookcallers(plugin) or [] + plugin_info: Dict[str, Union[str, List[str]]] = { + "name": plugin.__name__, + "hooks": [h.name for h in hookcallers], + } + distinfo = plugin_to_distinfo.get(plugin) + if distinfo: + plugin_info["version"] = distinfo.version + plugin_info["name"] = distinfo.project_name + plugins.append(plugin_info) + return plugins diff --git a/sqlite_utils/py.typed b/sqlite_utils/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py index 6918661..55b55a4 100644 --- a/sqlite_utils/recipes.py +++ b/sqlite_utils/recipes.py @@ -1,19 +1,76 @@ +from __future__ import annotations + +from typing import Callable, Optional + from dateutil import parser import json - -def parsedate(value, dayfirst=False, yearfirst=False): - "Parse a date and convert it to ISO date format: yyyy-mm-dd" - return ( - parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat() - ) +IGNORE: object = object() +SET_NULL: object = object() -def parsedatetime(value, dayfirst=False, yearfirst=False): - "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS" - return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() +def parsedate( + value: str, + dayfirst: bool = False, + yearfirst: bool = False, + errors: Optional[object] = None, +) -> Optional[str]: + """ + Parse a date and convert it to ISO date format: yyyy-mm-dd + \b + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null + """ + if not value: + return value + try: + return ( + parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst) + .date() + .isoformat() + ) + except parser.ParserError: + if errors is IGNORE: + return value + elif errors is SET_NULL: + return None + else: + raise -def jsonsplit(value, delimiter=",", type=str): - 'Convert a string like a,b,c into a JSON array ["a", "b", "c"]' +def parsedatetime( + value: str, + dayfirst: bool = False, + yearfirst: bool = False, + errors: Optional[object] = None, +) -> Optional[str]: + """ + Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS + \b + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null + """ + if not value: + return value + try: + return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() + except parser.ParserError: + if errors is IGNORE: + return value + elif errors is SET_NULL: + return None + else: + raise + + +def jsonsplit( + value: str, delimiter: str = ",", type: Callable[[str], object] = str +) -> str: + """ + Convert a string like a,b,c into a JSON array ["a", "b", "c"] + """ return json.dumps([type(s.strip()) for s in value.split(delimiter)]) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 00a3c02..b39b117 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -2,44 +2,154 @@ import base64 import contextlib import csv import enum +import hashlib +import importlib import io +import itertools import json import os -from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type +import sys +from typing import ( + Any, + BinaryIO, + Callable, + Dict, + Generator, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + Type, + TYPE_CHECKING, + TypeVar, + Union, + cast, +) import click -try: - import pysqlite3 as sqlite3 # type: ignore - import pysqlite3.dbapi2 # type: ignore +from . import recipes - OperationalError = pysqlite3.dbapi2.OperationalError -except ImportError: - # https://github.com/python/mypy/issues/1153#issuecomment-253842414 - import sqlite3 # type: ignore +if TYPE_CHECKING: + import sqlite3 # noqa: F401 + from sqlite3 import dbapi2 # noqa: F401 + + OperationalError = dbapi2.OperationalError +else: + try: + sqlite3 = importlib.import_module("pysqlite3") + dbapi2 = importlib.import_module("pysqlite3.dbapi2") + OperationalError = dbapi2.OperationalError + except ImportError: + import sqlite3 # noqa: F401 + from sqlite3 import dbapi2 # noqa: F401 + + OperationalError = dbapi2.OperationalError - OperationalError = sqlite3.OperationalError SPATIALITE_PATHS = ( "/usr/lib/x86_64-linux-gnu/mod_spatialite.so", + "/usr/lib/aarch64-linux-gnu/mod_spatialite.so", "/usr/local/lib/mod_spatialite.dylib", + "/usr/local/lib/mod_spatialite.so", + "/opt/homebrew/lib/mod_spatialite.dylib", ) +# Mainly so we can restore it if needed in the tests: +ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit() -def suggest_column_types(records): - all_column_types = {} +# Type alias for row dictionaries - values can be various SQLite-compatible types +RowValue = Union[None, int, float, str, bytes, bool, List[str]] +Row = Dict[str, RowValue] + +T = TypeVar("T") + + +class _CloseableIterator(Iterator[Row]): + """Iterator wrapper that closes a file when iteration is complete.""" + + def __init__(self, iterator: Iterator[Row], closeable: io.IOBase) -> None: + self._iterator = iterator + self._closeable = closeable + + def __iter__(self) -> "_CloseableIterator": + return self + + def __next__(self) -> Row: + try: + return next(self._iterator) + except StopIteration: + self._closeable.close() + raise + + def close(self) -> None: + self._closeable.close() + + +def maximize_csv_field_size_limit() -> None: + """ + Increase the CSV field size limit to the maximum possible. + """ + # https://stackoverflow.com/a/15063941 + field_size_limit = sys.maxsize + + while True: + try: + csv.field_size_limit(field_size_limit) + break + except OverflowError: + field_size_limit = int(field_size_limit / 10) + + +def find_spatialite() -> Optional[str]: + """ + The ``find_spatialite()`` function searches for the `SpatiaLite `__ + SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. + + You can use it in code like this: + + .. code-block:: python + + from sqlite_utils import Database + from sqlite_utils.utils import find_spatialite + + db = Database("mydb.db") + spatialite = find_spatialite() + if spatialite: + db.conn.enable_load_extension(True) + db.conn.load_extension(spatialite) + + # or use with db.init_spatialite like this + db.init_spatialite(find_spatialite()) + + """ + for path in SPATIALITE_PATHS: + if os.path.exists(path): + return path + return None + + +def suggest_column_types( + records: Iterable[Dict[str, Any]], +) -> Dict[str, type]: + all_column_types: Dict[str, Set[type]] = {} for record in records: for key, value in record.items(): all_column_types.setdefault(key, set()).add(type(value)) return types_for_column_types(all_column_types) -def types_for_column_types(all_column_types): - column_types = {} +def types_for_column_types( + all_column_types: Dict[str, Set[type]], +) -> Dict[str, type]: + column_types: Dict[str, type] = {} for key, types in all_column_types.items(): # Ignore null values if at least one other type present: if len(types) > 1: types.discard(None.__class__) + t: type if {None.__class__} == types: t = str elif len(types) == 1: @@ -61,7 +171,7 @@ def types_for_column_types(all_column_types): return column_types -def column_affinity(column_type): +def column_affinity(column_type: str) -> type: # Implementation of SQLite affinity rules from # https://www.sqlite.org/datatype3.html#determination_of_column_affinity assert isinstance(column_type, str) @@ -80,40 +190,42 @@ def column_affinity(column_type): return float -def decode_base64_values(doc): +def decode_base64_values(doc: Dict[str, Any]) -> Dict[str, Any]: # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them to_fix = [ k for k in doc if isinstance(doc[k], dict) - and doc[k].get("$base64") is True - and "encoded" in doc[k] + and cast(dict, doc[k]).get("$base64") is True + and "encoded" in cast(dict, doc[k]) ] if not to_fix: return doc - return dict(doc, **{k: base64.b64decode(doc[k]["encoded"]) for k in to_fix}) - - -def find_spatialite(): - for path in SPATIALITE_PATHS: - if os.path.exists(path): - return path - return None + return dict( + doc, **{k: base64.b64decode(cast(dict, doc[k])["encoded"]) for k in to_fix} + ) class UpdateWrapper: - def __init__(self, wrapped, update): + def __init__(self, wrapped: io.IOBase, update: Callable[[int], None]) -> None: self._wrapped = wrapped self._update = update - def __iter__(self): + def __iter__(self) -> Iterator[bytes]: for line in self._wrapped: self._update(len(line)) yield line + def read(self, size: int = -1) -> bytes: + data = self._wrapped.read(size) + self._update(len(data)) + return data + @contextlib.contextmanager -def file_progress(file, silent=False, **kwargs): +def file_progress( + file: io.IOBase, silent: bool = False, **kwargs: object +) -> Generator[Union[io.IOBase, "UpdateWrapper"], None, None]: if silent: yield file return @@ -126,8 +238,8 @@ def file_progress(file, silent=False, **kwargs): if fileno == 0: # 0 means stdin yield file else: - file_length = os.path.getsize(file.name) - with click.progressbar(length=file_length, **kwargs) as bar: + file_length = os.path.getsize(file.name) # type: ignore + with click.progressbar(length=file_length, **kwargs) as bar: # type: ignore yield UpdateWrapper(file, bar.update) @@ -146,12 +258,87 @@ class RowsFromFileBadJSON(RowsFromFileError): pass +class RowError(Exception): + pass + + +def _extra_key_strategy( + reader: Iterable[Dict[Optional[str], object]], + ignore_extras: Optional[bool] = False, + extras_key: Optional[str] = None, +) -> Iterable[Row]: + # Logic for handling CSV rows with more values than there are headings + for row in reader: + # DictReader adds a 'None' key with extra row values + if None not in row: + yield cast(Row, row) + elif ignore_extras: + # ignoring row.pop(none) because of this issue: + # https://github.com/simonw/sqlite-utils/issues/440#issuecomment-1155358637 + row.pop(None) + yield cast(Row, row) + elif not extras_key: + extras = row.pop(None) + raise RowError( + "Row {} contained these extra values: {}".format(row, extras) + ) + else: + extras_value = row.pop(None) + row_out = cast(Row, row) + row_out[extras_key] = cast(RowValue, extras_value) + yield row_out + + def rows_from_file( fp: BinaryIO, format: Optional[Format] = None, dialect: Optional[Type[csv.Dialect]] = None, encoding: Optional[str] = None, -) -> Tuple[Iterable[dict], Format]: + ignore_extras: Optional[bool] = False, + extras_key: Optional[str] = None, +) -> Tuple[Iterable[Row], Format]: + """ + Load a sequence of dictionaries from a file-like object containing one of four different formats. + + .. code-block:: python + + from sqlite_utils.utils import rows_from_file + import io + + rows, format = rows_from_file(io.StringIO("id,name\\n1,Cleo"))) + print(list(rows), format) + # Outputs [{'id': '1', 'name': 'Cleo'}] Format.CSV + + This defaults to attempting to automatically detect the format of the data, or you can pass in an + explicit format using the format= option. + + Returns a tuple of ``(rows_generator, format_used)`` where ``rows_generator`` can be iterated over + to return dictionaries, while ``format_used`` is a value from the ``sqlite_utils.utils.Format`` enum: + + .. code-block:: python + + class Format(enum.Enum): + CSV = 1 + TSV = 2 + JSON = 3 + NL = 4 + + If a CSV or TSV file includes rows with more fields than are declared in the header a + ``sqlite_utils.utils.RowError`` exception will be raised when you loop over the generator. + + You can instead ignore the extra data by passing ``ignore_extras=True``. + + Or pass ``extras_key="rest"`` to put those additional values in a list in a key called ``rest``. + + :param fp: a file-like object containing binary data + :param format: the format to use - omit this to detect the format + :param dialect: the CSV dialect to use - omit this to detect the dialect + :param encoding: the character encoding to use when reading CSV/TSV data + :param ignore_extras: ignore any extra fields on rows + :param extras_key: put any extra fields in a list with this key + """ + if ignore_extras and extras_key: + raise ValueError("Cannot use ignore_extras= and extras_key= together") if format == Format.JSON: decoded = json.load(fp) if isinstance(decoded, dict): @@ -168,18 +355,30 @@ def rows_from_file( reader = csv.DictReader(decoded_fp, dialect=dialect) else: reader = csv.DictReader(decoded_fp) - return reader, Format.CSV + rows = _extra_key_strategy(reader, ignore_extras, extras_key) + return _CloseableIterator(iter(rows), decoded_fp), Format.CSV elif format == Format.TSV: + rows, _ = rows_from_file( + fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding + ) return ( - rows_from_file( - fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding - )[0], + _extra_key_strategy( + cast(Iterable[Dict[Optional[str], object]], rows), + ignore_extras, + extras_key, + ), Format.TSV, ) elif format is None: # Detect the format, then call this recursively buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096) - first_bytes = buffered.peek(2048).strip() + try: + first_bytes = buffered.peek(2048).strip() + except AttributeError: + # Likely the user passed a TextIO when this needs a BytesIO + raise TypeError( + "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO" + ) if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): # TODO: Detect newline-JSON return rows_from_file(buffered, format=Format.JSON) @@ -187,18 +386,54 @@ def rows_from_file( dialect = csv.Sniffer().sniff( first_bytes.decode(encoding or "utf-8-sig", "ignore") ) - return rows_from_file( + rows, _ = rows_from_file( buffered, format=Format.CSV, dialect=dialect, encoding=encoding ) + # Make sure we return the format we detected + detected_format = Format.TSV if dialect.delimiter == "\t" else Format.CSV + return ( + _extra_key_strategy( + cast(Iterable[Dict[Optional[str], object]], rows), + ignore_extras, + extras_key, + ), + detected_format, + ) else: raise RowsFromFileError("Bad format") class TypeTracker: - def __init__(self): - self.trackers = {} + """ + Wrap an iterator of dictionaries and keep track of which SQLite column + types are the most likely fit for each of their keys. - def wrap(self, iterator): + Example usage: + + .. code-block:: python + + from sqlite_utils.utils import TypeTracker + import sqlite_utils + + db = sqlite_utils.Database(memory=True) + tracker = TypeTracker() + rows = [{"id": "1", "name": "Cleo", "id": "2", "name": "Cardi"}] + db["creatures"].insert_all(tracker.wrap(rows)) + print(tracker.types) + # Outputs {'id': 'integer', 'name': 'text'} + db["creatures"].transform(types=tracker.types) + """ + + def __init__(self) -> None: + self.trackers: Dict[str, "ValueTracker"] = {} + + def wrap(self, iterator: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]: + """ + Use this to loop through an existing iterator, tracking the column types + as part of the iteration. + + :param iterator: The iterator to wrap + """ for row in iterator: for key, value in row.items(): tracker = self.trackers.setdefault(key, ValueTracker()) @@ -206,41 +441,47 @@ class TypeTracker: yield row @property - def types(self): + def types(self) -> Dict[str, str]: + """ + A dictionary mapping column names to their detected types. This can be passed + to the ``db[table_name].transform(types=tracker.types)`` method. + """ return {key: tracker.guessed_type for key, tracker in self.trackers.items()} class ValueTracker: - def __init__(self): + couldbe: Dict[str, Callable[[object], bool]] + + def __init__(self) -> None: self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()} @classmethod - def get_tests(cls): + def get_tests(cls) -> List[str]: return [ key.split("test_")[-1] for key in cls.__dict__.keys() if key.startswith("test_") ] - def test_integer(self, value): + def test_integer(self, value: object) -> bool: try: - int(value) + int(cast(Any, value)) return True except (ValueError, TypeError): return False - def test_float(self, value): + def test_float(self, value: object) -> bool: try: - float(value) + float(cast(Any, value)) return True except (ValueError, TypeError): return False - def __repr__(self): + def __repr__(self) -> str: return self.guessed_type + ": possibilities = " + repr(self.couldbe) @property - def guessed_type(self): + def guessed_type(self) -> str: options = set(self.couldbe.keys()) # Return based on precedence for key in self.get_tests(): @@ -248,10 +489,10 @@ class ValueTracker: return key return "text" - def evaluate(self, value): + def evaluate(self, value: object) -> None: if not value or not self.couldbe: return - not_these = [] + not_these: List[str] = [] for name, test in self.couldbe.items(): if not test(value): not_these.append(name) @@ -260,21 +501,162 @@ class ValueTracker: class NullProgressBar: - def __init__(self, *args): + def __init__(self, *args: Iterable[T]) -> None: self.args = args - def __iter__(self): - yield from self.args[0] + def __iter__(self) -> Iterator[T]: + yield from self.args[0] # type: ignore - def update(self, value): + def update(self, value: int) -> None: pass @contextlib.contextmanager -def progressbar(*args, **kwargs): +def progressbar(*args: Iterable[T], **kwargs: Any) -> Generator[Any, None, None]: silent = kwargs.pop("silent") if silent: yield NullProgressBar(*args) else: - with click.progressbar(*args, **kwargs) as bar: + with click.progressbar(*args, **kwargs) as bar: # type: ignore yield bar + + +def _compile_code( + code: str, imports: Iterable[str], variable: str = "value" +) -> Callable[..., Any]: + globals_dict: Dict[str, Any] = {"r": recipes, "recipes": recipes} + # Handle imports first so they're available for all approaches + for import_ in imports: + globals_dict[import_.split(".")[0]] = __import__(import_) + + # If user defined a convert() function, return that + try: + exec(code, globals_dict) + return cast(Callable[..., object], globals_dict["convert"]) + except (AttributeError, SyntaxError, NameError, KeyError, TypeError): + pass + + # Check if code is a direct callable reference + # e.g. "r.parsedate" instead of "r.parsedate(value)" + try: + fn = eval(code, globals_dict) + if callable(fn): + return cast(Callable[..., object], fn) + except Exception: + pass + + # Try compiling their code as a function instead + body_variants = [code] + # If single line and no 'return', try adding the return + if "\n" not in code and not code.strip().startswith("return "): + body_variants.insert(0, "return {}".format(code)) + + code_o = None + for variant in body_variants: + new_code = ["def fn({}):".format(variable)] + for line in variant.split("\n"): + new_code.append(" {}".format(line)) + try: + code_o = compile("\n".join(new_code), "", "exec") + break + except SyntaxError: + # Try another variant, e.g. for 'return row["column"] = 1' + continue + + if code_o is None: + raise SyntaxError("Could not compile code") + + exec(code_o, globals_dict) + return cast(Callable[..., object], globals_dict["fn"]) + + +def chunks(sequence: Iterable[T], size: int) -> Iterable[Iterable[T]]: + """ + Iterate over chunks of the sequence of the given size. + + :param sequence: Any Python iterator + :param size: The size of each chunk + """ + iterator = iter(sequence) + for item in iterator: + yield itertools.chain([item], itertools.islice(iterator, size - 1)) + + +def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> str: + """ + ``record`` should be a Python dictionary. Returns a sha1 hash of the + keys and values in that record. + + If ``keys=`` is provided, uses just those keys to generate the hash. + + Example usage:: + + from sqlite_utils.utils import hash_record + + hashed = hash_record({"name": "Cleo", "twitter": "CleoPaws"}) + # Or with the keys= option: + hashed = hash_record( + {"name": "Cleo", "twitter": "CleoPaws", "age": 7}, + keys=("name", "twitter") + ) + + :param record: Record to generate a hash for + :param keys: Subset of keys to use for that hash + """ + to_hash: Dict[str, Any] = record + if keys is not None: + to_hash = {key: record[key] for key in keys} + return hashlib.sha1( + json.dumps(to_hash, separators=(",", ":"), sort_keys=True, default=repr).encode( + "utf8" + ) + ).hexdigest() + + +def dedupe_keys(keys: Iterable[str]) -> List[str]: + """ + Rename duplicates in a list of column names so every name is unique, + by appending ``_2``, ``_3``... to later occurrences - skipping any + suffix that would collide with another column in the list. + + Used when converting SQL query rows to dictionaries, where duplicate + column names would otherwise silently overwrite each other. + + :param keys: List of column names, possibly containing duplicates + """ + keys = list(keys) + taken = set(keys) + if len(taken) == len(keys): + # No duplicates - the common case + return keys + seen: set = set() + result = [] + for key in keys: + if key in seen: + new_key = key + suffix = 2 + while new_key in seen or new_key in taken: + new_key = "{}_{}".format(key, suffix) + suffix += 1 + key = new_key + seen.add(key) + result.append(key) + return result + + +def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]: + for key, value in d.items(): + if isinstance(value, dict): + for key2, value2 in _flatten(value): + yield key + "_" + key2, value2 + else: + yield key, value + + +def flatten(row: Dict[str, Any]) -> Dict[str, Any]: + """ + Turn a nested dict e.g. ``{"a": {"b": 1}}`` into a flat dict: ``{"a_b": 1}`` + + :param row: A Python dictionary, optionally with nested dictionaries + """ + return dict(_flatten(row)) diff --git a/tests/conftest.py b/tests/conftest.py index 4541689..728db7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,63 @@ from sqlite_utils import Database +from sqlite_utils.utils import sqlite3 import pytest +CREATE_TABLES = """ +create table Gosh (c1 text, c2 text, c3 text); +create table Gosh2 (c1 text, c2 text, c3 text); +""" + + +def pytest_addoption(parser): + parser.addoption( + "--sqlite-autocommit", + action="store_true", + default=False, + help=( + "Run every test against connections created with the Python 3.12+ " + "sqlite3.connect(autocommit=True) mode" + ), + ) + + +def pytest_configure(config): + import sys + + sys._called_from_test = True # type: ignore[attr-defined] + + if config.getoption("--sqlite-autocommit"): + if sys.version_info < (3, 12): + raise pytest.UsageError( + "--sqlite-autocommit requires Python 3.12 or higher" + ) + real_connect = sqlite3.connect + + def autocommit_connect(*args, **kwargs): + kwargs.setdefault("autocommit", True) + return real_connect(*args, **kwargs) + + sqlite3.connect = autocommit_connect + + +@pytest.fixture(autouse=True) +def close_all_databases(): + """Automatically close all Database objects created during a test.""" + databases = [] + original_init = Database.__init__ + + def tracking_init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + databases.append(self) + + Database.__init__ = tracking_init # type: ignore[method-assign] + yield + Database.__init__ = original_init # type: ignore[method-assign] + for db in databases: + try: + db.close() + except Exception: + pass + @pytest.fixture def fresh_db(): @@ -10,12 +67,19 @@ def fresh_db(): @pytest.fixture def existing_db(): database = Database(memory=True) - database.executescript( - """ + database.executescript(""" CREATE TABLE foo (text TEXT); INSERT INTO foo (text) values ("one"); INSERT INTO foo (text) values ("two"); INSERT INTO foo (text) values ("three"); - """ - ) + """) return database + + +@pytest.fixture +def db_path(tmpdir): + path = str(tmpdir / "test.db") + db = sqlite3.connect(path) + db.executescript(CREATE_TABLES) + db.close() + return path diff --git a/tests/ext.c b/tests/ext.c new file mode 100644 index 0000000..f5b3276 --- /dev/null +++ b/tests/ext.c @@ -0,0 +1,48 @@ +/* +** This file implements a SQLite extension with multiple entrypoints. +** +** The default entrypoint, sqlite3_ext_init, has a single function "a". +** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b". +** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c". +** +** Compiling instructions: +** https://www.sqlite.org/loadext.html#compiling_a_loadable_extension +** +*/ + +#include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1 + +// SQL function that returns back the value supplied during sqlite3_create_function() +static void func(sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC); +} + + +// The default entrypoint, since it matches the "ext.dylib"/"ext.so" name +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0); +} + +// Alternate entrypoint #1 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0); +} + +// Alternate entrypoint #2 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0); +} diff --git a/tests/test_analyze.py b/tests/test_analyze.py new file mode 100644 index 0000000..a4cd8a2 --- /dev/null +++ b/tests/test_analyze.py @@ -0,0 +1,51 @@ +import pytest + + +@pytest.fixture +def db(fresh_db): + fresh_db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db["one_index"].create_index(["name"]) + fresh_db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") + fresh_db["two_indexes"].create_index(["name"]) + fresh_db["two_indexes"].create_index(["species"]) + return fresh_db + + +def test_analyze_whole_database(db): + assert set(db.table_names()) == {"one_index", "two_indexes"} + db.analyze() + assert set(db.table_names()).issuperset( + {"one_index", "two_indexes", "sqlite_stat1"} + ) + assert list(db["sqlite_stat1"].rows) == [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, + {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, + ] + + +@pytest.mark.parametrize("method", ("db_method_with_name", "table_method")) +def test_analyze_one_table(db, method): + assert set(db.table_names()).issuperset({"one_index", "two_indexes"}) + if method == "db_method_with_name": + db.analyze("one_index") + elif method == "table_method": + db["one_index"].analyze() + + assert set(db.table_names()).issuperset( + {"one_index", "two_indexes", "sqlite_stat1"} + ) + assert list(db["sqlite_stat1"].rows) == [ + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"} + ] + + +def test_analyze_index_by_name(db): + assert set(db.table_names()) == {"one_index", "two_indexes"} + db.analyze("idx_two_indexes_species") + assert set(db.table_names()).issuperset( + {"one_index", "two_indexes", "sqlite_stat1"} + ) + assert list(db["sqlite_stat1"].rows) == [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, + ] diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 5795a7a..a2ce585 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -24,11 +24,35 @@ def db_to_analyze(fresh_db): return fresh_db +@pytest.fixture +def big_db_to_analyze_path(tmpdir): + path = str(tmpdir / "test.db") + db = Database(path) + categories = { + "A": 40, + "B": 30, + "C": 20, + "D": 10, + } + to_insert = [] + for category, count in categories.items(): + for _ in range(count): + to_insert.append( + { + "category": category, + "all_null": None, + } + ) + db["stuff"].insert_all(to_insert) + return path + + @pytest.mark.parametrize( - "column,expected", + "column,extra_kwargs,expected", [ ( "id", + {}, ColumnDetails( table="stuff", column="id", @@ -42,6 +66,7 @@ def db_to_analyze(fresh_db): ), ( "owner", + {}, ColumnDetails( table="stuff", column="owner", @@ -55,6 +80,7 @@ def db_to_analyze(fresh_db): ), ( "size", + {}, ColumnDetails( table="stuff", column="size", @@ -66,11 +92,41 @@ def db_to_analyze(fresh_db): least_common=None, ), ), + ( + "owner", + {"most_common": False}, + ColumnDetails( + table="stuff", + column="owner", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=4, + most_common=None, + least_common=[("Anne", 1), ("Terry...", 2)], + ), + ), + ( + "owner", + {"least_common": False}, + ColumnDetails( + table="stuff", + column="owner", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=4, + most_common=[("Joan", 3), ("Kumar", 2)], + least_common=None, + ), + ), ], ) -def test_analyze_column(db_to_analyze, column, expected): +def test_analyze_column(db_to_analyze, column, extra_kwargs, expected): assert ( - db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5) + db_to_analyze["stuff"].analyze_column( + column, common_limit=2, value_truncate=5, **extra_kwargs + ) == expected ) @@ -79,16 +135,15 @@ def test_analyze_column(db_to_analyze, column, expected): def db_to_analyze_path(db_to_analyze, tmpdir): path = str(tmpdir / "test.db") db = sqlite3.connect(path) - db.executescript("\n".join(db_to_analyze.conn.iterdump())) + sql = "\n".join(db_to_analyze.iterdump()) + db.executescript(sql) + db.close() return path def test_analyze_table(db_to_analyze_path): result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) - assert ( - result.output.strip() - == ( - """ + assert result.output.strip() == (""" stuff.id: (1/3) Total rows: 8 @@ -121,9 +176,7 @@ stuff.size: (3/3) Most common: 5: 5 - 3: 4""" - ).strip() - ) + 3: 4""").strip() def test_analyze_table_save(db_to_analyze_path): @@ -164,3 +217,101 @@ def test_analyze_table_save(db_to_analyze_path): "least_common": None, }, ] + + +@pytest.mark.parametrize( + "no_most,no_least", + ( + (False, False), + (True, False), + (False, True), + (True, True), + ), +) +def test_analyze_table_save_no_most_no_least_options( + no_most, no_least, big_db_to_analyze_path +): + args = [ + "analyze-tables", + big_db_to_analyze_path, + "--save", + "--common-limit", + "2", + "--column", + "category", + ] + if no_most: + args.append("--no-most") + if no_least: + args.append("--no-least") + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0 + rows = list(Database(big_db_to_analyze_path)["_analyze_tables_"].rows) + expected = { + "table": "stuff", + "column": "category", + "total_rows": 100, + "num_null": 0, + "num_blank": 0, + "num_distinct": 4, + "most_common": None, + "least_common": None, + } + if not no_most: + expected["most_common"] = '[["A", 40], ["B", 30]]' + if not no_least: + expected["least_common"] = '[["D", 10], ["C", 20]]' + + assert rows == [expected] + + +def test_analyze_table_column_all_nulls(big_db_to_analyze_path): + result = CliRunner().invoke( + cli.cli, + ["analyze-tables", big_db_to_analyze_path, "stuff", "--column", "all_null"], + ) + assert result.exit_code == 0 + assert result.output == ( + "stuff.all_null: (1/1)\n\n Total rows: 100\n" + " Null rows: 100\n" + " Blank rows: 0\n" + "\n" + " Distinct values: 0\n\n" + ) + + +@pytest.mark.parametrize( + "args,expected_error", + ( + (["-c", "bad_column"], "These columns were not found: bad_column\n"), + (["one", "-c", "age"], "These columns were not found: age\n"), + (["two", "-c", "age"], None), + ( + ["one", "-c", "age", "--column", "bad"], + "These columns were not found: age, bad\n", + ), + ), +) +def test_analyze_table_validate_columns(tmpdir, args, expected_error): + path = str(tmpdir / "test_validate_columns.db") + db = Database(path) + db["one"].insert( + { + "id": 1, + "name": "one", + } + ) + db["two"].insert( + { + "id": 1, + "age": 5, + } + ) + result = CliRunner().invoke( + cli.cli, + ["analyze-tables", path] + args, + catch_exceptions=False, + ) + assert result.exit_code == (1 if expected_error else 0) + if expected_error: + assert expected_error in result.output diff --git a/tests/test_atomic.py b/tests/test_atomic.py new file mode 100644 index 0000000..c3fd02f --- /dev/null +++ b/tests/test_atomic.py @@ -0,0 +1,382 @@ +import pytest + +from sqlite_utils.db import Database, _iter_complete_sql_statements +from sqlite_utils.utils import sqlite3 + + +@pytest.mark.parametrize( + "sql,expected", + ( + ( + "CREATE TABLE t(id); INSERT INTO t VALUES (1)", + ["CREATE TABLE t(id);", "INSERT INTO t VALUES (1)"], + ), + ( + "INSERT INTO t VALUES ('a;b');", + ["INSERT INTO t VALUES ('a;b');"], + ), + ( + "-- comment;\nCREATE TABLE t(id);", + ["-- comment;\nCREATE TABLE t(id);"], + ), + ( + """ + CREATE TRIGGER t_ai AFTER INSERT ON t + BEGIN + UPDATE t SET value = 'a;b' WHERE id = new.id; + INSERT INTO log VALUES ('x;y'); + END; + """, + [ + "CREATE TRIGGER t_ai AFTER INSERT ON t\n" + " BEGIN\n" + " UPDATE t SET value = 'a;b' WHERE id = new.id;\n" + " INSERT INTO log VALUES ('x;y');\n" + " END;" + ], + ), + ), +) +def test_iter_complete_sql_statements(sql, expected): + assert list(_iter_complete_sql_statements(sql)) == expected + + +def test_atomic_commits(fresh_db): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + + assert list(fresh_db["dogs"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_atomic_rolls_back(fresh_db): + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + raise RuntimeError("boom") + + assert not fresh_db["dogs"].exists() + + +def test_nested_atomic_rolls_back_to_savepoint(fresh_db): + fresh_db["dogs"].create({"id": int, "name": str}, pk="id") + + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}) + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") + fresh_db["dogs"].insert({"id": 3, "name": "Marnie"}) + + assert list(fresh_db["dogs"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 3, "name": "Marnie"}, + ] + + +def test_outer_atomic_rolls_back_released_savepoint(fresh_db): + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") + + assert not fresh_db["dogs"].exists() + + +def test_executescript_does_not_commit_open_atomic_block(fresh_db): + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db.executescript(""" + CREATE TABLE dogs(id INTEGER PRIMARY KEY, name TEXT); + CREATE TRIGGER dogs_ai AFTER INSERT ON dogs + BEGIN + UPDATE dogs SET name = upper(new.name) || '; updated' WHERE id = new.id; + END; + -- This comment has a semicolon; + INSERT INTO dogs VALUES (1, 'Cleo; the first'); + """) + raise RuntimeError("boom") + + assert not fresh_db["dogs"].exists() + + +def test_transform_does_not_commit_open_atomic_block(fresh_db): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") + + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) + fresh_db["dogs"].transform(rename={"age": "dog_age"}) + raise RuntimeError("boom") + + assert ( + fresh_db["dogs"].schema + == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n)' + ) + assert list(fresh_db["dogs"].rows) == [ + {"id": 1, "name": "Cleo", "age": "5"}, + ] + + +def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db["books"].insert( + {"id": 1, "title": "Book", "author_id": 1}, + pk="id", + foreign_keys={"author_id"}, + ) + + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "full_name"}) + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + assert ( + fresh_db["authors"].schema + == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "full_name" TEXT\n)' + ) + assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == [] + + +def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db["books"].insert( + {"id": 1, "title": "Book", "author_id": 1}, + pk="id", + foreign_keys={"author_id"}, + ) + + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "full_name"}) + raise RuntimeError("boom") + + assert ( + fresh_db["authors"].schema + == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n)' + ) + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == [] + + +def test_transform_detects_foreign_key_check_violations(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 2}, pk="id") + + with pytest.raises(sqlite3.IntegrityError): + fresh_db["books"].transform(add_foreign_keys=(("author_id", "authors", "id"),)) + + assert fresh_db["books"].foreign_keys == [] + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db): + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.execute("begin") + with fresh_db.atomic(): + fresh_db["t"].insert({"id": 2}, pk="id") + # Nothing is committed until the user's own transaction commits + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + # And with a commit instead, the atomic block's writes persist + fresh_db.execute("begin") + with fresh_db.atomic(): + fresh_db["t"].insert({"id": 3}, pk="id") + fresh_db.commit() + assert [r["id"] for r in fresh_db["t"].rows] == [1, 3] + + +def test_begin_commit_rollback(tmpdir): + path = str(tmpdir / "test.db") + db = Database(path) + db["t"].insert({"id": 1}, pk="id") + db.begin() + db["t"].insert({"id": 2}, pk="id") + assert db.conn.in_transaction + db.rollback() + assert not db.conn.in_transaction + assert [r["id"] for r in db["t"].rows] == [1] + db.begin() + db["t"].insert({"id": 3}, pk="id") + db.commit() + db.close() + db2 = Database(path) + assert [r["id"] for r in db2["t"].rows] == [1, 3] + db2.close() + + +def test_begin_inside_transaction_errors(fresh_db): + fresh_db.begin() + with pytest.raises(sqlite3.OperationalError): + fresh_db.begin() + fresh_db.rollback() + + +def test_commit_and_rollback_without_transaction_are_noops(fresh_db): + fresh_db.commit() + fresh_db.rollback() + assert not fresh_db.conn.in_transaction + + +def test_execute_write_commits_immediately(tmpdir): + path = str(tmpdir / "test.db") + db = Database(path) + db["t"].insert({"id": 1}, pk="id") + db.execute("insert into t (id) values (2)") + # No implicit transaction is left open + assert not db.conn.in_transaction + # A completely separate connection sees the row straight away + other = sqlite3.connect(path) + assert other.execute("select count(*) from t").fetchone()[0] == 2 + other.close() + db.close() + + +def test_execute_write_respects_explicit_transaction(fresh_db): + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.begin() + fresh_db.execute("insert into t (id) values (2)") + # Still inside the explicit transaction - not committed + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + + +def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): + # A BEGIN hidden behind a leading comment must not be auto-committed + # out from under the caller + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.execute("-- start a transaction\nbegin") + assert fresh_db.conn.in_transaction + fresh_db.execute("insert into t (id) values (2)") + fresh_db.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + + +def _sqlite_accepts_bom(): + try: + sqlite3.connect(":memory:").execute("\ufeffselect 1") + return True + except sqlite3.OperationalError: + return False + + +@pytest.mark.parametrize("begin_sql", ["; begin", "\ufeffbegin"]) +def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql): + # sqlite3 tolerates empty statements and a UTF-8 BOM before the first + # real token, so a BEGIN behind either must not be auto-committed + # out from under the caller + if begin_sql.startswith("\ufeff") and not _sqlite_accepts_bom(): + pytest.skip("This SQLite version rejects a leading byte order mark") + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.execute(begin_sql) + assert fresh_db.conn.in_transaction + fresh_db.execute("insert into t (id) values (2)") + fresh_db.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + + +def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): + # A failed write must not leave the driver's implicit transaction open - + # that would silently disable auto-commit for every subsequent write + path = str(tmpdir / "test.db") + db = Database(path) + db["t"].insert({"id": 1}, pk="id") + with pytest.raises(sqlite3.IntegrityError): + db.execute("insert into t (id) values (1)") + assert not db.conn.in_transaction + # Subsequent writes commit as normal and survive closing the connection + db["other"].insert({"id": 2}) + db.close() + db2 = Database(path) + assert db2["other"].exists() + db2.close() + + +def test_execute_failed_write_preserves_explicit_transaction(fresh_db): + # A failed write inside an explicit transaction must not roll back + # the caller's earlier work - only the caller decides that + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.begin() + fresh_db.execute("insert into t (id) values (2)") + with pytest.raises(sqlite3.IntegrityError): + fresh_db.execute("insert into t (id) values (1)") + assert fresh_db.conn.in_transaction + fresh_db.commit() + assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + + +def test_execute_failed_write_inside_atomic_preserves_block(fresh_db): + # A caught failure inside an atomic() block must leave the block's + # transaction open so its other work still commits + fresh_db["t"].insert({"id": 1}, pk="id") + with fresh_db.atomic(): + fresh_db.execute("insert into t (id) values (2)") + with pytest.raises(sqlite3.IntegrityError): + fresh_db.execute("insert into t (id) values (1)") + assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + + +def test_query_returning_commits_after_iteration(tmpdir): + if sqlite3.sqlite_version_info < (3, 35, 0): + import pytest as _pytest + + _pytest.skip("RETURNING requires SQLite 3.35.0 or higher") + path = str(tmpdir / "test.db") + db = Database(path) + db["t"].insert({"id": 1}, pk="id") + rows = list(db.query("insert into t (id) values (2) returning id")) + assert rows == [{"id": 2}] + assert not db.conn.in_transaction + other = sqlite3.connect(path) + assert other.execute("select count(*) from t").fetchone()[0] == 2 + other.close() + db.close() + + +TRIGGER_SQL = """ +create trigger no_bad before insert on t +when new.v = 'bad' +begin + select raise(rollback, 'trigger says no'); +end +""" + + +def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db): + # RAISE(ROLLBACK) rolls back the whole transaction and destroys every + # savepoint - atomic()'s cleanup must not mask the IntegrityError + # with "cannot rollback - no transaction is active" + fresh_db.execute("create table t (id integer primary key, v text)") + fresh_db.execute(TRIGGER_SQL) + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + with fresh_db.atomic(): + fresh_db.execute("insert into t (v) values ('bad')") + assert not fresh_db.conn.in_transaction + + +def test_nested_atomic_preserves_error_from_transaction_destroying_trigger( + fresh_db, +): + # The nested savepoint branch previously raised + # "no such savepoint" from ROLLBACK TO SAVEPOINT + fresh_db.execute("create table t (id integer primary key, v text)") + fresh_db.execute(TRIGGER_SQL) + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + with fresh_db.atomic(): + with fresh_db.atomic(): + fresh_db.execute("insert into t (v) values ('bad')") + assert not fresh_db.conn.in_transaction + + +def test_atomic_preserves_error_from_insert_or_rollback(fresh_db): + fresh_db["t"].insert({"id": 1}, pk="id") + with pytest.raises(sqlite3.IntegrityError): + with fresh_db.atomic(): + fresh_db.execute("insert or rollback into t (id) values (1)") + assert not fresh_db.conn.in_transaction diff --git a/tests/test_cli.py b/tests/test_cli.py index d4801ac..a2135b0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,28 +1,41 @@ from sqlite_utils import cli, Database from sqlite_utils.db import Index, ForeignKey from click.testing import CliRunner -from unittest import mock +from pathlib import Path +import subprocess +import sqlite3 +import sys import json import os import pytest -from sqlite_utils.utils import sqlite3, find_spatialite import textwrap -from .utils import collapse_whitespace + +def write_json(file_path, data): + with open(file_path, "w") as fp: + json.dump(data, fp) -CREATE_TABLES = """ -create table Gosh (c1 text, c2 text, c3 text); -create table Gosh2 (c1 text, c2 text, c3 text); -""" +def _supports_pragma_function_list(): + db = Database(memory=True) + try: + db.execute("select * from pragma_function_list()") + return True + except Exception: + return False + finally: + db.close() -@pytest.fixture -def db_path(tmpdir): - path = str(tmpdir / "test.db") - db = sqlite3.connect(path) - db.executescript(CREATE_TABLES) - return path +def _has_compiled_ext(): + for ext in ["dylib", "so", "dll"]: + path = Path(__file__).parent / f"ext.{ext}" + if path.is_file(): + return True + return False + + +COMPILED_EXTENSION_PATH = str(Path(__file__).parent / "ext") @pytest.mark.parametrize( @@ -42,7 +55,7 @@ def test_help(options): def test_tables(db_path): - result = CliRunner().invoke(cli.cli, ["tables", db_path]) + result = CliRunner().invoke(cli.cli, ["tables", db_path], catch_exceptions=False) assert '[{"table": "Gosh"},\n {"table": "Gosh2"}]' == result.output.strip() @@ -51,8 +64,8 @@ def test_views(db_path): result = CliRunner().invoke(cli.cli, ["views", db_path, "--table", "--schema"]) assert ( "view schema\n" - "------ --------------------------------------------\n" - "hello CREATE VIEW hello AS select sqlite_version()" + "------ ----------------------------------------------\n" + 'hello CREATE VIEW "hello" AS select sqlite_version()' ) == result.output.strip() @@ -121,15 +134,15 @@ def test_tables_schema(db_path): assert ( '[{"table": "Gosh", "schema": "CREATE TABLE Gosh (c1 text, c2 text, c3 text)"},\n' ' {"table": "Gosh2", "schema": "CREATE TABLE Gosh2 (c1 text, c2 text, c3 text)"},\n' - ' {"table": "lots", "schema": "CREATE TABLE [lots] (\\n [id] INTEGER,\\n [age] INTEGER\\n)"}]' + ' {"table": "lots", "schema": "CREATE TABLE \\"lots\\" (\\n \\"id\\" INTEGER,\\n \\"age\\" INTEGER\\n)"}]' ) == result.output.strip() @pytest.mark.parametrize( - "fmt,expected", + "options,expected", [ ( - "simple", + ["--fmt", "simple"], ( "c1 c2 c3\n" "----- ----- ----------\n" @@ -140,7 +153,18 @@ def test_tables_schema(db_path): ), ), ( - "rst", + ["-t"], + ( + "c1 c2 c3\n" + "----- ----- ----------\n" + "verb0 noun0 adjective0\n" + "verb1 noun1 adjective1\n" + "verb2 noun2 adjective2\n" + "verb3 noun3 adjective3" + ), + ), + ( + ["--fmt", "rst"], ( "===== ===== ==========\n" "c1 c2 c3\n" @@ -154,7 +178,7 @@ def test_tables_schema(db_path): ), ], ) -def test_output_table(db_path, fmt, expected): +def test_output_table(db_path, options, expected): db = Database(db_path) with db.conn: db["rows"].insert_all( @@ -167,16 +191,60 @@ def test_output_table(db_path, fmt, expected): for i in range(4) ] ) - result = CliRunner().invoke(cli.cli, ["rows", db_path, "rows", "-t", "--fmt", fmt]) - assert 0 == result.exit_code + result = CliRunner().invoke(cli.cli, ["rows", db_path, "rows"] + options) + assert result.exit_code == 0 assert expected == result.output.strip() +@pytest.mark.parametrize( + "fmt_option", [["--fmt", "simple"], ["-t"], ["--fmt", "github"]] +) +def test_output_table_no_headers(db_path, fmt_option): + # --no-headers should omit the header row from --fmt/--table output too, not + # just from --csv/--tsv (#566). Previously the flag was silently ignored for + # tabulate formats and the column names were always printed. + db = Database(db_path) + with db.conn: + db["dogs"].insert_all( + [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Pancakes", "age": 2}, + ] + ) + sql = "select id, name, age from dogs order by id" + + with_headers = CliRunner().invoke(cli.cli, ["query", db_path, sql] + fmt_option) + without_headers = CliRunner().invoke( + cli.cli, ["query", db_path, sql] + fmt_option + ["--no-headers"] + ) + assert with_headers.exit_code == 0 + assert without_headers.exit_code == 0 + + # The column names appear when headers are shown, and must not appear at all + # once --no-headers is passed. + assert "name" in with_headers.output + for header in ("id", "name", "age"): + assert ( + header not in without_headers.output + ), f"header {header!r} leaked into --no-headers output" + # The data is still all present. + for value in ("Cleo", "Pancakes", "1", "2", "4"): + assert value in without_headers.output + + # The rows command shares the same code path. + rows_no_headers = CliRunner().invoke( + cli.cli, ["rows", db_path, "dogs"] + fmt_option + ["--no-headers"] + ) + assert rows_no_headers.exit_code == 0 + assert "name" not in rows_no_headers.output + assert "Cleo" in rows_no_headers.output + + def test_create_index(db_path): db = Database(db_path) assert [] == db["Gosh"].indexes result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "c1"]) - assert 0 == result.exit_code + assert result.exit_code == 0 assert [ Index( seq=0, name="idx_Gosh_c1", unique=0, origin="c", partial=0, columns=["c1"] @@ -186,7 +254,7 @@ def test_create_index(db_path): result = CliRunner().invoke( cli.cli, ["create-index", db_path, "Gosh", "c2", "--name", "blah"] ) - assert 0 == result.exit_code + assert result.exit_code == 0 assert [ Index(seq=0, name="blah", unique=0, origin="c", partial=0, columns=["c2"]), Index( @@ -203,7 +271,7 @@ def test_create_index(db_path): "--unique", ] result = CliRunner().invoke(cli.cli, create_index_unique_args) - assert 0 == result.exit_code + assert result.exit_code == 0 assert [ Index( seq=0, @@ -215,14 +283,42 @@ def test_create_index(db_path): ) ] == db["Gosh2"].indexes # Trying to create the same index should fail - assert 0 != CliRunner().invoke(cli.cli, create_index_unique_args).exit_code - # ... unless we use --if-not-exists - assert ( - 0 - == CliRunner() - .invoke(cli.cli, create_index_unique_args + ["--if-not-exists"]) - .exit_code + assert CliRunner().invoke(cli.cli, create_index_unique_args).exit_code != 0 + # ... unless we use --if-not-exists or --ignore + for option in ("--if-not-exists", "--ignore"): + assert ( + CliRunner().invoke(cli.cli, create_index_unique_args + [option]).exit_code + == 0 + ) + + +def test_drop_index(db_path): + db = Database(db_path) + db["Gosh"].create_index(["c1"]) + assert [index.name for index in db["Gosh"].indexes] == ["idx_Gosh_c1"] + result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) + assert result.exit_code == 0 + assert db["Gosh"].indexes == [] + + result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) + assert result.exit_code == 1 + assert "No index named idx_Gosh_c1" in result.output + + result = CliRunner().invoke( + cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1", "--ignore"] ) + assert result.exit_code == 0 + + +def test_create_index_analyze(db_path): + db = Database(db_path) + assert "sqlite_stat1" not in db.table_names() + assert [] == db["Gosh"].indexes + result = CliRunner().invoke( + cli.cli, ["create-index", db_path, "Gosh", "c1", "--analyze"] + ) + assert result.exit_code == 0 + assert "sqlite_stat1" in db.table_names() def test_create_index_desc(db_path): @@ -232,43 +328,62 @@ def test_create_index_desc(db_path): assert result.exit_code == 0 assert ( db.execute("select sql from sqlite_master where type='index'").fetchone()[0] - == "CREATE INDEX [idx_Gosh_c1]\n ON [Gosh] ([c1] desc)" + == 'CREATE INDEX "idx_Gosh_c1"\n ON "Gosh" ("c1" desc)' ) @pytest.mark.parametrize( "col_name,col_type,expected_schema", ( - ("text", "TEXT", "CREATE TABLE [dogs] ( [name] TEXT , [text] TEXT)"), + ("text", "TEXT", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "text" TEXT)'), + ("text", "str", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "text" TEXT)'), + ("text", "STR", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "text" TEXT)'), ( "integer", "INTEGER", - "CREATE TABLE [dogs] ( [name] TEXT , [integer] INTEGER)", + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "integer" INTEGER)', ), - ("float", "FLOAT", "CREATE TABLE [dogs] ( [name] TEXT , [float] FLOAT)"), - ("blob", "blob", "CREATE TABLE [dogs] ( [name] TEXT , [blob] BLOB)"), - ("default", None, "CREATE TABLE [dogs] ( [name] TEXT , [default] TEXT)"), + ( + "integer", + "int", + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "integer" INTEGER)', + ), + ("float", "FLOAT", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "float" REAL)'), + ("blob", "blob", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), + ("blob", "BLOB", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), + ("blob", "bytes", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), + ("blob", "BYTES", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), + ("default", None, 'CREATE TABLE "dogs" (\n "name" TEXT\n, "default" TEXT)'), ), ) def test_add_column(db_path, col_name, col_type, expected_schema): db = Database(db_path) db.create_table("dogs", {"name": str}) - assert "CREATE TABLE [dogs] ( [name] TEXT )" == collapse_whitespace( - db["dogs"].schema - ) + assert db["dogs"].schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' args = ["add-column", db_path, "dogs", col_name] if col_type is not None: args.append(col_type) - assert 0 == CliRunner().invoke(cli.cli, args).exit_code - assert expected_schema == collapse_whitespace(db["dogs"].schema) + assert CliRunner().invoke(cli.cli, args).exit_code == 0 + assert db["dogs"].schema == expected_schema + + +@pytest.mark.parametrize("ignore", (True, False)) +def test_add_column_ignore(db_path, ignore): + db = Database(db_path) + db.create_table("dogs", {"name": str}) + args = ["add-column", db_path, "dogs", "name"] + (["--ignore"] if ignore else []) + result = CliRunner().invoke(cli.cli, args) + if ignore: + assert result.exit_code == 0 + else: + assert result.exit_code == 1 + assert result.output == "Error: duplicate column name: name\n" def test_add_column_not_null_default(db_path): db = Database(db_path) db.create_table("dogs", {"name": str}) - assert "CREATE TABLE [dogs] ( [name] TEXT )" == collapse_whitespace( - db["dogs"].schema - ) + assert db["dogs"].schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' args = [ "add-column", db_path, @@ -277,10 +392,11 @@ def test_add_column_not_null_default(db_path): "--not-null-default", "dogs'dawg", ] - assert 0 == CliRunner().invoke(cli.cli, args).exit_code - assert ( - "CREATE TABLE [dogs] ( [name] TEXT , [nickname] TEXT NOT NULL DEFAULT 'dogs''dawg')" - == collapse_whitespace(db["dogs"].schema) + assert CliRunner().invoke(cli.cli, args).exit_code == 0 + assert db["dogs"].schema == ( + 'CREATE TABLE "dogs" (\n' + ' "name" TEXT\n' + ", \"nickname\" TEXT NOT NULL DEFAULT 'dogs''dawg')" ) @@ -307,7 +423,7 @@ def test_add_foreign_key(db_path, args, assert_message): ] ) assert ( - 0 == CliRunner().invoke(cli.cli, ["add-foreign-key", db_path] + args).exit_code + CliRunner().invoke(cli.cli, ["add-foreign-key", db_path] + args).exit_code == 0 ), assert_message assert [ ForeignKey( @@ -319,7 +435,7 @@ def test_add_foreign_key(db_path, args, assert_message): result = CliRunner().invoke( cli.cli, ["add-foreign-key", db_path, "books", "author_id", "authors", "id"] ) - assert 0 != result.exit_code + assert result.exit_code != 0 assert ( "Error: Foreign key already exists for author_id => authors.id" == result.output.strip() @@ -330,13 +446,13 @@ def test_add_foreign_key(db_path, args, assert_message): cli.cli, ["add-foreign-key", db_path, "books", "author_id", "authors", "id", "--ignore"], ) - assert 0 == result.exit_code + assert result.exit_code == 0 # Error if we try against an invalid column result = CliRunner().invoke( cli.cli, ["add-foreign-key", db_path, "books", "author_id", "authors", "bad"] ) - assert 0 != result.exit_code + assert result.exit_code != 0 assert "Error: No such column: authors.bad" == result.output.strip() @@ -348,10 +464,12 @@ def test_add_column_foreign_key(db_path): result = CliRunner().invoke( cli.cli, ["add-column", db_path, "books", "author_id", "--fk", "authors"] ) - assert 0 == result.exit_code, result.output - assert ( - "CREATE TABLE [books] ( [title] TEXT , [author_id] INTEGER, FOREIGN KEY([author_id]) REFERENCES [authors]([id]) )" - == collapse_whitespace(db["books"].schema) + assert result.exit_code == 0, result.output + assert db["books"].schema == ( + 'CREATE TABLE "books" (\n' + ' "title" TEXT,\n' + ' "author_id" INTEGER REFERENCES "authors"("id")\n' + ")" ) # Try it again with a custom --fk-col result = CliRunner().invoke( @@ -367,18 +485,19 @@ def test_add_column_foreign_key(db_path): "name", ], ) - assert 0 == result.exit_code, result.output - assert ( - "CREATE TABLE [books] ( [title] TEXT , [author_id] INTEGER, [author_name_ref] TEXT, " - "FOREIGN KEY([author_id]) REFERENCES [authors]([id]), " - "FOREIGN KEY([author_name_ref]) REFERENCES [authors]([name]) )" - == collapse_whitespace(db["books"].schema) + assert result.exit_code == 0, result.output + assert db["books"].schema == ( + 'CREATE TABLE "books" (\n' + ' "title" TEXT,\n' + ' "author_id" INTEGER REFERENCES "authors"("id"),\n' + ' "author_name_ref" TEXT REFERENCES "authors"("name")\n' + ")" ) # Throw an error if the --fk table does not exist result = CliRunner().invoke( cli.cli, ["add-column", db_path, "books", "author_id", "--fk", "bobcats"] ) - assert 0 != result.exit_code + assert result.exit_code != 0 assert "table 'bobcats' does not exist" in str(result.exception) @@ -402,7 +521,7 @@ def test_index_foreign_keys(db_path): db = Database(db_path) assert [] == db["books"].indexes result = CliRunner().invoke(cli.cli, ["index-foreign-keys", db_path]) - assert 0 == result.exit_code + assert result.exit_code == 0 assert [["author_id"], ["author_name_ref"]] == [ i.columns for i in db["books"].indexes ] @@ -414,7 +533,7 @@ def test_enable_fts(db_path): result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) - assert 0 == result.exit_code + assert result.exit_code == 0 assert "Gosh_fts" == db["Gosh"].detect_fts() # Table names with restricted chars are handled correctly. @@ -433,19 +552,44 @@ def test_enable_fts(db_path): "porter", ], ) - assert 0 == result.exit_code + assert result.exit_code == 0 assert "http://example.com_fts" == db["http://example.com"].detect_fts() # Check tokenize was set to porter assert ( - "CREATE VIRTUAL TABLE [http://example.com_fts] USING FTS4 (\n" - " [c1],\n" + 'CREATE VIRTUAL TABLE "http://example.com_fts" USING FTS4 (\n' + ' "c1",\n' " tokenize='porter',\n" - " content=[http://example.com]" + ' content="http://example.com"' "\n)" ) == db["http://example.com_fts"].schema db["http://example.com"].drop() +def test_enable_fts_replace(db_path): + db = Database(db_path) + assert db["Gosh"].detect_fts() is None + result = CliRunner().invoke( + cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] + ) + assert result.exit_code == 0 + assert "Gosh_fts" == db["Gosh"].detect_fts() + assert db["Gosh_fts"].columns_dict == {"c1": str} + + # This should throw an error + result2 = CliRunner().invoke( + cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] + ) + assert result2.exit_code == 1 + assert result2.output == 'Error: table "Gosh_fts" already exists\n' + + # This should work + result3 = CliRunner().invoke( + cli.cli, ["enable-fts", db_path, "Gosh", "c2", "--fts4", "--replace"] + ) + assert result3.exit_code == 0 + assert db["Gosh_fts"].columns_dict == {"c2": str} + + def test_enable_fts_with_triggers(db_path): Database(db_path)["Gosh"].insert_all([{"c1": "baz"}]) exit_code = ( @@ -456,7 +600,7 @@ def test_enable_fts_with_triggers(db_path): ) .exit_code ) - assert 0 == exit_code + assert exit_code == 0 def search(q): return ( @@ -477,7 +621,7 @@ def test_populate_fts(db_path): .invoke(cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"]) .exit_code ) - assert 0 == exit_code + assert exit_code == 0 def search(q): return ( @@ -492,7 +636,7 @@ def test_populate_fts(db_path): exit_code = ( CliRunner().invoke(cli.cli, ["populate-fts", db_path, "Gosh", "c1"]).exit_code ) - assert 0 == exit_code + assert exit_code == 0 assert [("martha",)] == search("martha") @@ -510,13 +654,13 @@ def test_disable_fts(db_path): "Gosh_fts_docsize", } == set(db.table_names()) exit_code = CliRunner().invoke(cli.cli, ["disable-fts", db_path, "Gosh"]).exit_code - assert 0 == exit_code + assert exit_code == 0 assert {"Gosh", "Gosh2"} == set(db.table_names()) def test_vacuum(db_path): result = CliRunner().invoke(cli.cli, ["vacuum", db_path]) - assert 0 == result.exit_code + assert result.exit_code == 0 def test_dump(db_path): @@ -545,7 +689,7 @@ def test_optimize(db_path, tables): db["Gosh2"].enable_fts(["c1", "c2", "c3"], fts_version="FTS5") size_before_optimize = os.stat(db_path).st_size result = CliRunner().invoke(cli.cli, ["optimize", db_path] + tables) - assert 0 == result.exit_code + assert result.exit_code == 0 size_after_optimize = os.stat(db_path).st_size # Weirdest thing: tests started failing because size after # ended up larger than size before in some cases. I think @@ -553,11 +697,10 @@ def test_optimize(db_path, tables): assert size_after_optimize <= (size_before_optimize + 10000) # Soundness check that --no-vacuum doesn't throw errors: result = CliRunner().invoke(cli.cli, ["optimize", "--no-vacuum", db_path]) - assert 0 == result.exit_code + assert result.exit_code == 0 -@pytest.mark.parametrize("tables", ([], ["fts4_table"], ["fts5_table"])) -def test_rebuild_fts(db_path, tables): +def test_rebuild_fts_fixes_docsize_error(db_path): db = Database(db_path, recursive_triggers=False) records = [ { @@ -568,363 +711,22 @@ def test_rebuild_fts(db_path, tables): for i in range(10000) ] with db.conn: - db["fts4_table"].insert_all(records, pk="c1") db["fts5_table"].insert_all(records, pk="c1") - db["fts4_table"].enable_fts( - ["c1", "c2", "c3"], fts_version="FTS4", create_triggers=True - ) db["fts5_table"].enable_fts( ["c1", "c2", "c3"], fts_version="FTS5", create_triggers=True ) # Search should work - assert list(db["fts4_table"].search("verb1")) assert list(db["fts5_table"].search("verb1")) - # Deleting _fts_segments to break FTS4 - with db.conn: - db["fts4_table_fts_segments"].delete_where() - # Now this should error: - with pytest.raises(sqlite3.DatabaseError): - list(db["fts4_table"].search("verb1")) # Replicate docsize error from this issue for FTS5 # https://github.com/simonw/sqlite-utils/issues/149 assert db["fts5_table_fts_docsize"].count == 10000 db["fts5_table"].insert_all(records, replace=True) assert db["fts5_table"].count == 10000 assert db["fts5_table_fts_docsize"].count == 20000 - # Running rebuild-fts should fix both issues - print(["rebuild-fts", db_path] + tables) - result = CliRunner().invoke(cli.cli, ["rebuild-fts", db_path] + tables) - assert 0 == result.exit_code - fixed_tables = tables or ["fts4_table", "fts5_table"] - if "fts4_table" in fixed_tables: - assert list(db["fts4_table"].search("verb1")) - else: - with pytest.raises(sqlite3.DatabaseError): - list(db["fts4_table"].search("verb1")) - if "fts5_table" in fixed_tables: - assert db["fts5_table_fts_docsize"].count == 10000 - else: - assert db["fts5_table_fts_docsize"].count == 20000 - - -def test_insert_simple(tmpdir): - json_path = str(tmpdir / "dog.json") - db_path = str(tmpdir / "dogs.db") - open(json_path, "w").write(json.dumps({"name": "Cleo", "age": 4})) - result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path]) - assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == list( - Database(db_path).query("select * from dogs") - ) - db = Database(db_path) - assert ["dogs"] == db.table_names() - assert [] == db["dogs"].indexes - - -def test_insert_from_stdin(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "dogs", "-"], - input=json.dumps({"name": "Cleo", "age": 4}), - ) - assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == list( - Database(db_path).query("select * from dogs") - ) - - -def test_insert_invalid_json_error(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "dogs", "-"], - input="name,age\nCleo,4", - ) - assert result.exit_code == 1 - assert ( - result.output - == "Error: Invalid JSON - use --csv for CSV or --tsv for TSV files\n" - ) - - -def test_insert_json_flatten(tmpdir): - db_path = str(tmpdir / "flat.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "items", "-", "--flatten"], - input=json.dumps({"nested": {"data": 4}}), - ) + # Running rebuild-fts should fix this + result = CliRunner().invoke(cli.cli, ["rebuild-fts", db_path, "fts5_table"]) assert result.exit_code == 0 - assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}] - - -def test_insert_json_flatten_nl(tmpdir): - db_path = str(tmpdir / "flat.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "items", "-", "--flatten", "--nl"], - input="\n".join( - json.dumps(item) - for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}] - ), - ) - assert result.exit_code == 0 - assert list(Database(db_path).query("select * from items")) == [ - {"nested_data": 4, "nested_other": None}, - {"nested_data": None, "nested_other": 3}, - ] - - -def test_insert_with_primary_key(db_path, tmpdir): - json_path = str(tmpdir / "dog.json") - open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] - ) - assert 0 == result.exit_code - assert [{"id": 1, "age": 4, "name": "Cleo"}] == list( - Database(db_path).query("select * from dogs") - ) - db = Database(db_path) - assert ["id"] == db["dogs"].pks - - -def test_insert_multiple_with_primary_key(db_path, tmpdir): - json_path = str(tmpdir / "dogs.json") - dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] - open(json_path, "w").write(json.dumps(dogs)) - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] - ) - assert 0 == result.exit_code - db = Database(db_path) - assert dogs == list(db.query("select * from dogs order by id")) - assert ["id"] == db["dogs"].pks - - -def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): - json_path = str(tmpdir / "dogs.json") - dogs = [ - {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} - for i in range(1, 21) - ] - open(json_path, "w").write(json.dumps(dogs)) - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--pk", "breed"] - ) - assert 0 == result.exit_code - db = Database(db_path) - assert dogs == list(db.query("select * from dogs order by breed, id")) - assert {"breed", "id"} == set(db["dogs"].pks) - assert ( - "CREATE TABLE [dogs] (\n" - " [breed] TEXT,\n" - " [id] INTEGER,\n" - " [name] TEXT,\n" - " [age] INTEGER,\n" - " PRIMARY KEY ([id], [breed])\n" - ")" - ) == db["dogs"].schema - - -def test_insert_not_null_default(db_path, tmpdir): - json_path = str(tmpdir / "dogs.json") - dogs = [ - {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10} - for i in range(1, 21) - ] - open(json_path, "w").write(json.dumps(dogs)) - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "dogs", json_path, "--pk", "id"] - + ["--not-null", "name", "--not-null", "age"] - + ["--default", "score", "5", "--default", "age", "1"], - ) - assert 0 == result.exit_code - db = Database(db_path) - assert ( - "CREATE TABLE [dogs] (\n" - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT NOT NULL,\n" - " [age] INTEGER NOT NULL DEFAULT '1',\n" - " [score] INTEGER DEFAULT '5'\n)" - ) == db["dogs"].schema - - -def test_insert_binary_base64(db_path): - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "files", "-"], - input=r'{"content": {"$base64": true, "encoded": "aGVsbG8="}}', - ) - assert 0 == result.exit_code, result.output - db = Database(db_path) - actual = list(db.query("select content from files")) - assert actual == [{"content": b"hello"}] - - -def test_insert_newline_delimited(db_path): - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl"], - input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', - ) - assert 0 == result.exit_code, result.output - db = Database(db_path) - assert [ - {"foo": "bar", "n": 1}, - {"foo": "baz", "n": 2}, - ] == list(db.query("select foo, n from from_json_nl")) - - -def test_insert_ignore(db_path, tmpdir): - db = Database(db_path) - db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") - json_path = str(tmpdir / "dogs.json") - open(json_path, "w").write(json.dumps([{"id": 1, "name": "Bailey"}])) - # Should raise error without --ignore - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] - ) - assert 0 != result.exit_code, result.output - # If we use --ignore it should run OK - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--ignore"] - ) - assert 0 == result.exit_code, result.output - # ... but it should actually have no effect - assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs")) - - -@pytest.mark.parametrize( - "content,options", - [ - ("foo\tbar\tbaz\n1\t2\tcat,dog", ["--tsv"]), - ('foo,bar,baz\n1,2,"cat,dog"', ["--csv"]), - ('foo;bar;baz\n1;2;"cat,dog"', ["--csv", "--delimiter", ";"]), - # --delimiter implies --csv: - ('foo;bar;baz\n1;2;"cat,dog"', ["--delimiter", ";"]), - ("foo,bar,baz\n1,2,|cat,dog|", ["--csv", "--quotechar", "|"]), - ("foo,bar,baz\n1,2,|cat,dog|", ["--quotechar", "|"]), - ], -) -def test_insert_csv_tsv(content, options, db_path, tmpdir): - db = Database(db_path) - file_path = str(tmpdir / "insert.csv-tsv") - open(file_path, "w").write(content) - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "data", file_path] + options, - catch_exceptions=False, - ) - assert 0 == result.exit_code - assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows) - - -@pytest.mark.parametrize( - "options", - ( - ["--tsv", "--nl"], - ["--tsv", "--csv"], - ["--csv", "--nl"], - ["--csv", "--nl", "--tsv"], - ), -) -def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir): - file_path = str(tmpdir / "insert.csv-tsv") - open(file_path, "w").write("foo") - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "data", file_path] + options - ) - assert 0 != result.exit_code - assert "Error: Use just one of --nl, --csv or --tsv" == result.output.strip() - - -def test_insert_replace(db_path, tmpdir): - test_insert_multiple_with_primary_key(db_path, tmpdir) - json_path = str(tmpdir / "insert-replace.json") - db = Database(db_path) - assert 20 == db["dogs"].count - insert_replace_dogs = [ - {"id": 1, "name": "Insert replaced 1", "age": 4}, - {"id": 2, "name": "Insert replaced 2", "age": 4}, - {"id": 21, "name": "Fresh insert 21", "age": 6}, - ] - open(json_path, "w").write(json.dumps(insert_replace_dogs)) - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"] - ) - assert 0 == result.exit_code, result.output - assert 21 == db["dogs"].count - assert ( - list(db.query("select * from dogs where id in (1, 2, 21) order by id")) - == insert_replace_dogs - ) - - -def test_insert_truncate(db_path): - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl", "--batch-size=1"], - input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', - ) - assert 0 == result.exit_code, result.output - db = Database(db_path) - assert [ - {"foo": "bar", "n": 1}, - {"foo": "baz", "n": 2}, - ] == list(db.query("select foo, n from from_json_nl")) - # Truncate and insert new rows - result = CliRunner().invoke( - cli.cli, - [ - "insert", - db_path, - "from_json_nl", - "-", - "--nl", - "--truncate", - "--batch-size=1", - ], - input='{"foo": "bam", "n": 3}\n{"foo": "bat", "n": 4}', - ) - assert 0 == result.exit_code, result.output - assert [ - {"foo": "bam", "n": 3}, - {"foo": "bat", "n": 4}, - ] == list(db.query("select foo, n from from_json_nl")) - - -def test_insert_alter(db_path, tmpdir): - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl"], - input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', - ) - assert 0 == result.exit_code, result.output - # Should get an error with incorrect shaped additional data - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl"], - input='{"foo": "bar", "baz": 5}', - ) - assert 0 != result.exit_code, result.output - # If we run it again with --alter it should work correctly - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl", "--alter"], - input='{"foo": "bar", "baz": 5}', - ) - assert 0 == result.exit_code, result.output - # Soundness check the database itself - db = Database(db_path) - assert {"foo": str, "n": int, "baz": int} == db["from_json_nl"].columns_dict - assert [ - {"foo": "bar", "n": 1, "baz": None}, - {"foo": "baz", "n": 2, "baz": None}, - {"foo": "bar", "baz": 5, "n": None}, - ] == list(db.query("select foo, n, baz from from_json_nl")) + assert db["fts5_table_fts_docsize"].count == 10000 @pytest.mark.parametrize( @@ -946,7 +748,7 @@ def test_query_csv(db_path, format, expected): result = CliRunner().invoke( cli.cli, [db_path, "select id, name, age from dogs", format] ) - assert 0 == result.exit_code + assert result.exit_code == 0 assert result.output.replace("\r", "") == expected # Test the no-headers option: result = CliRunner().invoke( @@ -979,6 +781,11 @@ _one_query = "select id, name, age from dogs where id = 1" (_one_query, ["--nl"], '{"id": 1, "name": "Cleo", "age": 4}'), (_one_query, ["--arrays"], '[[1, "Cleo", 4]]'), (_one_query, ["--arrays", "--nl"], '[1, "Cleo", 4]'), + ( + "select id, dog(age) from dogs", + ["--functions", "def dog(i):\n return i * 7"], + '[{"id": 1, "dog(age)": 28},\n {"id": 2, "dog(age)": 14}]', + ), ], ) def test_query_json(db_path, sql, args, expected): @@ -994,6 +801,185 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() +def test_query_sql_from_stdin(db_path): + # https://github.com/simonw/sqlite-utils/issues/765 + db = Database(db_path) + with db.conn: + db["dogs"].insert_all( + [ + {"id": 1, "age": 4, "name": "Cleo"}, + {"id": 2, "age": 2, "name": "Pancakes"}, + ] + ) + result = CliRunner().invoke( + cli.cli, + ["query", db_path, "-"], + input="select name from dogs order by name", + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output) == [{"name": "Cleo"}, {"name": "Pancakes"}] + + +def test_query_json_empty(db_path): + result = CliRunner().invoke( + cli.cli, + [db_path, "select * from sqlite_master where 0"], + ) + assert result.output.strip() == "[]" + + +def test_query_json_duplicate_columns_are_deduped(db_path): + # https://github.com/simonw/sqlite-utils/issues/624 + result = CliRunner().invoke( + cli.cli, + [db_path, "select 1 as id, 2 as id, 'x' as value, 'y' as value"], + ) + assert result.output.strip() == ( + '[{"id": 1, "id_2": 2, "value": "x", "value_2": "y"}]' + ) + + +def test_query_csv_duplicate_columns_are_preserved(db_path): + # CSV output should keep the duplicate headers, not rename them + result = CliRunner().invoke( + cli.cli, + [db_path, "select 1 as id, 2 as id", "--csv"], + ) + assert result.output.replace("\r", "").strip() == "id,id\n1,2" + + +def test_query_invalid_function(db_path): + result = CliRunner().invoke( + cli.cli, [db_path, "select bad()", "--functions", "def invalid_python"] + ) + assert result.exit_code == 1 + assert result.output.startswith("Error: Error in functions definition:") + + +TEST_FUNCTIONS = """ +def zero(): + return 0 + +def one(a): + return a + +def _two(a, b): + return a + b + +def two(a, b): + return _two(a, b) +""" + + +def test_query_complex_function(db_path): + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select zero(), one(1), two(1, 2)", + "--functions", + TEST_FUNCTIONS, + ], + ) + assert result.exit_code == 0 + assert json.loads(result.output.strip()) == [ + {"zero()": 0, "one(1)": 1, "two(1, 2)": 3} + ] + + +@pytest.mark.skipif( + not _supports_pragma_function_list(), + reason="Needs SQLite version that supports pragma_function_list()", +) +def test_hidden_functions_are_hidden(db_path): + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select name from pragma_function_list()", + "--functions", + TEST_FUNCTIONS, + ], + ) + assert result.exit_code == 0 + functions = {r["name"] for r in json.loads(result.output.strip())} + assert "zero" in functions + assert "one" in functions + assert "two" in functions + assert "_two" not in functions + + +def test_query_functions_from_file(db_path, tmp_path): + # Create a temporary file with function definitions + functions_file = tmp_path / "my_functions.py" + functions_file.write_text(TEST_FUNCTIONS) + + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select zero(), one(1), two(1, 2)", + "--functions", + str(functions_file), + ], + ) + assert result.exit_code == 0 + assert json.loads(result.output.strip()) == [ + {"zero()": 0, "one(1)": 1, "two(1, 2)": 3} + ] + + +def test_query_functions_file_not_found(db_path): + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select zero()", + "--functions", + "nonexistent.py", + ], + ) + assert result.exit_code == 1 + assert "File not found: nonexistent.py" in result.output + + +def test_query_functions_multiple_invocations(db_path): + # Test using --functions multiple times + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select triple(2), quadruple(2)", + "--functions", + "def triple(x):\n return x * 3", + "--functions", + "def quadruple(x):\n return x * 4", + ], + ) + assert result.exit_code == 0 + assert json.loads(result.output.strip()) == [{"triple(2)": 6, "quadruple(2)": 8}] + + +def test_query_functions_file_and_inline(db_path, tmp_path): + # Test combining file and inline code + functions_file = tmp_path / "file_funcs.py" + functions_file.write_text("def triple(x):\n return x * 3") + + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select triple(2), quadruple(2)", + "--functions", + str(functions_file), + "--functions", + "def quadruple(x):\n return x * 4", + ], + ) + assert result.exit_code == 0 + assert json.loads(result.output.strip()) == [{"triple(2)": 6, "quadruple(2)": 8}] + + LOREM_IPSUM_COMPRESSED = ( b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8e" b"f\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J" @@ -1083,12 +1069,9 @@ def test_query_json_with_json_cols(db_path): result = CliRunner().invoke( cli.cli, [db_path, "select id, name, friends from dogs"] ) - assert ( - r""" + assert r""" [{"id": 1, "name": "Cleo", "friends": "[{\"name\": \"Pancakes\"}, {\"name\": \"Bailey\"}]"}] - """.strip() - == result.output.strip() - ) + """.strip() == result.output.strip() # With --json-cols: result = CliRunner().invoke( cli.cli, [db_path, "select id, name, friends from dogs", "--json-cols"] @@ -1102,9 +1085,37 @@ def test_query_json_with_json_cols(db_path): assert expected == result_rows.output.strip() +def test_query_json_unicode_not_escaped_by_default(db_path): + db = Database(db_path) + with db.conn: + db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text"]) + assert result.exit_code == 0 + assert result.output.strip() == '[{"id": 1, "text": "Japanese 日本語"}]' + # Same for --nl + result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text", "--nl"]) + assert result.exit_code == 0 + assert result.output.strip() == '{"id": 1, "text": "Japanese 日本語"}' + + +@pytest.mark.parametrize("command", ["query", "rows"]) +def test_query_json_ascii_option(db_path, command): + db = Database(db_path) + with db.conn: + db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + if command == "query": + args = [db_path, "select id, text from text", "--ascii"] + else: + args = ["rows", db_path, "text", "--ascii"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0 + expected = '[{"id": 1, "text": "Japanese ' + "\\u65e5\\u672c\\u8a9e" + '"}]' + assert result.output.strip() == expected + + @pytest.mark.parametrize( "content,is_binary", - [(b"\x00\x0Fbinary", True), ("this is text", False), (1, False), (1.5, False)], + [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], ) def test_query_raw(db_path, content, is_binary): Database(db_path)["files"].insert({"content": content}) @@ -1117,32 +1128,19 @@ def test_query_raw(db_path, content, is_binary): assert result.output == str(content) -@pytest.mark.skipif(not find_spatialite(), reason="Could not find SpatiaLite extension") -@pytest.mark.skipif( - not hasattr(sqlite3.Connection, "enable_load_extension"), - reason="sqlite3.Connection missing enable_load_extension", +@pytest.mark.parametrize( + "content,is_binary", + [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], ) -@pytest.mark.parametrize("use_spatialite_shortcut", [True, False]) -def test_query_load_extension(use_spatialite_shortcut): - # Without --load-extension: - result = CliRunner().invoke(cli.cli, [":memory:", "select spatialite_version()"]) - assert result.exit_code == 1 - assert "no such function: spatialite_version" in result.output - # With --load-extension: - if use_spatialite_shortcut: - load_extension = "spatialite" - else: - load_extension = find_spatialite() +def test_query_raw_lines(db_path, content, is_binary): + Database(db_path)["files"].insert_all({"content": content} for _ in range(3)) result = CliRunner().invoke( - cli.cli, - [ - ":memory:", - "select spatialite_version()", - "--load-extension={}".format(load_extension), - ], + cli.cli, [db_path, "select content from files", "--raw-lines"] ) - assert result.exit_code == 0, result.stdout - assert ["spatialite_version()"] == list(json.loads(result.output)[0].keys()) + if is_binary: + assert result.stdout_bytes == b"\n".join(content for _ in range(3)) + b"\n" + else: + assert result.output == "\n".join(str(content) for _ in range(3)) + "\n" def test_query_memory_does_not_create_file(tmpdir): @@ -1176,6 +1174,37 @@ def test_query_memory_does_not_create_file(tmpdir): ["--nl", "-c", "age", "-c", "name"], '{"age": 4, "name": "Cleo"}\n{"age": 2, "name": "Pancakes"}', ), + # --limit and --offset + ( + ["-c", "name", "--limit", "1"], + '[{"name": "Cleo"}]', + ), + ( + ["-c", "name", "--limit", "1", "--offset", "1"], + '[{"name": "Pancakes"}]', + ), + # --where + ( + ["-c", "name", "--where", "id = 1"], + '[{"name": "Cleo"}]', + ), + ( + ["-c", "name", "--where", "id = :id", "-p", "id", "1"], + '[{"name": "Cleo"}]', + ), + ( + ["-c", "name", "--where", "id = :id", "--param", "id", "1"], + '[{"name": "Cleo"}]', + ), + # --order + ( + ["-c", "id", "--order", "id desc", "--limit", "1"], + '[{"id": 2}]', + ), + ( + ["-c", "id", "--order", "id", "--limit", "1"], + '[{"id": 1}]', + ), ], ) def test_rows(db_path, args, expected): @@ -1199,32 +1228,80 @@ def test_upsert(db_path, tmpdir): {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Nixie", "age": 4}, ] - open(json_path, "w").write(json.dumps(insert_dogs)) + write_json(json_path, insert_dogs) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"], catch_exceptions=False, ) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output assert 2 == db["dogs"].count # Now run the upsert to update just their ages upsert_dogs = [ {"id": 1, "age": 5}, {"id": 2, "age": 5}, ] - open(json_path, "w").write(json.dumps(upsert_dogs)) + write_json(json_path, upsert_dogs) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"], catch_exceptions=False, ) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output assert list(db.query("select * from dogs order by id")) == [ {"id": 1, "name": "Cleo", "age": 5}, {"id": 2, "name": "Nixie", "age": 5}, ] +def test_upsert_pk_inferred_from_existing_table(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + db = Database(db_path) + insert_dogs = [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Nixie", "age": 4}, + ] + write_json(json_path, insert_dogs) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", json_path, "--pk", "id"], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + + write_json( + json_path, + [ + {"id": 1, "age": 5}, + {"id": 2, "age": 5}, + ], + ) + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "dogs", json_path], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + assert list(db.query("select * from dogs order by id")) == [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Nixie", "age": 5}, + ] + + +def test_upsert_analyze(db_path, tmpdir): + db = Database(db_path) + db["rows"].insert({"id": 1, "foo": "x", "n": 3}, pk="id") + db["rows"].create_index(["n"]) + assert "sqlite_stat1" not in db.table_names() + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "rows", "-", "--nl", "--analyze", "--pk", "id"], + input='{"id": 2, "foo": "bar", "n": 1}', + ) + assert result.exit_code == 0, result.output + assert "sqlite_stat1" in db.table_names() + + def test_upsert_flatten(tmpdir): db_path = str(tmpdir / "flat.db") db = Database(db_path) @@ -1244,31 +1321,28 @@ def test_upsert_alter(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") db = Database(db_path) insert_dogs = [{"id": 1, "name": "Cleo"}] - open(json_path, "w").write(json.dumps(insert_dogs)) + write_json(json_path, insert_dogs) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] ) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output # Should fail with error code if no --alter upsert_dogs = [{"id": 1, "age": 5}] - open(json_path, "w").write(json.dumps(upsert_dogs)) + write_json(json_path, upsert_dogs) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] ) - assert 1 == result.exit_code - assert ( - "Error: no such column: age\n\n" - "sql = UPDATE [dogs] SET [age] = ? WHERE [id] = ?\n" - "parameters = [5, 1]" - ) == result.output.strip() + assert result.exit_code == 1 + # Could be one of two errors depending on SQLite version + assert ("Try using --alter to add additional columns") in result.output.strip() # Should succeed with --alter result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"] ) - assert 0 == result.exit_code - assert [ + assert result.exit_code == 0 + assert list(db.query("select * from dogs order by id")) == [ {"id": 1, "name": "Cleo", "age": 5}, - ] == list(db.query("select * from dogs order by id")) + ] @pytest.mark.parametrize( @@ -1282,7 +1356,7 @@ def test_upsert_alter(db_path, tmpdir): "age", "integer", ], - ("CREATE TABLE [t] (\n [name] TEXT,\n [age] INTEGER\n)"), + ('CREATE TABLE "t" (\n "name" TEXT,\n "age" INTEGER\n)'), ), # All types: ( @@ -1301,24 +1375,32 @@ def test_upsert_alter(db_path, tmpdir): "id", ], ( - "CREATE TABLE [t] (\n" - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT,\n" - " [age] INTEGER,\n" - " [weight] FLOAT,\n" - " [thumbnail] BLOB\n" + 'CREATE TABLE "t" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT,\n' + ' "age" INTEGER,\n' + ' "weight" FLOAT,\n' + ' "thumbnail" BLOB\n' ")" ), ), # Not null: ( ["name", "text", "--not-null", "name"], - ("CREATE TABLE [t] (\n" " [name] TEXT NOT NULL\n" ")"), + ('CREATE TABLE "t" (\n' ' "name" TEXT NOT NULL\n' ")"), ), # Default: ( ["age", "integer", "--default", "age", "3"], - ("CREATE TABLE [t] (\n" " [age] INTEGER DEFAULT '3'\n" ")"), + ('CREATE TABLE "t" (\n' " \"age\" INTEGER DEFAULT '3'\n" ")"), + ), + # Compound primary key + ( + ["category", "text", "name", "text", "--pk", "category", "--pk", "name"], + ( + 'CREATE TABLE "t" (\n "category" TEXT,\n "name" TEXT,\n' + ' PRIMARY KEY ("category", "name")\n)' + ), ), ], ) @@ -1335,7 +1417,7 @@ def test_create_table(args, schema): + args, catch_exceptions=False, ) - assert 0 == result.exit_code + assert result.exit_code == 0 db = Database("test.db") assert schema == db["t"].schema @@ -1365,19 +1447,19 @@ def test_create_table_foreign_key(): result = runner.invoke( cli.cli, ["create-table", "books.db"] + args, catch_exceptions=False ) - assert 0 == result.exit_code + assert result.exit_code == 0 db = Database("books.db") assert ( - "CREATE TABLE [authors] (\n" - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT\n" + 'CREATE TABLE "authors" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT\n' ")" ) == db["authors"].schema assert ( - "CREATE TABLE [books] (\n" - " [id] INTEGER PRIMARY KEY,\n" - " [title] TEXT,\n" - " [author_id] INTEGER REFERENCES [authors]([id])\n" + 'CREATE TABLE "books" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "title" TEXT,\n' + ' "author_id" INTEGER REFERENCES "authors"("id")\n' ")" ) == db["books"].schema @@ -1390,7 +1472,7 @@ def test_create_table_error_if_table_exists(): result = runner.invoke( cli.cli, ["create-table", "test.db", "dogs", "id", "integer"] ) - assert 1 == result.exit_code + assert result.exit_code == 1 assert ( 'Error: Table "dogs" already exists. Use --replace to delete and replace it.' == result.output.strip() @@ -1405,8 +1487,8 @@ def test_create_table_ignore(): result = runner.invoke( cli.cli, ["create-table", "test.db", "dogs", "id", "integer", "--ignore"] ) - assert 0 == result.exit_code - assert "CREATE TABLE [dogs] (\n [name] TEXT\n)" == db["dogs"].schema + assert result.exit_code == 0 + assert 'CREATE TABLE "dogs" (\n "name" TEXT\n)' == db["dogs"].schema def test_create_table_replace(): @@ -1417,8 +1499,8 @@ def test_create_table_replace(): result = runner.invoke( cli.cli, ["create-table", "test.db", "dogs", "id", "integer", "--replace"] ) - assert 0 == result.exit_code - assert "CREATE TABLE [dogs] (\n [id] INTEGER\n)" == db["dogs"].schema + assert result.exit_code == 0 + assert 'CREATE TABLE "dogs" (\n "id" INTEGER\n)' == db["dogs"].schema def test_create_view(): @@ -1428,8 +1510,10 @@ def test_create_view(): result = runner.invoke( cli.cli, ["create-view", "test.db", "version", "select sqlite_version()"] ) - assert 0 == result.exit_code - assert "CREATE VIEW version AS select sqlite_version()" == db["version"].schema + assert result.exit_code == 0 + assert ( + 'CREATE VIEW "version" AS select sqlite_version()' == db["version"].schema + ) def test_create_view_error_if_view_exists(): @@ -1440,7 +1524,7 @@ def test_create_view_error_if_view_exists(): result = runner.invoke( cli.cli, ["create-view", "test.db", "version", "select sqlite_version()"] ) - assert 1 == result.exit_code + assert result.exit_code == 1 assert ( 'Error: View "version" already exists. Use --replace to delete and replace it.' == result.output.strip() @@ -1462,9 +1546,10 @@ def test_create_view_ignore(): "--ignore", ], ) - assert 0 == result.exit_code + assert result.exit_code == 0 assert ( - "CREATE VIEW version AS select sqlite_version() + 1" == db["version"].schema + 'CREATE VIEW "version" AS select sqlite_version() + 1' + == db["version"].schema ) @@ -1483,8 +1568,10 @@ def test_create_view_replace(): "--replace", ], ) - assert 0 == result.exit_code - assert "CREATE VIEW version AS select sqlite_version()" == db["version"].schema + assert result.exit_code == 0 + assert ( + 'CREATE VIEW "version" AS select sqlite_version()' == db["version"].schema + ) def test_drop_table(): @@ -1501,7 +1588,7 @@ def test_drop_table(): "t", ], ) - assert 0 == result.exit_code + assert result.exit_code == 0 assert "t" not in db.table_names() @@ -1518,14 +1605,32 @@ def test_drop_table_error(): "t2", ], ) - assert 1 == result.exit_code + assert result.exit_code == 1 assert 'Error: Table "t2" does not exist' == result.output.strip() - # Using --ignore supresses that error + # Using --ignore suppresses that error result = runner.invoke( cli.cli, ["drop-table", "test.db", "t2", "--ignore"], ) - assert 0 == result.exit_code + assert result.exit_code == 0 + + +def test_drop_table_on_view_errors(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["t"].insert({"id": 1}) + db.create_view("v", "select * from t") + result = runner.invoke(cli.cli, ["drop-table", "test.db", "v"]) + assert result.exit_code == 1 + assert 'Error: "v" is a view, not a table - use drop-view to drop it' == ( + result.output.strip() + ) + assert "v" in db.view_names() + # --ignore exits cleanly but must still not drop the view + result = runner.invoke(cli.cli, ["drop-table", "test.db", "v", "--ignore"]) + assert result.exit_code == 0 + assert "v" in db.view_names() def test_drop_view(): @@ -1542,10 +1647,27 @@ def test_drop_view(): "hello", ], ) - assert 0 == result.exit_code + assert result.exit_code == 0 assert "hello" not in db.view_names() +def test_drop_view_on_table_errors(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["t"].insert({"id": 1}) + result = runner.invoke(cli.cli, ["drop-view", "test.db", "t"]) + assert result.exit_code == 1 + assert 'Error: "t" is a table, not a view - use drop-table to drop it' == ( + result.output.strip() + ) + assert "t" in db.table_names() + # --ignore exits cleanly but must still not drop the table + result = runner.invoke(cli.cli, ["drop-view", "test.db", "t", "--ignore"]) + assert result.exit_code == 0 + assert "t" in db.table_names() + + def test_drop_view_error(): runner = CliRunner() with runner.isolated_filesystem(): @@ -1559,14 +1681,14 @@ def test_drop_view_error(): "t2", ], ) - assert 1 == result.exit_code + assert result.exit_code == 1 assert 'Error: View "t2" does not exist' == result.output.strip() - # Using --ignore supresses that error + # Using --ignore suppresses that error result = runner.invoke( cli.cli, ["drop-view", "test.db", "t2", "--ignore"], ) - assert 0 == result.exit_code + assert result.exit_code == 0 def test_enable_wal(): @@ -1577,8 +1699,8 @@ def test_enable_wal(): db = Database(dbname) db["t"].create({"pk": int}, pk="pk") assert db.journal_mode == "delete" - result = runner.invoke(cli.cli, ["enable-wal"] + dbs) - assert 0 == result.exit_code + result = runner.invoke(cli.cli, ["enable-wal"] + dbs, catch_exceptions=False) + assert result.exit_code == 0 for dbname in dbs: db = Database(dbname) assert db.journal_mode == "wal" @@ -1594,7 +1716,7 @@ def test_disable_wal(): db.enable_wal() assert db.journal_mode == "wal" result = runner.invoke(cli.cli, ["disable-wal"] + dbs) - assert 0 == result.exit_code + assert result.exit_code == 0 for dbname in dbs: db = Database(dbname) assert db.journal_mode == "delete" @@ -1672,9 +1794,9 @@ def test_add_foreign_keys(db_path): [], ( 'CREATE TABLE "dogs" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [age] INTEGER NOT NULL DEFAULT '1',\n" - " [name] TEXT\n" + ' "id" INTEGER PRIMARY KEY,\n' + " \"age\" INTEGER NOT NULL DEFAULT '1',\n" + ' "name" TEXT\n' ")" ), ), @@ -1682,9 +1804,9 @@ def test_add_foreign_keys(db_path): ["--type", "age", "text"], ( 'CREATE TABLE "dogs" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [age] TEXT NOT NULL DEFAULT '1',\n" - " [name] TEXT\n" + ' "id" INTEGER PRIMARY KEY,\n' + " \"age\" TEXT NOT NULL DEFAULT '1',\n" + ' "name" TEXT\n' ")" ), ), @@ -1692,8 +1814,8 @@ def test_add_foreign_keys(db_path): ["--drop", "age"], ( 'CREATE TABLE "dogs" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT\n" + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT\n' ")" ), ), @@ -1701,9 +1823,9 @@ def test_add_foreign_keys(db_path): ["--rename", "age", "age2", "--rename", "id", "pk"], ( 'CREATE TABLE "dogs" (\n' - " [pk] INTEGER PRIMARY KEY,\n" - " [age2] INTEGER NOT NULL DEFAULT '1',\n" - " [name] TEXT\n" + ' "pk" INTEGER PRIMARY KEY,\n' + " \"age2\" INTEGER NOT NULL DEFAULT '1',\n" + ' "name" TEXT\n' ")" ), ), @@ -1711,9 +1833,9 @@ def test_add_foreign_keys(db_path): ["--not-null", "name"], ( 'CREATE TABLE "dogs" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [age] INTEGER NOT NULL DEFAULT '1',\n" - " [name] TEXT NOT NULL\n" + ' "id" INTEGER PRIMARY KEY,\n' + " \"age\" INTEGER NOT NULL DEFAULT '1',\n" + ' "name" TEXT NOT NULL\n' ")" ), ), @@ -1721,9 +1843,9 @@ def test_add_foreign_keys(db_path): ["--not-null-false", "age"], ( 'CREATE TABLE "dogs" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [age] INTEGER DEFAULT '1',\n" - " [name] TEXT\n" + ' "id" INTEGER PRIMARY KEY,\n' + " \"age\" INTEGER DEFAULT '1',\n" + ' "name" TEXT\n' ")" ), ), @@ -1731,9 +1853,9 @@ def test_add_foreign_keys(db_path): ["--pk", "name"], ( 'CREATE TABLE "dogs" (\n' - " [id] INTEGER,\n" - " [age] INTEGER NOT NULL DEFAULT '1',\n" - " [name] TEXT PRIMARY KEY\n" + ' "id" INTEGER,\n' + " \"age\" INTEGER NOT NULL DEFAULT '1',\n" + ' "name" TEXT PRIMARY KEY\n' ")" ), ), @@ -1741,9 +1863,9 @@ def test_add_foreign_keys(db_path): ["--pk-none"], ( 'CREATE TABLE "dogs" (\n' - " [id] INTEGER,\n" - " [age] INTEGER NOT NULL DEFAULT '1',\n" - " [name] TEXT\n" + ' "id" INTEGER,\n' + " \"age\" INTEGER NOT NULL DEFAULT '1',\n" + ' "name" TEXT\n' ")" ), ), @@ -1751,9 +1873,9 @@ def test_add_foreign_keys(db_path): ["--default", "name", "Turnip"], ( 'CREATE TABLE "dogs" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [age] INTEGER NOT NULL DEFAULT '1',\n" - " [name] TEXT DEFAULT 'Turnip'\n" + ' "id" INTEGER PRIMARY KEY,\n' + " \"age\" INTEGER NOT NULL DEFAULT '1',\n" + " \"name\" TEXT DEFAULT 'Turnip'\n" ")" ), ), @@ -1761,9 +1883,9 @@ def test_add_foreign_keys(db_path): ["--default-none", "age"], ( 'CREATE TABLE "dogs" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [age] INTEGER NOT NULL,\n" - " [name] TEXT\n" + ' "id" INTEGER PRIMARY KEY,\n' + ' "age" INTEGER NOT NULL,\n' + ' "name" TEXT\n' ")" ), ), @@ -1771,9 +1893,9 @@ def test_add_foreign_keys(db_path): ["-o", "name", "--column-order", "age", "-o", "id"], ( 'CREATE TABLE "dogs" (\n' - " [name] TEXT,\n" - " [age] INTEGER NOT NULL DEFAULT '1',\n" - " [id] INTEGER PRIMARY KEY\n" + ' "name" TEXT,\n' + " \"age\" INTEGER NOT NULL DEFAULT '1',\n" + ' "id" INTEGER PRIMARY KEY\n' ")" ), ), @@ -1795,10 +1917,133 @@ def test_transform(db_path, args, expected_schema): assert schema == expected_schema -def test_transform_drop_foreign_key(db_path): +def test_transform_sql(db_path): + db = Database(db_path) + with db.conn: + db["dogs"].insert( + {"id": 1, "age": 4, "name": "Cleo"}, + not_null={"age"}, + defaults={"age": 1}, + pk="id", + ) + original_schema = db["dogs"].schema + + result = CliRunner().invoke( + cli.cli, ["transform", db_path, "dogs", "--drop", "name", "--sql"] + ) + + assert result.exit_code == 0, result.output + assert 'CREATE TABLE "dogs_new_' in result.output + assert '"age" INTEGER NOT NULL DEFAULT' in result.output + assert 'DROP TABLE "dogs";' in result.output + assert 'ALTER TABLE "dogs_new_' in result.output + assert db["dogs"].schema == original_schema + + +@pytest.mark.parametrize( + "initial_strict,args,expected_strict", + ( + (False, [], False), + (True, [], True), + (False, ["--strict"], True), + (True, ["--no-strict"], False), + ), +) +def test_transform_strict_option(db_path, initial_strict, args, expected_strict): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db["dogs"].create({"id": int}, strict=initial_strict) + + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs"] + args) + + assert result.exit_code == 0, result.output + assert db["dogs"].strict is expected_strict + + +@pytest.mark.parametrize( + "initial_strict,flag,sql_is_strict", + ( + (False, "--strict", True), + (True, "--no-strict", False), + ), +) +def test_transform_strict_option_sql(db_path, initial_strict, flag, sql_is_strict): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db["dogs"].create({"id": int}, strict=initial_strict) + + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", flag, "--sql"]) + + assert result.exit_code == 0, result.output + assert (") STRICT;" in result.output) is sql_is_strict + assert db["dogs"].strict is initial_strict + + +def test_transform_strict_option_with_invalid_data(db_path): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + dogs = db["dogs"] + dogs.create({"id": int}) + dogs.insert({"id": "not-an-integer"}) + + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", "--strict"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, sqlite3.IntegrityError) + assert dogs.strict is False + assert list(dogs.rows) == [{"id": "not-an-integer"}] + assert not any(name.startswith("dogs_new_") for name in db.table_names()) + + +@pytest.mark.parametrize( + "extra_args,expected_schema", + ( + ( + ["--drop-foreign-key", "country"], + ( + 'CREATE TABLE "places" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT,\n' + ' "country" INTEGER,\n' + ' "city" INTEGER REFERENCES "city"("id"),\n' + ' "continent" INTEGER\n' + ")" + ), + ), + ( + ["--drop-foreign-key", "country", "--drop-foreign-key", "city"], + ( + 'CREATE TABLE "places" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT,\n' + ' "country" INTEGER,\n' + ' "city" INTEGER,\n' + ' "continent" INTEGER\n' + ")" + ), + ), + ( + ["--add-foreign-key", "continent", "continent", "id"], + ( + 'CREATE TABLE "places" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT,\n' + ' "country" INTEGER REFERENCES "country"("id"),\n' + ' "city" INTEGER REFERENCES "city"("id"),\n' + ' "continent" INTEGER REFERENCES "continent"("id")\n' + ")" + ), + ), + ), +) +def test_transform_add_or_drop_foreign_key(db_path, extra_args, expected_schema): db = Database(db_path) with db.conn: # Create table with three foreign keys so we can drop two of them + db["continent"].insert({"id": 1, "name": "Europe"}, pk="id") db["country"].insert({"id": 1, "name": "France"}, pk="id") db["city"].insert({"id": 24, "name": "Paris"}, pk="id") db["places"].insert( @@ -1807,6 +2052,7 @@ def test_transform_drop_foreign_key(db_path): "name": "Caveau de la Huchette", "country": 1, "city": 24, + "continent": 1, }, foreign_keys=("country", "city"), pk="id", @@ -1817,25 +2063,16 @@ def test_transform_drop_foreign_key(db_path): "transform", db_path, "places", - "--drop-foreign-key", - "country", - ], + ] + + extra_args, ) - print(result.output) assert result.exit_code == 0 schema = db["places"].schema - assert schema == ( - 'CREATE TABLE "places" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT,\n" - " [country] INTEGER,\n" - " [city] INTEGER REFERENCES [city]([id])\n" - ")" - ) + assert schema == expected_schema _common_other_schema = ( - "CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)" + 'CREATE TABLE "species" (\n "id" INTEGER PRIMARY KEY,\n "species" TEXT\n)' ) @@ -1846,10 +2083,9 @@ _common_other_schema = ( [], ( 'CREATE TABLE "trees" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [address] TEXT,\n" - " [species_id] INTEGER,\n" - " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ' "id" INTEGER PRIMARY KEY,\n' + ' "address" TEXT,\n' + ' "species_id" INTEGER REFERENCES "species"("id")\n' ")" ), _common_other_schema, @@ -1858,22 +2094,20 @@ _common_other_schema = ( ["--table", "custom_table"], ( 'CREATE TABLE "trees" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [address] TEXT,\n" - " [custom_table_id] INTEGER,\n" - " FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n" + ' "id" INTEGER PRIMARY KEY,\n' + ' "address" TEXT,\n' + ' "custom_table_id" INTEGER REFERENCES "custom_table"("id")\n' ")" ), - "CREATE TABLE [custom_table] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", + 'CREATE TABLE "custom_table" (\n "id" INTEGER PRIMARY KEY,\n "species" TEXT\n)', ), ( ["--fk-column", "custom_fk"], ( 'CREATE TABLE "trees" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [address] TEXT,\n" - " [custom_fk] INTEGER,\n" - " FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n" + ' "id" INTEGER PRIMARY KEY,\n' + ' "address" TEXT,\n' + ' "custom_fk" INTEGER REFERENCES "species"("id")\n' ")" ), _common_other_schema, @@ -1881,12 +2115,11 @@ _common_other_schema = ( ( ["--rename", "name", "name2"], 'CREATE TABLE "trees" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [address] TEXT,\n" - " [species_id] INTEGER,\n" - " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ' "id" INTEGER PRIMARY KEY,\n' + ' "address" TEXT,\n' + ' "species_id" INTEGER REFERENCES "species"("id")\n' ")", - "CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", + 'CREATE TABLE "species" (\n "id" INTEGER PRIMARY KEY,\n "species" TEXT\n)', ), ], ) @@ -1920,7 +2153,8 @@ def test_insert_encoding(tmpdir): ) assert latin1_csv.decode("latin-1").split("\n")[2].split(",")[1] == "São Paulo" csv_path = str(tmpdir / "test.csv") - open(csv_path, "wb").write(latin1_csv) + with open(csv_path, "wb") as fp: + fp.write(latin1_csv) # First attempt should error: bad_result = CliRunner().invoke( cli.cli, @@ -1935,7 +2169,16 @@ def test_insert_encoding(tmpdir): # Using --encoding=latin-1 should work good_result = CliRunner().invoke( cli.cli, - ["insert", db_path, "places", csv_path, "--encoding", "latin-1", "--csv"], + [ + "insert", + db_path, + "places", + csv_path, + "--encoding", + "latin-1", + "--csv", + "--no-detect-types", + ], catch_exceptions=False, ) assert good_result.exit_code == 0 @@ -1994,15 +2237,32 @@ def test_search(tmpdir, fts, extra_arg, expected): assert result.output.replace("\r", "") == expected +def test_search_quote(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["creatures"].insert({"name": "dog."}).enable_fts(["name"]) + # Without --quote should return an error + error_result = CliRunner().invoke(cli.cli, ["search", db_path, "creatures", 'dog"']) + assert error_result.exit_code == 1 + assert error_result.output == ( + "Error: unterminated string\n\n" + "Try running this again with the --quote option\n" + ) + # With --quote it should work + result = CliRunner().invoke( + cli.cli, ["search", db_path, "creatures", 'dog"', "--quote"] + ) + assert result.exit_code == 0 + assert result.output.strip() == '[{"rowid": 1, "name": "dog."}]' + + def test_indexes(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db.conn.executescript( - """ + db.conn.executescript(""" create table Gosh (c1 text, c2 text, c3 text); create index Gosh_idx on Gosh(c2, c3 desc); - """ - ) + """) result = CliRunner().invoke( cli.cli, ["indexes", str(db_path)], @@ -2079,7 +2339,11 @@ _TRIGGERS_EXPECTED = ( @pytest.mark.parametrize( "extra_args,expected", - [([], _TRIGGERS_EXPECTED), (["articles"], _TRIGGERS_EXPECTED), (["counter"], "")], + [ + ([], _TRIGGERS_EXPECTED), + (["articles"], _TRIGGERS_EXPECTED), + (["counter"], "[]\n"), + ], ) def test_triggers(tmpdir, extra_args, expected): db_path = str(tmpdir / "test.db") @@ -2089,16 +2353,12 @@ def test_triggers(tmpdir, extra_args, expected): pk="id", ) db["counter"].insert({"count": 1}) - db.conn.execute( - textwrap.dedent( - """ + db.conn.execute(textwrap.dedent(""" CREATE TRIGGER blah AFTER INSERT ON articles BEGIN UPDATE counter SET count = count + 1; END - """ - ) - ) + """)) args = ["triggers", db_path] if extra_args: args.extend(extra_args) @@ -2117,34 +2377,34 @@ def test_triggers(tmpdir, extra_args, expected): ( [], ( - "CREATE TABLE [dogs] (\n" - " [id] INTEGER,\n" - " [name] TEXT\n" + 'CREATE TABLE "dogs" (\n' + ' "id" INTEGER,\n' + ' "name" TEXT\n' ");\n" - "CREATE TABLE [chickens] (\n" - " [id] INTEGER,\n" - " [name] TEXT,\n" - " [breed] TEXT\n" + 'CREATE TABLE "chickens" (\n' + ' "id" INTEGER,\n' + ' "name" TEXT,\n' + ' "breed" TEXT\n' ");\n" - "CREATE INDEX [idx_chickens_breed]\n" - " ON [chickens] ([breed]);\n" + 'CREATE INDEX "idx_chickens_breed"\n' + ' ON "chickens" ("breed");\n' ), ), ( ["dogs"], - ("CREATE TABLE [dogs] (\n" " [id] INTEGER,\n" " [name] TEXT\n" ")\n"), + ('CREATE TABLE "dogs" (\n' ' "id" INTEGER,\n' ' "name" TEXT\n' ")\n"), ), ( ["chickens", "dogs"], ( - "CREATE TABLE [chickens] (\n" - " [id] INTEGER,\n" - " [name] TEXT,\n" - " [breed] TEXT\n" + 'CREATE TABLE "chickens" (\n' + ' "id" INTEGER,\n' + ' "name" TEXT,\n' + ' "breed" TEXT\n' ")\n" - "CREATE TABLE [dogs] (\n" - " [id] INTEGER,\n" - " [name] TEXT\n" + 'CREATE TABLE "dogs" (\n' + ' "id" INTEGER,\n' + ' "name" TEXT\n' ")\n" ), ), @@ -2168,11 +2428,10 @@ def test_schema(tmpdir, options, expected): def test_long_csv_column_value(tmpdir): db_path = str(tmpdir / "test.db") csv_path = str(tmpdir / "test.csv") - csv_file = open(csv_path, "w") - long_string = "a" * 131073 - csv_file.write("id,text\n") - csv_file.write("1,{}\n".format(long_string)) - csv_file.close() + with open(csv_path, "w") as csv_file: + long_string = "a" * 131073 + csv_file.write("id,text\n") + csv_file.write("1,{}\n".format(long_string)) result = CliRunner().invoke( cli.cli, ["insert", db_path, "bigtable", csv_path, "--csv"], @@ -2186,32 +2445,33 @@ def test_long_csv_column_value(tmpdir): @pytest.mark.parametrize( - "args", + "args,tsv", ( - ["--csv", "--no-headers"], - ["--no-headers"], + (["--csv", "--no-headers"], False), + (["--no-headers"], False), + (["--tsv", "--no-headers"], True), ), ) -def test_csv_import_no_headers(tmpdir, args): +def test_import_no_headers(tmpdir, args, tsv): db_path = str(tmpdir / "test.db") csv_path = str(tmpdir / "test.csv") - csv_file = open(csv_path, "w") - csv_file.write("Cleo,Dog,5\n") - csv_file.write("Tracy,Spider,7\n") - csv_file.close() + with open(csv_path, "w") as csv_file: + sep = "\t" if tsv else "," + csv_file.write("Cleo{sep}Dog{sep}5\n".format(sep=sep)) + csv_file.write("Tracy{sep}Spider{sep}7\n".format(sep=sep)) result = CliRunner().invoke( cli.cli, - ["insert", db_path, "creatures", csv_path] + args, + ["insert", db_path, "creatures", csv_path] + args + ["--no-detect-types"], catch_exceptions=False, ) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output db = Database(db_path) schema = db["creatures"].schema assert schema == ( - "CREATE TABLE [creatures] (\n" - " [untitled_1] TEXT,\n" - " [untitled_2] TEXT,\n" - " [untitled_3] TEXT\n" + 'CREATE TABLE "creatures" (\n' + ' "untitled_1" TEXT,\n' + ' "untitled_2" TEXT,\n' + ' "untitled_3" TEXT\n' ")" ) rows = list(db["creatures"].rows) @@ -2250,64 +2510,166 @@ def test_csv_insert_bom(tmpdir): fp.write(b"\xef\xbb\xbfname,age\nCleo,5") result = CliRunner().invoke( cli.cli, - ["insert", db_path, "broken", bom_csv_path, "--encoding", "utf-8", "--csv"], + [ + "insert", + db_path, + "broken", + bom_csv_path, + "--encoding", + "utf-8", + "--csv", + "--no-detect-types", + ], catch_exceptions=False, ) assert result.exit_code == 0 result2 = CliRunner().invoke( cli.cli, - ["insert", db_path, "fixed", bom_csv_path, "--csv"], + ["insert", db_path, "fixed", bom_csv_path, "--csv", "--no-detect-types"], catch_exceptions=False, ) assert result2.exit_code == 0 db = Database(db_path) tables = db.execute("select name, sql from sqlite_master").fetchall() assert tables == [ - ("broken", "CREATE TABLE [broken] (\n [\ufeffname] TEXT,\n [age] TEXT\n)"), - ("fixed", "CREATE TABLE [fixed] (\n [name] TEXT,\n [age] TEXT\n)"), + ("broken", 'CREATE TABLE "broken" (\n "\ufeffname" TEXT,\n "age" TEXT\n)'), + ("fixed", 'CREATE TABLE "fixed" (\n "name" TEXT,\n "age" TEXT\n)'), ] -@pytest.mark.parametrize("option_or_env_var", (None, "-d", "--detect-types")) -def test_insert_detect_types(tmpdir, option_or_env_var): +def test_insert_detect_types(tmpdir): + """Test that type detection is the default behavior""" db_path = str(tmpdir / "test.db") data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5" - extra = [] - if option_or_env_var: - extra = [option_or_env_var] - def _test(): - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "creatures", "-", "--csv"] + extra, - catch_exceptions=False, - input=data, - ) - assert result.exit_code == 0 - db = Database(db_path) - assert list(db["creatures"].rows) == [ - {"name": "Cleo", "age": 6, "weight": 45.5}, - {"name": "Dori", "age": 1, "weight": 3.5}, - ] - - if option_or_env_var is None: - # Use environemnt variable instead of option - with mock.patch.dict(os.environ, {"SQLITE_UTILS_DETECT_TYPES": "1"}): - _test() - else: - _test() + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "-", "--csv"], + catch_exceptions=False, + input=data, + ) + assert result.exit_code == 0 + db = Database(db_path) + assert list(db["creatures"].rows) == [ + {"name": "Cleo", "age": 6, "weight": 45.5}, + {"name": "Dori", "age": 1, "weight": 3.5}, + ] -@pytest.mark.parametrize( - "input,expected", - ( - ({"foo": {"bar": 1}}, {"foo_bar": 1}), - ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}), - ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}), - ), -) -def test_flatten_helper(input, expected): - assert dict(cli._flatten(input)) == expected +@pytest.mark.parametrize("command", ("insert", "upsert")) +@pytest.mark.parametrize("option", ("-d", "--detect-types")) +def test_detect_types_flag_removed(tmpdir, command, option): + # The old no-op flag was removed in 4.0 - it should now error + db_path = str(tmpdir / "test.db") + result = CliRunner().invoke( + cli.cli, + [command, db_path, "creatures", "-", "--csv", "--pk", "id", option], + input="id,name\n1,Cleo", + ) + assert result.exit_code == 2 + assert "No such option" in result.output + + +def test_upsert_detect_types(tmpdir): + """Test that type detection is the default behavior for upsert""" + db_path = str(tmpdir / "test.db") + data = "id,name,age,weight\n1,Cleo,6,45.5\n2,Dori,1,3.5" + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "creatures", "-", "--csv", "--pk", "id"], + catch_exceptions=False, + input=data, + ) + assert result.exit_code == 0 + db = Database(db_path) + assert list(db["creatures"].rows) == [ + {"id": 1, "name": "Cleo", "age": 6, "weight": 45.5}, + {"id": 2, "name": "Dori", "age": 1, "weight": 3.5}, + ] + + +def test_csv_detect_types_creates_real_columns(tmpdir): + """Test that CSV import creates REAL columns for floats (default behavior)""" + db_path = str(tmpdir / "test.db") + data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5" + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "-", "--csv"], + catch_exceptions=False, + input=data, + ) + assert result.exit_code == 0 + db = Database(db_path) + # Check that the schema uses REAL for the weight column + assert db["creatures"].schema == ( + 'CREATE TABLE "creatures" (\n' + ' "name" TEXT,\n' + ' "age" INTEGER,\n' + ' "weight" REAL\n' + ")" + ) + + +def test_insert_no_detect_types(tmpdir): + """Test that --no-detect-types treats all columns as TEXT""" + db_path = str(tmpdir / "test.db") + data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5" + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "-", "--csv", "--no-detect-types"], + catch_exceptions=False, + input=data, + ) + assert result.exit_code == 0 + db = Database(db_path) + # All columns should be TEXT when --no-detect-types is used + assert list(db["creatures"].rows) == [ + {"name": "Cleo", "age": "6", "weight": "45.5"}, + {"name": "Dori", "age": "1", "weight": "3.5"}, + ] + assert db["creatures"].schema == ( + 'CREATE TABLE "creatures" (\n' + ' "name" TEXT,\n' + ' "age" TEXT,\n' + ' "weight" TEXT\n' + ")" + ) + + +def test_upsert_no_detect_types(tmpdir): + """Test that --no-detect-types treats all columns as TEXT for upsert""" + db_path = str(tmpdir / "test.db") + data = "id,name,age,weight\n1,Cleo,6,45.5\n2,Dori,1,3.5" + result = CliRunner().invoke( + cli.cli, + [ + "upsert", + db_path, + "creatures", + "-", + "--csv", + "--pk", + "id", + "--no-detect-types", + ], + catch_exceptions=False, + input=data, + ) + assert result.exit_code == 0 + db = Database(db_path) + # All columns should be TEXT when --no-detect-types is used + assert list(db["creatures"].rows) == [ + {"id": "1", "name": "Cleo", "age": "6", "weight": "45.5"}, + {"id": "2", "name": "Dori", "age": "1", "weight": "3.5"}, + ] + assert db["creatures"].schema == ( + 'CREATE TABLE "creatures" (\n' + ' "id" TEXT PRIMARY KEY,\n' + ' "name" TEXT,\n' + ' "age" TEXT,\n' + ' "weight" TEXT\n' + ")" + ) def test_integer_overflow_error(tmpdir): @@ -2320,6 +2682,212 @@ def test_integer_overflow_error(tmpdir): assert result.exit_code == 1 assert result.output == ( "Error: Python int too large to convert to SQLite INTEGER\n\n" - "sql = INSERT INTO [items] ([bignumber]) VALUES (?);\n" + 'sql = INSERT INTO "items" ("bignumber") VALUES (?)\n' "parameters = [34223049823094832094802398430298048240]\n" ) + + +def test_python_dash_m(): + "Tool can be run using python -m sqlite_utils" + result = subprocess.run( + [sys.executable, "-m", "sqlite_utils", "--help"], stdout=subprocess.PIPE + ) + assert result.returncode == 0 + assert b"Commands for interacting with a SQLite database" in result.stdout + + +@pytest.mark.parametrize("enable_wal", (False, True)) +def test_create_database(tmpdir, enable_wal): + db_path = tmpdir / "test.db" + assert not db_path.exists() + args = ["create-database", str(db_path)] + if enable_wal: + args.append("--enable-wal") + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0, result.output + assert db_path.exists() + assert db_path.read_binary()[:16] == b"SQLite format 3\x00" + db = Database(str(db_path)) + if enable_wal: + assert db.journal_mode == "wal" + else: + assert db.journal_mode == "delete" + + +@pytest.mark.parametrize( + "options,expected", + ( + ( + [], + [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, + {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, + ], + ), + ( + ["one_index"], + [ + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, + ], + ), + ( + ["idx_two_indexes_name"], + [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, + ], + ), + ), +) +def test_analyze(tmpdir, options, expected): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") + db["one_index"].create_index(["name"]) + db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") + db["two_indexes"].create_index(["name"]) + db["two_indexes"].create_index(["species"]) + result = CliRunner().invoke(cli.cli, ["analyze", db_path] + options) + assert result.exit_code == 0 + assert list(db["sqlite_stat1"].rows) == expected + + +def test_rename_table(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["one"].insert({"id": 1, "name": "Cleo"}, pk="id") + # First try a non-existent table + result_error = CliRunner().invoke( + cli.cli, + ["rename-table", db_path, "missing", "two"], + catch_exceptions=False, + ) + assert result_error.exit_code == 1 + assert result_error.output == ( + 'Error: Table "missing" could not be renamed. ' "no such table: missing\n" + ) + # And check --ignore works + result_error2 = CliRunner().invoke( + cli.cli, + ["rename-table", db_path, "missing", "two", "--ignore"], + catch_exceptions=False, + ) + assert result_error2.exit_code == 0 + previous_columns = db["one"].columns_dict + # Now try for a table that exists + result = CliRunner().invoke( + cli.cli, + ["rename-table", db_path, "one", "two"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert db["two"].columns_dict == previous_columns + + +def test_duplicate_table(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["one"].insert({"id": 1, "name": "Cleo"}, pk="id") + # First try a non-existent table + result_error = CliRunner().invoke( + cli.cli, + ["duplicate", db_path, "missing", "two"], + catch_exceptions=False, + ) + assert result_error.exit_code == 1 + assert result_error.output == 'Error: Table "missing" does not exist\n' + # And check --ignore works + result_error2 = CliRunner().invoke( + cli.cli, + ["duplicate", db_path, "missing", "two", "--ignore"], + catch_exceptions=False, + ) + assert result_error2.exit_code == 0 + # Now try for a table that exists + result = CliRunner().invoke( + cli.cli, + ["duplicate", db_path, "one", "two"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert db["one"].columns_dict == db["two"].columns_dict + assert list(db["one"].rows) == list(db["two"].rows) + + +@pytest.mark.skipif(not _has_compiled_ext(), reason="Requires compiled ext.c") +@pytest.mark.parametrize( + "entrypoint,should_pass,should_fail", + ( + (None, ("a",), ("b", "c")), + ("sqlite3_ext_b_init", ("b"), ("a", "c")), + ("sqlite3_ext_c_init", ("c"), ("a", "b")), + ), +) +def test_load_extension(entrypoint, should_pass, should_fail): + ext = COMPILED_EXTENSION_PATH + if entrypoint: + ext += ":" + entrypoint + for func in should_pass: + result = CliRunner().invoke( + cli.cli, + ["memory", "select {}()".format(func), "--load-extension", ext], + catch_exceptions=False, + ) + assert result.exit_code == 0 + for func in should_fail: + result = CliRunner().invoke( + cli.cli, + ["memory", "select {}()".format(func), "--load-extension", ext], + catch_exceptions=False, + ) + assert result.exit_code == 1 + + +@pytest.mark.parametrize("strict", (False, True)) +def test_create_table_strict(strict): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + result = runner.invoke( + cli.cli, + ["create-table", "test.db", "items", "id", "integer", "w", "float"] + + (["--strict"] if strict else []), + ) + assert result.exit_code == 0 + assert db["items"].strict == strict or not db.supports_strict + # Should have a floating point column + assert db["items"].columns_dict == {"id": int, "w": float} + + +@pytest.mark.parametrize("method", ("insert", "upsert")) +@pytest.mark.parametrize("strict", (False, True)) +def test_insert_upsert_strict(tmpdir, method, strict): + db_path = str(tmpdir / "test.db") + result = CliRunner().invoke( + cli.cli, + [method, db_path, "items", "-", "--csv", "--pk", "id"] + + (["--strict"] if strict else []), + input="id\n1", + ) + assert result.exit_code == 0 + db = Database(db_path) + assert db["items"].strict == strict or not db.supports_strict + + +def test_extract_bad_column_clean_error(db_path): + db = Database(db_path) + db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + result = CliRunner().invoke(cli.cli, ["extract", db_path, "trees", "nope"]) + assert result.exit_code == 1 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.startswith("Error: Invalid columns") + + +def test_extract_view_clean_error(db_path): + db = Database(db_path) + db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + db.create_view("v", "select * from trees") + result = CliRunner().invoke(cli.cli, ["extract", db_path, "v", "species"]) + assert result.exit_code == 1 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.startswith("Error:") diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py new file mode 100644 index 0000000..514f4ac --- /dev/null +++ b/tests/test_cli_bulk.py @@ -0,0 +1,123 @@ +from click.testing import CliRunner +from sqlite_utils import cli, Database +import pathlib +import pytest +import subprocess +import sys +import time + + +@pytest.fixture +def test_db_and_path(tmpdir): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = Database(db_path) + db["example"].insert_all( + [ + {"id": 1, "name": "One"}, + {"id": 2, "name": "Two"}, + ], + pk="id", + ) + return db, db_path + + +def test_cli_bulk(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "bulk", + db_path, + "insert into example (id, name) values (:id, myupper(:name))", + "-", + "--nl", + "--functions", + "myupper = lambda s: s.upper()", + ], + input='{"id": 3, "name": "Three"}\n{"id": 4, "name": "Four"}\n', + ) + assert result.exit_code == 0, result.output + assert [ + {"id": 1, "name": "One"}, + {"id": 2, "name": "Two"}, + {"id": 3, "name": "THREE"}, + {"id": 4, "name": "FOUR"}, + ] == list(db["example"].rows) + + +def test_cli_bulk_multiple_functions(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "bulk", + db_path, + "insert into example (id, name) values (:id, myupper(mylower(:name)))", + "-", + "--nl", + "--functions", + "myupper = lambda s: s.upper()", + "--functions", + "mylower = lambda s: s.lower()", + ], + input='{"id": 3, "name": "ThReE"}\n{"id": 4, "name": "FoUr"}\n', + ) + assert result.exit_code == 0, result.output + assert [ + {"id": 1, "name": "One"}, + {"id": 2, "name": "Two"}, + {"id": 3, "name": "THREE"}, + {"id": 4, "name": "FOUR"}, + ] == list(db["example"].rows) + + +def test_cli_bulk_batch_size(test_db_and_path): + db, db_path = test_db_and_path + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "sqlite_utils", + "bulk", + db_path, + "insert into example (id, name) values (:id, :name)", + "-", + "--nl", + "--batch-size", + "2", + ], + stdin=subprocess.PIPE, + stdout=sys.stdout, + ) + # Writing one record should not commit + proc.stdin.write(b'{"id": 3, "name": "Three"}\n\n') + proc.stdin.flush() + time.sleep(1) + assert db["example"].count == 2 + + # Writing another should trigger a commit: + proc.stdin.write(b'{"id": 4, "name": "Four"}\n\n') + proc.stdin.flush() + time.sleep(1) + assert db["example"].count == 4 + + proc.stdin.close() + proc.wait() + assert proc.returncode == 0 + + +def test_cli_bulk_error(test_db_and_path): + _, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "bulk", + db_path, + "insert into example (id, name) value (:id, :name)", + "-", + "--nl", + ], + input='{"id": 3, "name": "Three"}', + ) + assert result.exit_code == 1 + assert result.output == 'Error: near "value": syntax error\n' diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 8ce5203..6c3f5c5 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -35,39 +35,40 @@ def fresh_db_and_path(tmpdir): "return value.replace('October', 'Spooktober')", # Return is optional: "value.replace('October', 'Spooktober')", + # Multiple lines are supported: + "v = value.replace('October', 'Spooktober')\nreturn v", + # Can also define a convert() function + "def convert(value): return value.replace('October', 'Spooktober')", + # ... with imports + "import re\n\ndef convert(value): return value.replace('October', 'Spooktober')", ], ) -def test_convert_single_line(test_db_and_path, code): - db, db_path = test_db_and_path - result = CliRunner().invoke(cli.cli, ["convert", db_path, "example", "dt", code]) - assert 0 == result.exit_code, result.output - assert [ - {"id": 1, "dt": "5th Spooktober 2019 12:04"}, - {"id": 2, "dt": "6th Spooktober 2019 00:05:06"}, - {"id": 3, "dt": ""}, - {"id": 4, "dt": None}, - ] == list(db["example"].rows) - - -def test_convert_multiple_lines(test_db_and_path): - db, db_path = test_db_and_path +def test_convert_code(fresh_db_and_path, code): + db, db_path = fresh_db_and_path + db["t"].insert({"text": "October"}) result = CliRunner().invoke( - cli.cli, - [ - "convert", - db_path, - "example", - "dt", - "v = value.replace('October', 'Spooktober')\nreturn v.upper()", - ], + cli.cli, ["convert", db_path, "t", "text", code], catch_exceptions=False ) - assert 0 == result.exit_code, result.output - assert [ - {"id": 1, "dt": "5TH SPOOKTOBER 2019 12:04"}, - {"id": 2, "dt": "6TH SPOOKTOBER 2019 00:05:06"}, - {"id": 3, "dt": ""}, - {"id": 4, "dt": None}, - ] == list(db["example"].rows) + assert result.exit_code == 0, result.output + value = list(db["t"].rows)[0]["text"] + assert value == "Spooktober" + + +@pytest.mark.parametrize( + "bad_code", + ( + "def foo(value)", + "$", + ), +) +def test_convert_code_errors(fresh_db_and_path, bad_code): + db, db_path = fresh_db_and_path + db["t"].insert({"text": "October"}) + result = CliRunner().invoke( + cli.cli, ["convert", db_path, "t", "text", bad_code], catch_exceptions=False + ) + assert result.exit_code == 1 + assert result.output == "Error: Could not compile code\n" def test_convert_import(test_db_and_path): @@ -79,12 +80,12 @@ def test_convert_import(test_db_and_path): db_path, "example", "dt", - "return re.sub('O..', 'OXX', value)", + "return re.sub('O..', 'OXX', value) if value else value", "--import", "re", ], ) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output assert [ {"id": 1, "dt": "5th OXXober 2019 12:04"}, {"id": 2, "dt": "6th OXXober 2019 00:05:06"}, @@ -93,6 +94,27 @@ def test_convert_import(test_db_and_path): ] == list(db["example"].rows) +def test_convert_import_nested(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["example"].insert({"xml": ''}) + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "xml", + 'xml.etree.ElementTree.fromstring(value).attrib["name"]', + "--import", + "xml.etree.ElementTree", + ], + ) + assert result.exit_code == 0, result.output + assert [ + {"xml": "Cleo"}, + ] == list(db["example"].rows) + + def test_convert_dryrun(test_db_and_path): db, db_path = test_db_and_path result = CliRunner().invoke( @@ -157,6 +179,61 @@ def test_convert_dryrun(test_db_and_path): assert result.output.strip().split("\n")[-1] == "Would affect 1 row" +def test_convert_multi_dryrun(test_db_and_path): + db_path = test_db_and_path[1] + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "{'foo': 'bar', 'baz': 1}", + "--dry-run", + "--multi", + ], + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + "5th October 2019 12:04\n" + " --- becomes:\n" + '{"foo": "bar", "baz": 1}\n' + "\n" + "6th October 2019 00:05:06\n" + " --- becomes:\n" + '{"foo": "bar", "baz": 1}\n' + "\n" + "\n" + " --- becomes:\n" + "\n" + "\n" + "None\n" + " --- becomes:\n" + "None\n" + "\n" + "Would affect 4 rows" + ) + + +def test_convert_multi_dryrun_unicode_not_escaped(test_db_and_path): + db_path = test_db_and_path[1] + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "{'text': 'Japanese 日本語'}", + "--dry-run", + "--multi", + ], + ) + assert result.exit_code == 0 + # Preview should match what jsonify_if_needed() would actually store + assert '{"text": "Japanese 日本語"}' in result.output + + @pytest.mark.parametrize("drop", (True, False)) def test_convert_output_column(test_db_and_path, drop): db, db_path = test_db_and_path @@ -165,14 +242,14 @@ def test_convert_output_column(test_db_and_path, drop): db_path, "example", "dt", - "value.replace('October', 'Spooktober')", + "value.replace('October', 'Spooktober') if value else value", "--output", "newcol", ] if drop: args += ["--drop"] result = CliRunner().invoke(cli.cli, args) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output expected = [ { "id": 1, @@ -219,7 +296,7 @@ def test_convert_output_column_output_type(test_db_and_path, output_type, expect cli.cli, args, ) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output assert expected == list(db.execute("select id, new_id from example")) @@ -313,16 +390,14 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): ], pk="id", ) - code = textwrap.dedent( - """ + code = textwrap.dedent(""" if value == 1: return {"is_str": "", "is_float": 1.2, "is_int": None} elif value == 2: return {"is_float": 1, "is_int": 12} elif value == 3: return {"is_bytes": b"blah"} - """ - ) + """) result = CliRunner().invoke( cli.cli, [ @@ -348,9 +423,9 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): {"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None}, ] assert db["rows"].schema == ( - "CREATE TABLE [rows] (\n" - " [id] INTEGER PRIMARY KEY\n" - ", [is_str] TEXT, [is_float] FLOAT, [is_int] INTEGER, [is_bytes] BLOB)" + 'CREATE TABLE "rows" (\n' + ' "id" INTEGER PRIMARY KEY\n' + ', "is_str" TEXT, "is_float" REAL, "is_int" INTEGER, "is_bytes" BLOB)' ) @@ -370,7 +445,7 @@ def test_recipe_jsonsplit(tmpdir, delimiter): code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter) args = ["convert", db_path, "example", "tags", code] result = CliRunner().invoke(cli.cli, args) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output assert list(db["example"].rows) == [ {"id": 1, "tags": '["foo", "bar"]'}, {"id": 2, "tags": '["bar", "baz"]'}, @@ -398,7 +473,7 @@ def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): code = "recipes.jsonsplit(value, type={})".format(type) args = ["convert", db_path, "example", "records", code] result = CliRunner().invoke(cli.cli, args) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output assert json.loads(db["example"].get(1)["records"]) == expected_array @@ -416,7 +491,7 @@ def test_recipe_jsonsplit_output(fresh_db_and_path, drop): if drop: args += ["--drop"] result = CliRunner().invoke(cli.cli, args) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output expected = { "id": 1, "records": "1,2,3", @@ -477,7 +552,7 @@ def test_convert_where(test_db_and_path): "id = :id", "-p", "id", - 2, + "2", ], ) assert result.exit_code == 0, result.output @@ -506,12 +581,132 @@ def test_convert_where_multi(fresh_db_and_path): "id = :id", "-p", "id", - 2, + "2", "--multi", ], ) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output assert list(db["names"].rows) == [ {"id": 1, "name": "Cleo", "upper": None}, {"id": 2, "name": "Bants", "upper": "BANTS"}, ] + + +def test_convert_code_standard_input(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "names", + "name", + "-", + ], + input="value.upper()", + ) + assert result.exit_code == 0, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "CLEO"}, + ] + + +def test_convert_hyphen_workaround(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + result = CliRunner().invoke( + cli.cli, + ["convert", db_path, "names", "name", '"-"'], + ) + assert result.exit_code == 0, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "-"}, + ] + + +def test_convert_initialization_pattern(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "names", + "name", + "-", + ], + input="import random\nrandom.seed(1)\ndef convert(value): return random.randint(0, 100)", + ) + assert result.exit_code == 0, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "17"}, + ] + + +def test_convert_handles_falsey_values(fresh_db_and_path): + # Falsey values like 0 should be converted (issue #527) + db, db_path = fresh_db_and_path + args = [ + "convert", + db_path, + "t", + "x", + "-", + ] + db["t"].insert_all([{"x": 0}, {"x": 1}]) + assert db["t"].get(1)["x"] == 0 + assert db["t"].get(2)["x"] == 1 + result = CliRunner().invoke(cli.cli, args, input="value + 1") + assert result.exit_code == 0, result.output + assert db["t"].get(1)["x"] == 1 + assert db["t"].get(2)["x"] == 2 + + +@pytest.mark.parametrize( + "code", + [ + # Direct callable reference (issue #686) + "r.parsedate", + "recipes.parsedate", + # Traditional call syntax still works + "r.parsedate(value)", + "recipes.parsedate(value)", + ], +) +def test_convert_callable_reference(test_db_and_path, code): + """Test that callable references like r.parsedate work without (value)""" + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, ["convert", db_path, "example", "dt", code], catch_exceptions=False + ) + assert result.exit_code == 0, result.output + rows = list(db["example"].rows) + assert rows[0]["dt"] == "2019-10-05" + assert rows[1]["dt"] == "2019-10-06" + assert rows[2]["dt"] == "" + assert rows[3]["dt"] is None + + +def test_convert_callable_reference_with_import(fresh_db_and_path): + """Test callable reference from an imported module""" + db, db_path = fresh_db_and_path + db["example"].insert({"id": 1, "data": '{"name": "test"}'}) + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "data", + "json.loads", + "--import", + "json", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + # json.loads returns a dict, which sqlite stores as JSON string + row = db["example"].get(1) + assert row["data"] == '{"name": "test"}' diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py new file mode 100644 index 0000000..df6f80c --- /dev/null +++ b/tests/test_cli_insert.py @@ -0,0 +1,890 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner +import json +import pytest +import subprocess +import sys +import time + + +def test_insert_simple(tmpdir): + json_path = str(tmpdir / "dog.json") + db_path = str(tmpdir / "dogs.db") + with open(json_path, "w") as fp: + fp.write(json.dumps({"name": "Cleo", "age": 4})) + result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path]) + assert result.exit_code == 0 + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") + ) + db = Database(db_path) + assert ["dogs"] == db.table_names() + assert [] == db["dogs"].indexes + + +def test_insert_from_stdin(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-"], + input=json.dumps({"name": "Cleo", "age": 4}), + ) + assert result.exit_code == 0 + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") + ) + + +def test_insert_invalid_json_error(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-"], + input="name,age\nCleo,4", + ) + assert result.exit_code == 1 + assert result.output == ( + "Error: Invalid JSON - use --csv for CSV or --tsv for TSV files\n\n" + "JSON error: Expecting value: line 1 column 1 (char 0)\n" + ) + + +def test_insert_json_flatten(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten"], + input=json.dumps({"nested": {"data": 4}}), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}] + + +def test_insert_json_flatten_nl(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten", "--nl"], + input="\n".join( + json.dumps(item) + for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}] + ), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [ + {"nested_data": 4, "nested_other": None}, + {"nested_data": None, "nested_other": 3}, + ] + + +@pytest.mark.parametrize( + "args,expected_pks", + ( + (["--pk", "id"], ["id"]), + (["--pk", "id", "--pk", "name"], ["id", "name"]), + ), +) +def test_insert_with_primary_keys(db_path, tmpdir, args, expected_pks): + json_path = str(tmpdir / "dog.json") + with open(json_path, "w") as fp: + fp.write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) + result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path] + args) + assert result.exit_code == 0 + assert [{"id": 1, "age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") + ) + db = Database(db_path) + assert db["dogs"].pks == expected_pks + + +def test_insert_multiple_with_primary_key(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] + with open(json_path, "w") as fp: + fp.write(json.dumps(dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert result.exit_code == 0 + db = Database(db_path) + assert dogs == list(db.query("select * from dogs order by id")) + assert ["id"] == db["dogs"].pks + + +def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + dogs = [ + {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} + for i in range(1, 21) + ] + with open(json_path, "w") as fp: + fp.write(json.dumps(dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--pk", "breed"] + ) + assert result.exit_code == 0 + db = Database(db_path) + assert dogs == list(db.query("select * from dogs order by breed, id")) + assert {"breed", "id"} == set(db["dogs"].pks) + assert ( + 'CREATE TABLE "dogs" (\n' + ' "breed" TEXT,\n' + ' "id" INTEGER,\n' + ' "name" TEXT,\n' + ' "age" INTEGER,\n' + ' PRIMARY KEY ("id", "breed")\n' + ")" + ) == db["dogs"].schema + + +def test_insert_not_null_default(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + dogs = [ + {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10} + for i in range(1, 21) + ] + with open(json_path, "w") as fp: + fp.write(json.dumps(dogs)) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", json_path, "--pk", "id"] + + ["--not-null", "name", "--not-null", "age"] + + ["--default", "score", "5", "--default", "age", "1"], + ) + assert result.exit_code == 0 + db = Database(db_path) + assert ( + 'CREATE TABLE "dogs" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT NOT NULL,\n' + " \"age\" INTEGER NOT NULL DEFAULT '1',\n" + " \"score\" INTEGER DEFAULT '5'\n)" + ) == db["dogs"].schema + + +def test_insert_binary_base64(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "files", "-"], + input=r'{"content": {"$base64": true, "encoded": "aGVsbG8="}}', + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + actual = list(db.query("select content from files")) + assert actual == [{"content": b"hello"}] + + +def test_insert_newline_delimited(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl"], + input='{"foo": "bar", "n": 1}\n\n{"foo": "baz", "n": 2}', + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert [ + {"foo": "bar", "n": 1}, + {"foo": "baz", "n": 2}, + ] == list(db.query("select foo, n from from_json_nl")) + + +def test_insert_ignore(db_path, tmpdir): + db = Database(db_path) + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + json_path = str(tmpdir / "dogs.json") + with open(json_path, "w") as fp: + fp.write(json.dumps([{"id": 1, "name": "Bailey"}])) + # Should raise error without --ignore + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert result.exit_code != 0, result.output + # If we use --ignore it should run OK + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--ignore"] + ) + assert result.exit_code == 0, result.output + # ... but it should actually have no effect + assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs")) + + +@pytest.mark.parametrize( + "content,options", + [ + ("foo\tbar\tbaz\n1\t2\tcat,dog", ["--tsv"]), + ('foo,bar,baz\n1,2,"cat,dog"', ["--csv"]), + ('foo;bar;baz\n1;2;"cat,dog"', ["--csv", "--delimiter", ";"]), + # --delimiter implies --csv: + ('foo;bar;baz\n1;2;"cat,dog"', ["--delimiter", ";"]), + ("foo,bar,baz\n1,2,|cat,dog|", ["--csv", "--quotechar", "|"]), + ("foo,bar,baz\n1,2,|cat,dog|", ["--quotechar", "|"]), + ], +) +def test_insert_csv_tsv(content, options, db_path, tmpdir): + db = Database(db_path) + file_path = str(tmpdir / "insert.csv-tsv") + with open(file_path, "w") as fp: + fp.write(content) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "data", file_path] + options + ["--no-detect-types"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows) + + +@pytest.mark.parametrize("empty_null", (True, False)) +def test_insert_csv_empty_null(db_path, empty_null): + options = ["--csv", "--no-detect-types"] + if empty_null: + options.append("--empty-null") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "data", "-"] + options, + catch_exceptions=False, + input="foo,bar,baz\n1,,cat,dog", + ) + assert result.exit_code == 0 + db = Database(db_path) + assert [r for r in db["data"].rows] == [ + {"foo": "1", "bar": None if empty_null else "", "baz": "cat"} + ] + + +@pytest.mark.parametrize( + "input,args", + ( + ( + json.dumps( + [{"name": "One"}, {"name": "Two"}, {"name": "Three"}, {"name": "Four"}] + ), + [], + ), + ("name\nOne\nTwo\nThree\nFour\n", ["--csv"]), + ), +) +def test_insert_stop_after(tmpdir, input, args): + db_path = str(tmpdir / "data.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "rows", "-", "--stop-after", "2"] + args, + input=input, + ) + assert result.exit_code == 0 + assert [{"name": "One"}, {"name": "Two"}] == list( + Database(db_path).query("select * from rows") + ) + + +@pytest.mark.parametrize( + "options", + ( + ["--tsv", "--nl"], + ["--tsv", "--csv"], + ["--csv", "--nl"], + ["--csv", "--nl", "--tsv"], + ), +) +def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir): + file_path = str(tmpdir / "insert.csv-tsv") + with open(file_path, "w") as fp: + fp.write("foo") + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "data", file_path] + options + ) + assert result.exit_code != 0 + assert "Error: Use just one of --nl, --csv or --tsv" == result.output.strip() + + +def test_insert_replace(db_path, tmpdir): + test_insert_multiple_with_primary_key(db_path, tmpdir) + json_path = str(tmpdir / "insert-replace.json") + db = Database(db_path) + assert db["dogs"].count == 20 + insert_replace_dogs = [ + {"id": 1, "name": "Insert replaced 1", "age": 4}, + {"id": 2, "name": "Insert replaced 2", "age": 4}, + {"id": 21, "name": "Fresh insert 21", "age": 6}, + ] + with open(json_path, "w") as fp: + fp.write(json.dumps(insert_replace_dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"] + ) + assert result.exit_code == 0, result.output + assert db["dogs"].count == 21 + assert ( + list(db.query("select * from dogs where id in (1, 2, 21) order by id")) + == insert_replace_dogs + ) + + +def test_insert_truncate(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl", "--batch-size=1"], + input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert [ + {"foo": "bar", "n": 1}, + {"foo": "baz", "n": 2}, + ] == list(db.query("select foo, n from from_json_nl")) + # Truncate and insert new rows + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "from_json_nl", + "-", + "--nl", + "--truncate", + "--batch-size=1", + ], + input='{"foo": "bam", "n": 3}\n{"foo": "bat", "n": 4}', + ) + assert result.exit_code == 0, result.output + assert [ + {"foo": "bam", "n": 3}, + {"foo": "bat", "n": 4}, + ] == list(db.query("select foo, n from from_json_nl")) + + +def test_insert_alter(db_path, tmpdir): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl"], + input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + ) + assert result.exit_code == 0, result.output + # Should get an error with incorrect shaped additional data + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl"], + input='{"foo": "bar", "baz": 5}', + ) + assert result.exit_code != 0, result.output + # If we run it again with --alter it should work correctly + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl", "--alter"], + input='{"foo": "bar", "baz": 5}', + ) + assert result.exit_code == 0, result.output + # Soundness check the database itself + db = Database(db_path) + assert {"foo": str, "n": int, "baz": int} == db["from_json_nl"].columns_dict + assert [ + {"foo": "bar", "n": 1, "baz": None}, + {"foo": "baz", "n": 2, "baz": None}, + {"foo": "bar", "baz": 5, "n": None}, + ] == list(db.query("select foo, n, baz from from_json_nl")) + + +def test_insert_analyze(db_path): + db = Database(db_path) + db["rows"].insert({"foo": "x", "n": 3}) + db["rows"].create_index(["n"]) + assert "sqlite_stat1" not in db.table_names() + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "rows", "-", "--nl", "--analyze"], + input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + ) + assert result.exit_code == 0, result.output + assert "sqlite_stat1" in db.table_names() + + +def test_insert_lines(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_lines", "-", "--lines"], + input='First line\nSecond line\n{"foo": "baz"}', + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert [ + {"line": "First line"}, + {"line": "Second line"}, + {"line": '{"foo": "baz"}'}, + ] == list(db.query("select line from from_lines")) + + +def test_insert_text(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_text", "-", "--text"], + input='First line\nSecond line\n{"foo": "baz"}', + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert [{"text": 'First line\nSecond line\n{"foo": "baz"}'}] == list( + db.query("select text from from_text") + ) + + +@pytest.mark.parametrize( + "options,input", + ( + ([], '[{"id": "1", "name": "Bob"}, {"id": "2", "name": "Cat"}]'), + (["--csv", "--no-detect-types"], "id,name\n1,Bob\n2,Cat"), + (["--nl"], '{"id": "1", "name": "Bob"}\n{"id": "2", "name": "Cat"}'), + ), +) +def test_insert_convert_json_csv_jsonnl(db_path, options, input): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "rows", "-", "--convert", '{**row, **{"extra": 1}}'] + + options, + input=input, + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query("select id, name, extra from rows")) + assert rows == [ + {"id": "1", "name": "Bob", "extra": 1}, + {"id": "2", "name": "Cat", "extra": 1}, + ] + + +def test_insert_convert_text(db_path): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "text", + "-", + "--text", + "--convert", + '{"text": text.upper()}', + ], + input="This is text\nwill be upper now", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query('select "text" from "text"')) + assert rows == [{"text": "THIS IS TEXT\nWILL BE UPPER NOW"}] + + +def test_insert_convert_text_returning_iterator(db_path): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "text", + "-", + "--text", + "--convert", + '({"word": w} for w in text.split())', + ], + input="A bunch of words", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query('select "word" from "text"')) + assert rows == [{"word": "A"}, {"word": "bunch"}, {"word": "of"}, {"word": "words"}] + + +def test_insert_convert_lines(db_path): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "all", + "-", + "--lines", + "--convert", + '{"line": line.upper()}', + ], + input="This is text\nwill be upper now", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query('select "line" from "all"')) + assert rows == [{"line": "THIS IS TEXT"}, {"line": "WILL BE UPPER NOW"}] + + +def test_insert_convert_row_modifying_in_place(db_path): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "rows", + "-", + "--convert", + 'row["is_chicken"] = True', + ], + input='{"name": "Azi"}', + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query("select name, is_chicken from rows")) + assert rows == [{"name": "Azi", "is_chicken": 1}] + + +@pytest.mark.parametrize( + "options,expected_error", + ( + ( + ["--text", "--convert", "1"], + "Error: --convert must return dict or iterator\n", + ), + (["--convert", "1"], "Error: Rows must all be dictionaries, got: 1\n"), + ), +) +def test_insert_convert_error_messages(db_path, options, expected_error): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "rows", + "-", + ] + + options, + input='{"name": "Azi"}', + ) + assert result.exit_code == 1 + assert result.output == expected_error + + +def test_insert_streaming_batch_size_1(db_path): + # https://github.com/simonw/sqlite-utils/issues/364 + # Streaming with --batch-size 1 should commit on each record + # Can't use CliRunner().invoke() here bacuse we need to + # run assertions in between writing to process stdin + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "sqlite_utils", + "insert", + db_path, + "rows", + "-", + "--nl", + "--batch-size", + "1", + ], + stdin=subprocess.PIPE, + stdout=sys.stdout, + ) + proc.stdin.write(b'{"name": "Azi"}\n') + proc.stdin.flush() + + def try_until(expected): + tries = 0 + while True: + rows = list(Database(db_path)["rows"].rows) + if rows == expected: + return + tries += 1 + if tries > 10: + assert False, "Expected {}, got {}".format(expected, rows) + time.sleep(tries * 0.1) + + try_until([{"name": "Azi"}]) + proc.stdin.write(b'{"name": "Suna"}\n') + proc.stdin.flush() + try_until([{"name": "Azi"}, {"name": "Suna"}]) + proc.stdin.close() + proc.wait() + assert proc.returncode == 0 + + +def test_insert_csv_headers_only(tmpdir): + """Test that CSV with only header row (no data) works with --detect-types (issue #702)""" + db_path = str(tmpdir / "test.db") + csv_path = str(tmpdir / "headers_only.csv") + with open(csv_path, "w") as fp: + fp.write("id,name,age\n") + # Should not crash with --detect-types (which is now the default) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "data", csv_path, "--csv"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + # Table should not exist since there were no data rows + db = Database(db_path) + assert not db["data"].exists() + + +def test_insert_into_view_errors(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["t"].insert({"id": 1}) + db.create_view("v", "select * from t") + db.close() + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "v", "-"], input='{"id": 2}' + ) + assert result.exit_code == 1 + assert result.output.strip() == "Error: Table v is actually a view" + + +def test_insert_csv_detect_types_leaves_existing_table_alone(db_path): + # Type detection is the default for CSV/TSV inserts, but it must only + # apply to tables created by this command - transforming a pre-existing + # table would rewrite its column types and corrupt data such as + # TEXT zip codes with leading zeros + db = Database(db_path) + db["places"].insert({"name": "Boston", "zip": "01234"}) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "places", "-", "--csv"], + catch_exceptions=False, + input="name,zip\nSF,94107", + ) + assert result.exit_code == 0, result.output + assert db["places"].columns_dict["zip"] is str + assert list(db["places"].rows) == [ + {"name": "Boston", "zip": "01234"}, + {"name": "SF", "zip": "94107"}, + ] + + +def test_insert_csv_detect_types_new_table(db_path): + # A table created by the insert still gets detected types + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "data", "-", "--csv"], + catch_exceptions=False, + input="name,age,weight\nCleo,5,12.5", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert db["data"].columns_dict == {"name": str, "age": int, "weight": float} + + +@pytest.mark.parametrize( + "command,extra_args,input_text,expected_row", + ( + ( + "insert", + [], + "zipcode,score\n01234,9.5\n", + {"zipcode": "01234", "score": 9.5}, + ), + ( + "upsert", + ["--pk", "id"], + "id,zipcode,score\n1,01234,9.5\n", + {"id": 1, "zipcode": "01234", "score": 9.5}, + ), + ), +) +def test_insert_upsert_csv_type_overrides_detected_types( + db_path, command, extra_args, input_text, expected_row +): + result = CliRunner().invoke( + cli.cli, + [ + command, + db_path, + "places", + "-", + "--csv", + ] + + extra_args + + [ + "--type", + "zipcode", + "text", + ], + catch_exceptions=False, + input=input_text, + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + expected_columns = {"zipcode": str, "score": float} + if command == "upsert": + expected_columns = {"id": int, **expected_columns} + assert db["places"].columns_dict == expected_columns + assert list(db["places"].rows) == [expected_row] + + +def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): + db = Database(db_path) + db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id") + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "places", "-", "--csv", "--pk", "id"], + catch_exceptions=False, + input="id,name,zip\n2,SF,94107", + ) + assert result.exit_code == 0, result.output + assert db["places"].columns_dict["zip"] is str + assert db["places"].get(1)["zip"] == "01234" + + +def test_insert_invalid_pk_clean_error(db_path): + # An invalid --pk against an existing table should be a clean CLI + # error, not a raw InvalidColumns traceback + db = Database(db_path) + db["t"].insert({"a": 1}) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "t", "-", "--pk", "badcol"], + input='{"a": 2}', + ) + assert result.exit_code == 1 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.startswith("Error: Invalid primary key column") + + +# --code tests, see https://github.com/simonw/sqlite-utils/issues/684 +CODE_ROWS_FUNCTION = """ +def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} +""" + +CODE_ROWS_ITERABLE = """ +rows = [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, +] +""" + + +@pytest.mark.parametrize("code", (CODE_ROWS_FUNCTION, CODE_ROWS_ITERABLE)) +def test_insert_code(tmpdir, code): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", code, "--pk", "id"], + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert db["creatures"].pks == ["id"] + assert list(db["creatures"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_insert_code_from_file(tmpdir): + db_path = str(tmpdir / "dogs.db") + code_path = str(tmpdir / "gen.py") + with open(code_path, "w") as fp: + fp.write(CODE_ROWS_FUNCTION) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", code_path], + ) + assert result.exit_code == 0, result.output + assert list(Database(db_path)["creatures"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_upsert_code(tmpdir): + db_path = str(tmpdir / "dogs.db") + db = Database(db_path) + db["creatures"].insert_all( + [{"id": 1, "name": "old"}, {"id": 2, "name": "Suna"}], pk="id" + ) + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--pk", "id"], + ) + assert result.exit_code == 0, result.output + assert list(db["creatures"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_insert_code_requires_file_or_code(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke(cli.cli, ["insert", db_path, "creatures"]) + assert result.exit_code == 1 + assert "Provide either a FILE argument or --code" in result.output + + +def test_insert_code_mutually_exclusive_with_file(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "-", "--code", CODE_ROWS_FUNCTION], + input="{}", + ) + assert result.exit_code == 1 + assert "--code cannot be used with a FILE argument" in result.output + + +def test_insert_code_rejects_input_format_options(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--csv"], + ) + assert result.exit_code == 1 + assert "--code cannot be used with input format options" in result.output + + +def test_insert_code_missing_rows(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "x = 1"], + ) + assert result.exit_code == 1 + assert "must define a 'rows' function or iterable" in result.output + + +def test_insert_code_single_dict(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "creatures", + "--code", + 'rows = {"id": 1, "name": "Cleo"}', + "--pk", + "id", + ], + ) + assert result.exit_code == 0, result.output + assert list(Database(db_path)["creatures"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_insert_code_not_iterable(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "rows = 5"], + ) + assert result.exit_code == 1 + assert "must define a 'rows' function or iterable" in result.output + + +def test_insert_code_syntax_error(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "def rows(:"], + ) + assert result.exit_code == 1 + assert "Error in --code" in result.output + + +def test_insert_code_file_not_found(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "missing.py"], + ) + assert result.exit_code == 1 + assert "File not found: missing.py" in result.output diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index e465927..2ed4aaa 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -1,5 +1,5 @@ +import click import json - import pytest from click.testing import CliRunner @@ -24,7 +24,8 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): sql_from = "stdin" else: csv_path = str(tmpdir / "test.csv") - open(csv_path, "w").write(content) + with open(csv_path, "w") as fp: + fp.write(content) result = CliRunner().invoke( cli.cli, ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], @@ -46,7 +47,8 @@ def test_memory_tsv(tmpdir, use_stdin): else: input = None path = str(tmpdir / "chickens.tsv") - open(path, "w").write(data) + with open(path, "w") as fp: + fp.write(data) path = path + ":tsv" sql_from = "chickens" result = CliRunner().invoke( @@ -71,7 +73,8 @@ def test_memory_json(tmpdir, use_stdin): else: input = None path = str(tmpdir / "chickens.json") - open(path, "w").write(data) + with open(path, "w") as fp: + fp.write(data) path = path + ":json" sql_from = "chickens" result = CliRunner().invoke( @@ -96,7 +99,8 @@ def test_memory_json_nl(tmpdir, use_stdin): else: input = None path = str(tmpdir / "chickens.json") - open(path, "w").write(data) + with open(path, "w") as fp: + fp.write(data) path = path + ":nl" sql_from = "chickens" result = CliRunner().invoke( @@ -152,6 +156,24 @@ def test_memory_csv_encoding(tmpdir, use_stdin): } +def test_memory_csv_headers_only(tmpdir): + csv_path = str(tmpdir / "headers_only.csv") + with open(csv_path, "w") as fp: + fp.write("id,name,age\n") + + result = CliRunner().invoke( + cli.cli, + ["memory", csv_path, "", "--schema"], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output.strip() == ( + 'CREATE VIEW "t1" AS select * from "headers_only";\n' + 'CREATE VIEW "t" AS select * from "headers_only";' + ) + + @pytest.mark.parametrize("extra_args", ([], ["select 1"])) def test_memory_dump(extra_args): result = CliRunner().invoke( @@ -160,18 +182,21 @@ def test_memory_dump(extra_args): input="id,name\n1,Cleo\n2,Bants", ) assert result.exit_code == 0 - assert result.output.strip() == ( + expected = ( "BEGIN TRANSACTION;\n" - 'CREATE TABLE "stdin" (\n' - " [id] INTEGER,\n" - " [name] TEXT\n" + 'CREATE TABLE IF NOT EXISTS "stdin" (\n' + ' "id" INTEGER,\n' + ' "name" TEXT\n' ");\n" "INSERT INTO \"stdin\" VALUES(1,'Cleo');\n" "INSERT INTO \"stdin\" VALUES(2,'Bants');\n" - "CREATE VIEW t1 AS select * from [stdin];\n" - "CREATE VIEW t AS select * from [stdin];\n" + 'CREATE VIEW "t1" AS select * from "stdin";\n' + 'CREATE VIEW "t" AS select * from "stdin";\n' "COMMIT;" ) + # Using sqlite-dump it won't have IF NOT EXISTS + expected_alternative = expected.replace("IF NOT EXISTS ", "") + assert result.output.strip() in (expected, expected_alternative) @pytest.mark.parametrize("extra_args", ([], ["select 1"])) @@ -184,11 +209,11 @@ def test_memory_schema(extra_args): assert result.exit_code == 0 assert result.output.strip() == ( 'CREATE TABLE "stdin" (\n' - " [id] INTEGER,\n" - " [name] TEXT\n" + ' "id" INTEGER,\n' + ' "name" TEXT\n' ");\n" - "CREATE VIEW t1 AS select * from [stdin];\n" - "CREATE VIEW t AS select * from [stdin];" + 'CREATE VIEW "t1" AS select * from "stdin";\n' + 'CREATE VIEW "t" AS select * from "stdin";' ) @@ -220,3 +245,111 @@ def test_memory_no_detect_types(option): {"id": "1", "name": "Cleo", "weight": "45.5"}, {"id": "2", "name": "Bants", "weight": "3.5"}, ] + + +def test_memory_flatten(): + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin", "--flatten"], + input=json.dumps( + { + "httpRequest": { + "latency": "0.112114537s", + "requestMethod": "GET", + }, + "insertId": "6111722f000b5b4c4d4071e2", + } + ), + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + { + "httpRequest_latency": "0.112114537s", + "httpRequest_requestMethod": "GET", + "insertId": "6111722f000b5b4c4d4071e2", + } + ] + + +def test_memory_analyze(): + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "--analyze"], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + assert result.output == ( + "stdin.id: (1/2)\n\n" + " Total rows: 2\n" + " Null rows: 0\n" + " Blank rows: 0\n\n" + " Distinct values: 2\n\n" + "stdin.name: (2/2)\n\n" + " Total rows: 2\n" + " Null rows: 0\n" + " Blank rows: 0\n\n" + " Distinct values: 2\n\n" + ) + + +def test_memory_two_files_with_same_stem(tmpdir): + (tmpdir / "one").mkdir() + (tmpdir / "two").mkdir() + one = tmpdir / "one" / "data.csv" + two = tmpdir / "two" / "data.csv" + one.write_text("id,name\n1,Cleo\n2,Bants", encoding="utf-8") + two.write_text("id,name\n3,Blue\n4,Lila", encoding="utf-8") + result = CliRunner().invoke(cli.cli, ["memory", str(one), str(two), "", "--schema"]) + assert result.exit_code == 0 + assert result.output == ( + 'CREATE TABLE "data" (\n' + ' "id" INTEGER,\n' + ' "name" TEXT\n' + ");\n" + 'CREATE VIEW "t1" AS select * from "data";\n' + 'CREATE VIEW "t" AS select * from "data";\n' + 'CREATE TABLE "data_2" (\n' + ' "id" INTEGER,\n' + ' "name" TEXT\n' + ");\n" + 'CREATE VIEW "t2" AS select * from "data_2";\n' + ) + + +def test_memory_functions(): + result = CliRunner().invoke( + cli.cli, + ["memory", "select hello()", "--functions", "hello = lambda: 'Hello'"], + ) + assert result.exit_code == 0 + assert result.output.strip() == '[{"hello()": "Hello"}]' + + +def test_memory_functions_multiple(): + result = CliRunner().invoke( + cli.cli, + [ + "memory", + "select triple(2), quadruple(2)", + "--functions", + "def triple(x):\n return x * 3", + "--functions", + "def quadruple(x):\n return x * 4", + ], + ) + assert result.exit_code == 0 + assert result.output.strip() == '[{"triple(2)": 6, "quadruple(2)": 8}]' + + +def test_memory_return_db(tmpdir): + # https://github.com/simonw/sqlite-utils/issues/643 + from sqlite_utils.cli import cli + + path = str(tmpdir / "dogs.csv") + with open(path, "w") as f: + f.write("id,name\n1,Cleo") + + with click.Context(cli) as ctx: # type: ignore[attr-defined] + db = ctx.invoke(cli.commands["memory"], paths=(path,), return_db=True) + + assert db.table_names() == ["dogs"] diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py new file mode 100644 index 0000000..0f29e36 --- /dev/null +++ b/tests/test_cli_migrate.py @@ -0,0 +1,507 @@ +import pathlib + +from click.testing import CliRunner +import pytest +import sqlite_utils +import sqlite_utils.cli + +TWO_MIGRATIONS = """ +from sqlite_utils import Migrations + +m = Migrations("hello") + +@m() +def foo(db): + db["foo"].insert({"hello": "world"}) + +@m() +def bar(db): + db["bar"].insert({"hello": "world"}) +""" + + +@pytest.fixture +def two_migrations(tmpdir): + path = pathlib.Path(tmpdir) + (path / "foo").mkdir() + migrations_py = path / "foo" / "migrations.py" + migrations_py.write_text(TWO_MIGRATIONS, "utf-8") + return path, migrations_py + + +@pytest.fixture +def two_sets_same_migration_name(tmpdir): + path = pathlib.Path(tmpdir) + migrations_py = path / "migrations.py" + migrations_py.write_text( + """ +from sqlite_utils import Migrations + +creatures = Migrations("creatures") + +@creatures() +def create_table(db): + db["creatures"].insert({"name": "Cleo"}) + +@creatures() +def add_weight(db): + db["creature_weights"].insert({"weight": 4.2}) + +sales = Migrations("sales") + +@sales() +def create_table(db): + db["sales"].insert({"id": 1}) + +@sales() +def add_weight(db): + db["sales_weights"].insert({"weight": 10}) +""", + "utf-8", + ) + return path, migrations_py + + +@pytest.mark.parametrize("arg", ("TMPDIR", "TMPDIR/foo/migrations.py", "TMPDIR/foo/")) +def test_basic(two_migrations, arg): + path, _ = two_migrations + db_path = str(path / "test.db") + + runner = CliRunner() + + def _list(): + list_result = runner.invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, "--list", arg.replace("TMPDIR", str(path))], + ) + assert list_result.exit_code == 0 + return list_result.output + + assert _list() == ( + "Migrations for: hello\n\n" + " Applied:\n\n" + " Pending:\n" + " foo\n" + " bar\n\n" + ) + + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, arg.replace("TMPDIR", str(path))] + ) + assert result.exit_code == 0, result.output + + list_output = _list() + assert "Migrations for: hello\n\n Applied:\n " in list_output + prior_to_pending = list_output.split(" Pending")[0] + assert " foo" in prior_to_pending + assert " bar" in prior_to_pending + assert " Pending:\n (none)" in list_output + + db = sqlite_utils.Database(db_path) + assert db["foo"].exists() + assert db["bar"].exists() + assert db["_sqlite_migrations"].exists() + rows = list(db["_sqlite_migrations"].rows) + assert len(rows) == 2 + assert rows[0]["name"] == "foo" + assert rows[1]["name"] == "bar" + + +def test_list_same_migration_names_in_different_sets(capsys): + applied = sqlite_utils.Migrations("applied") + + @applied(name="foo") + def applied_foo(db): + db["applied"].insert({"hello": "world"}) + + pending = sqlite_utils.Migrations("pending") + + @pending(name="foo") + def pending_foo(db): + db["pending"].insert({"hello": "world"}) + + db = sqlite_utils.Database(memory=True) + applied.apply(db) + + sqlite_utils.cli._display_migration_list(db, [applied, pending]) + + output = capsys.readouterr().out + assert ( + "Migrations for: pending\n\n" " Applied:\n\n" " Pending:\n" " foo\n\n" + ) in output + + +def test_verbose(tmpdir): + path = pathlib.Path(tmpdir) + (path / "foo").mkdir() + migrations_py = path / "foo" / "migrations.py" + migrations_py.write_text( + """ +from sqlite_utils import Migrations + +m = Migrations("hello") + +@m() +def foo(db): + db["dogs"].insert({"id": 1, "name": "Cleo"}) + """, + "utf-8", + ) + db_path = str(path / "test.db") + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py)] + ) + assert result.exit_code == 0 + + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"] + ) + assert result.exit_code == 0 + expected = """ +Schema before: + + CREATE TABLE "_sqlite_migrations" ( + "id" INTEGER PRIMARY KEY, + "migration_set" TEXT, + "name" TEXT, + "applied_at" TEXT + ); + CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name" + ON "_sqlite_migrations" ("migration_set", "name"); + CREATE TABLE "dogs" ( + "id" INTEGER, + "name" TEXT + ); + +Schema after: + + (unchanged) +""".strip() + assert expected in result.output + + new_migration = """ +@m() +def bar(db): + db["dogs"].add_column("age", int) + db["dogs"].add_column("weight", float) + db["dogs"].transform() +""" + migrations_py.write_text(migrations_py.read_text("utf-8") + new_migration) + + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"] + ) + assert result.exit_code == 0 + expected_diff = """ +Schema diff: + + ON "_sqlite_migrations" ("migration_set", "name"); + CREATE TABLE "dogs" ( + "id" INTEGER, +- "name" TEXT ++ "name" TEXT, ++ "age" INTEGER, ++ "weight" REAL + ); +""".strip() + assert expected_diff in result.output + + +def test_stop_before(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(path / "foo" / "migrations.py"), + "--stop-before", + "bar", + ], + ) + assert result.exit_code == 0 + db = sqlite_utils.Database(db_path) + assert db["foo"].exists() + assert not db["bar"].exists() + + +def test_stop_before_multiple_sets_unqualified(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + (path / "foo" / "migrations2.py").write_text( + """ +from sqlite_utils import Migrations + +m = Migrations("hello2") + +@m() +def foo(db): + db["foo"].insert({"hello": "world"}) + """, + "utf-8", + ) + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(path / "foo" / "migrations.py"), + str(path / "foo" / "migrations2.py"), + "--stop-before", + "foo", + ], + ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert db.table_names() == ["_sqlite_migrations"] + assert list(db["_sqlite_migrations"].rows) == [] + + +def test_stop_before_qualified_only_affects_named_set(two_sets_same_migration_name): + path, migrations_py = two_sets_same_migration_name + db_path = str(path / "test.db") + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(migrations_py), + "--stop-before", + "creatures:add_weight", + ], + ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert db["creatures"].exists() + assert not db["creature_weights"].exists() + assert db["sales"].exists() + assert db["sales_weights"].exists() + + +def test_stop_before_multiple_qualified(two_sets_same_migration_name): + path, migrations_py = two_sets_same_migration_name + db_path = str(path / "test.db") + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(migrations_py), + "--stop-before", + "creatures:add_weight", + "--stop-before", + "sales:add_weight", + ], + ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert db["creatures"].exists() + assert not db["creature_weights"].exists() + assert db["sales"].exists() + assert not db["sales_weights"].exists() + + +LEGACY_MIGRATIONS = """ +import datetime + +class _Migration: + def __init__(self, name, fn): + self.name = name + self.fn = fn + +class _Applied: + def __init__(self, name, applied_at): + self.name = name + self.applied_at = applied_at + +class LegacyMigrations: + # Mimics the sqlite-migrate 0.x Migrations class, in particular + # apply(db, stop_before=None) taking a single string + migrations_table = "_sqlite_migrations" + + def __init__(self, name): + self.name = name + self._migrations = [] + + def __call__(self, fn): + self._migrations.append(_Migration(fn.__name__, fn)) + return fn + + def ensure_migrations_table(self, db): + db[self.migrations_table].create( + {"migration_set": str, "name": str, "applied_at": str}, + pk=("migration_set", "name"), + if_not_exists=True, + ) + + def applied(self, db): + self.ensure_migrations_table(db) + return [ + _Applied(row["name"], row["applied_at"]) + for row in db[self.migrations_table].rows_where( + "migration_set = ?", [self.name] + ) + ] + + def pending(self, db): + applied = {m.name for m in self.applied(db)} + return [m for m in self._migrations if m.name not in applied] + + def apply(self, db, stop_before=None): + for migration in self.pending(db): + if migration.name == stop_before: + return + migration.fn(db) + db[self.migrations_table].insert( + { + "migration_set": self.name, + "name": migration.name, + "applied_at": str( + datetime.datetime.now(datetime.timezone.utc) + ), + } + ) + +legacy = LegacyMigrations("legacy_set") + +@legacy +def first(db): + db["first"].insert({"hello": "world"}) + +@legacy +def second(db): + db["second"].insert({"hello": "world"}) +""" + + +def test_stop_before_unknown_name_errors(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, str(path), "--stop-before", "fooo"], + ) + assert result.exit_code == 1 + assert "--stop-before did not match any migrations: fooo" in result.output + # Nothing should have been applied + db = sqlite_utils.Database(db_path) + assert "foo" not in db.table_names() + assert "bar" not in db.table_names() + + +def test_stop_before_with_legacy_migrations_class(tmpdir): + path = pathlib.Path(tmpdir) + (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") + db_path = str(path / "test.db") + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, str(path), "--stop-before", "second"], + ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert "first" in db.table_names() + assert "second" not in db.table_names() + + +def test_stop_before_multiple_values_for_legacy_set_errors(tmpdir): + path = pathlib.Path(tmpdir) + (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") + db_path = str(path / "test.db") + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(path), + "--stop-before", + "legacy_set:first", + "--stop-before", + "legacy_set:second", + ], + ) + assert result.exit_code == 1 + assert "single --stop-before" in result.output + + +def test_list_does_not_create_database_file(two_migrations): + path, _ = two_migrations + db_path = path / "test.db" + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", str(db_path), str(path), "--list"] + ) + assert result.exit_code == 0, result.output + assert "Pending:\n foo\n bar" in result.output + # Listing migrations must not create the database file + assert not db_path.exists() + + +def test_list_does_not_upgrade_legacy_migrations_table(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + db = sqlite_utils.Database(db_path) + db["_sqlite_migrations"].create( + {"migration_set": str, "name": str, "applied_at": str}, + pk=("migration_set", "name"), + ) + db["_sqlite_migrations"].insert( + {"migration_set": "hello", "name": "foo", "applied_at": "x"} + ) + db.close() + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"] + ) + assert result.exit_code == 0, result.output + assert "foo - x" in result.output + # --list must not perform the one-way legacy schema upgrade + db2 = sqlite_utils.Database(db_path) + assert db2["_sqlite_migrations"].pks == ["migration_set", "name"] + db2.close() + + +def test_stop_before_applied_migration_errors(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + migrations_path = str(path / "foo" / "migrations.py") + # Apply everything first + first = CliRunner().invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, migrations_path, "--stop-before", "bar"], + ) + assert first.exit_code == 0 + # foo is now applied - stopping before it is an error, and bar + # must not be applied as a side effect + result = CliRunner().invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, migrations_path, "--stop-before", "foo"], + ) + assert result.exit_code != 0 + assert "already been applied" in result.output + db = sqlite_utils.Database(db_path) + assert not db["bar"].exists() + + +def test_list_with_legacy_class_is_read_only(tmpdir): + # Legacy sqlite-migrate classes create the _sqlite_migrations table + # from their pending()/applied() methods - --list must roll that + # back so it stays a read-only operation as documented + path = pathlib.Path(tmpdir) + (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") + db_path = str(path / "test.db") + db = sqlite_utils.Database(db_path) + db["existing"].insert({"id": 1}) + db.close() + result = CliRunner().invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"] + ) + assert result.exit_code == 0, result.output + assert "first" in result.output + db2 = sqlite_utils.Database(db_path) + assert "_sqlite_migrations" not in db2.table_names() + db2.close() diff --git a/tests/test_column_casing.py b/tests/test_column_casing.py new file mode 100644 index 0000000..ce11345 --- /dev/null +++ b/tests/test_column_casing.py @@ -0,0 +1,233 @@ +""" +SQLite treats column names as case-insensitive. These tests exercise the +places where sqlite-utils performs Python-side lookups of column names +provided by the caller, which should match the schema case-insensitively. + +https://github.com/simonw/sqlite-utils/issues/760 +""" + +import pytest + +from sqlite_utils import Database +from sqlite_utils.db import ForeignKey + + +def test_insert_populates_last_pk_case_insensitively(fresh_db): + books = fresh_db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.insert({"Id": 1, "Title": "One"}, pk="id") + assert books.last_pk == 1 + + +def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db): + books = fresh_db["books"] + books.create({"Author": str, "Position": int, "Title": str}) + books.insert( + {"Author": "Sue", "Position": 1, "Title": "One"}, pk=("author", "position") + ) + assert books.last_pk == ("Sue", 1) + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_pk_case_differs_from_schema(use_old_upsert): + db = Database(memory=True, use_old_upsert=use_old_upsert) + books = db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.insert({"Id": 1, "Title": "One"}) + books.upsert({"id": 1, "title": "Won"}, pk="id") + assert list(books.rows) == [{"Id": 1, "Title": "Won"}] + assert books.last_pk == 1 + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_record_key_case_differs_from_pk(use_old_upsert): + # all_columns comes from the record keys, pk= from the caller + db = Database(memory=True, use_old_upsert=use_old_upsert) + books = db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.upsert({"ID": 1, "Title": "One"}, pk="id") + assert list(books.rows) == [{"Id": 1, "Title": "One"}] + assert books.last_pk == 1 + + +def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db): + # pk is inferred from the existing schema as "Id", records use "id" + books = fresh_db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.upsert({"id": 1, "title": "One"}) + assert list(books.rows) == [{"Id": 1, "Title": "One"}] + assert books.last_pk == 1 + + +def test_upsert_list_mode_pk_case_insensitive(fresh_db): + books = fresh_db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.upsert_all([["id", "title"], [1, "One"]], pk="Id") + assert list(books.rows) == [{"Id": 1, "Title": "One"}] + assert books.last_pk == 1 + + +def test_lookup_pk_case_insensitive(fresh_db): + fresh_db["species"].create({"ID": int, "Name": str}, pk="ID") + fresh_db["species"].insert({"ID": 5, "Name": "Palm"}) + fresh_db["species"].create_index(["Name"], unique=True) + assert fresh_db["species"].lookup({"Name": "Palm"}, pk="id") == 5 + + +def test_lookup_does_not_create_redundant_index(fresh_db): + fresh_db["species"].create({"id": int, "Name": str}, pk="id") + fresh_db["species"].create_index(["Name"], unique=True) + fresh_db["species"].lookup({"name": "Palm"}) + assert len(fresh_db["species"].indexes) == 1 + + +def test_create_table_transform_same_columns_different_case(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db["t"].insert({"Name": "Cleo", "Age": 5}) + fresh_db.create_table("t", {"name": str, "age": int}, transform=True) + # Schema casing is preserved - SQLite considers these the same columns + assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} + assert list(fresh_db["t"].rows) == [{"Name": "Cleo", "Age": 5}] + + +def test_create_table_transform_case_insensitive_with_changes(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db.create_table("t", {"name": str, "age": str, "size": int}, transform=True) + # age changed type, size added, Name untouched + assert fresh_db["t"].columns_dict == {"Name": str, "Age": str, "size": int} + + +def test_transform_types_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str, "Age": str}) + fresh_db["t"].transform(types={"age": int}) + assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} + + +def test_transform_rename_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str}) + fresh_db["t"].transform(rename={"name": "title"}) + assert fresh_db["t"].columns_dict == {"title": str} + + +def test_transform_drop_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db["t"].transform(drop=["name"]) + assert fresh_db["t"].columns_dict == {"Age": int} + + +def test_transform_not_null_and_defaults_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db["t"].transform(not_null={"name"}, defaults={"age": 3}) + columns = {c.name: c for c in fresh_db["t"].columns} + assert columns["Name"].notnull + assert fresh_db["t"].default_values == {"Age": 3} + + +def test_transform_pk_case_insensitive(fresh_db): + fresh_db["t"].create({"Id": int, "Name": str}) + fresh_db["t"].transform(pk="id") + assert fresh_db["t"].pks == ["Id"] + assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} + + +def test_transform_drop_foreign_keys_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create( + {"id": int, "Parent_ID": int}, + pk="id", + foreign_keys=[("Parent_ID", "parent", "Id")], + ) + fresh_db["child"].transform(drop_foreign_keys=["parent_id"]) + assert fresh_db["child"].foreign_keys == [] + + +def test_add_foreign_key_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") + fresh_db["child"].add_foreign_key("parent_id", "parent", "id") + fks = fresh_db["child"].foreign_keys + assert len(fks) == 1 + # The foreign key should use the schema casing of the columns + assert fks[0].column == "Parent_ID" + assert fks[0].other_column == "Id" + + +def test_add_foreign_keys_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") + fresh_db.add_foreign_keys([("child", "parent_id", "parent", "id")]) + fks = fresh_db["child"].foreign_keys + assert len(fks) == 1 + assert fks[0].column == "Parent_ID" + assert fks[0].other_column == "Id" + + +def test_add_foreign_key_detects_existing_case_insensitively(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create( + {"id": int, "Parent_ID": int}, + pk="id", + foreign_keys=[("Parent_ID", "parent", "Id")], + ) + # ignore=True should treat this as already existing, not add a duplicate + fresh_db["child"].add_foreign_key("parent_id", "parent", "id", ignore=True) + assert len(fresh_db["child"].foreign_keys) == 1 + + +def test_add_column_fk_col_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create({"id": int}, pk="id") + fresh_db["child"].add_column("parent_id", int, fk="parent", fk_col="id") + fks = fresh_db["child"].foreign_keys + assert len(fks) == 1 + assert fks[0].other_column == "Id" + + +def test_extract_case_insensitive(fresh_db): + fresh_db["trees"].insert({"id": 1, "Species": "Palm"}, pk="id") + fresh_db["trees"].extract("species") + assert fresh_db["trees"].columns_dict == {"id": int, "Species_id": int} + assert list(fresh_db["Species"].rows) == [{"id": 1, "Species": "Palm"}] + + +def test_convert_multi_case_insensitive(fresh_db): + fresh_db["t"].insert({"id": 1, "Name": "Cleo"}, pk="id") + fresh_db["t"].convert("name", lambda v: {"upper": v.upper()}, multi=True) + assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "upper": "CLEO"}] + + +def test_convert_output_case_insensitive(fresh_db): + fresh_db["t"].insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id") + fresh_db["t"].convert("name", lambda v: v.upper(), output="upper") + assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "Upper": "CLEO"}] + + +def test_create_table_sql_pk_case_insensitive(fresh_db): + fresh_db["t"].create({"Id": int, "Name": str}, pk="id") + # Should not have created an extra lowercase "id" column + assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} + assert fresh_db["t"].pks == ["Id"] + + +def test_create_table_not_null_and_defaults_case_insensitive(fresh_db): + fresh_db["t"].create( + {"Name": str, "Age": int}, not_null={"name"}, defaults={"age": 1} + ) + columns = {c.name: c for c in fresh_db["t"].columns} + assert columns["Name"].notnull + assert fresh_db["t"].default_values == {"Age": 1} + + +def test_create_table_foreign_keys_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create( + {"id": int, "Parent_ID": int}, + pk="id", + foreign_keys=[("parent_id", "parent", "id")], + ) + fks = fresh_db["child"].foreign_keys + assert fks == [ + ForeignKey( + table="child", column="Parent_ID", other_table="parent", other_column="Id" + ) + ] diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 924df66..a619fba 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -1,4 +1,8 @@ from sqlite_utils import Database +from sqlite_utils.db import TransactionError +from sqlite_utils.utils import sqlite3 +import pytest +import sys def test_recursive_triggers(): @@ -9,3 +13,101 @@ def test_recursive_triggers(): def test_recursive_triggers_off(): db = Database(memory=True, recursive_triggers=False) assert not db.execute("PRAGMA recursive_triggers").fetchone()[0] + + +def test_memory_name(): + db1 = Database(memory_name="shared") + db2 = Database(memory_name="shared") + db1["dogs"].insert({"name": "Cleo"}) + assert list(db2["dogs"].rows) == [{"name": "Cleo"}] + + +def test_sqlite_version(): + db = Database(memory=True) + version = db.sqlite_version + assert isinstance(version, tuple) + as_string = ".".join(map(str, version)) + actual = next(db.query("select sqlite_version() as v"))["v"] + assert actual == as_string + + +def test_database_context_manager(tmpdir): + path = str(tmpdir / "test.db") + with Database(path) as db: + db["t"].insert({"id": 1}) + # Raw writes commit automatically too + db.execute("insert into t (id) values (2)") + # An explicitly opened transaction left uncommitted on purpose: + db.begin() + db.execute("insert into t (id) values (3)") + # The connection is closed... + with pytest.raises(sqlite3.ProgrammingError): + db.execute("select 1") + # ... and the open explicit transaction was rolled back, not committed + db2 = Database(path) + assert [r["id"] for r in db2["t"].rows] == [1, 2] + db2.close() + + +@pytest.mark.parametrize("memory", [True, False]) +def test_database_close(tmpdir, memory): + if memory: + db = Database(memory=True) + else: + db = Database(str(tmpdir / "test.db")) + assert db.execute("select 1 + 1").fetchone()[0] == 2 + db.close() + with pytest.raises(sqlite3.ProgrammingError): + db.execute("select 1 + 1") + + +@pytest.mark.skipif( + sys.version_info < (3, 12), + reason="sqlite3.connect(autocommit=) requires Python 3.12", +) +@pytest.mark.parametrize("autocommit", [True, False]) +def test_autocommit_connections_are_rejected(tmpdir, autocommit): + # These connection modes break commit()/rollback() in ways that + # silently lose data, so the constructor refuses them + conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=autocommit) + with pytest.raises(TransactionError): + Database(conn) + conn.close() + + +@pytest.mark.skipif( + sys.version_info < (3, 12), + reason="sqlite3.LEGACY_TRANSACTION_CONTROL requires Python 3.12", +) +def test_legacy_transaction_control_connection_is_accepted(tmpdir): + conn = sqlite3.connect( + str(tmpdir / "test.db"), autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL + ) + db = Database(conn) + db["t"].insert({"id": 1}, pk="id") + assert [r["id"] for r in db["t"].rows] == [1] + db.close() + + +def test_memory_attribute_for_memory_true(): + db = Database(memory=True) + assert db.memory is True + assert db.memory_name is None + + +def test_memory_attribute_for_memory_name(): + db = Database(memory_name="shared_attr") + assert db.memory is True + assert db.memory_name == "shared_attr" + + +def test_memory_attribute_for_memory_string_path(): + db = Database(":memory:") + assert db.memory is True + assert db.memory_name is None + + +def test_memory_attribute_for_file_path(tmpdir): + db = Database(str(tmpdir / "file.db")) + assert db.memory is False + assert db.memory_name is None diff --git a/tests/test_convert.py b/tests/test_convert.py index 796a08f..ea3fd96 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -15,6 +15,14 @@ import pytest lambda value: value.upper(), {"title": "MIXED CASE", "abstract": "ABSTRACT"}, ), + ( + "title", + lambda value: {"upper": value.upper(), "lower": value.lower()}, + { + "title": '{"upper": "MIXED CASE", "lower": "mixed case"}', + "abstract": "Abstract", + }, + ), ), ) def test_convert(fresh_db, columns, fn, expected): @@ -42,6 +50,17 @@ def test_convert_where(fresh_db, where, where_args): assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}] +def test_convert_handles_falsey_values(fresh_db): + # Falsey values like 0 should be converted (issue #527) + table = fresh_db["table"] + table.insert_all([{"x": 0}, {"x": 1}]) + assert table.get(1)["x"] == 0 + assert table.get(2)["x"] == 1 + table.convert("x", lambda x: x + 1) + assert table.get(1)["x"] == 1 + assert table.get(2)["x"] == 2 + + @pytest.mark.parametrize( "drop,expected", ( @@ -58,7 +77,7 @@ def test_convert_output(fresh_db, drop, expected): def test_convert_output_multiple_column_error(fresh_db): table = fresh_db["table"] - with pytest.raises(AssertionError) as excinfo: + with pytest.raises(ValueError) as excinfo: table.convert(["title", "other"], lambda v: v, output="out") assert "output= can only be used with a single column" in str(excinfo.value) @@ -81,10 +100,24 @@ def test_convert_multi(fresh_db): table = fresh_db["table"] table.insert({"title": "Mixed Case"}) table.convert( - "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True + "title", + lambda v: { + "upper": v.upper(), + "lower": v.lower(), + "both": { + "upper": v.upper(), + "lower": v.lower(), + }, + }, + multi=True, ) assert list(table.rows) == [ - {"title": "Mixed Case", "upper": "MIXED CASE", "lower": "mixed case"} + { + "title": "Mixed Case", + "upper": "MIXED CASE", + "lower": "mixed case", + "both": '{"upper": "MIXED CASE", "lower": "mixed case"}', + } ] @@ -115,3 +148,12 @@ def test_convert_multi_exception(fresh_db): table.insert({"title": "Mixed Case"}) with pytest.raises(BadMultiValues): table.convert("title", lambda v: v.upper(), multi=True) + + +def test_convert_repeated(fresh_db): + table = fresh_db["table"] + col = "num" + table.insert({col: 1}) + table.convert(col, lambda x: x * 2) + table.convert(col, lambda _x: 0) + assert table.get(1) == {col: 0} diff --git a/tests/test_create.py b/tests/test_create.py index ff36f90..d281eb4 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -3,12 +3,16 @@ from sqlite_utils.db import ( Database, DescIndex, AlterError, + InvalidColumns, NoObviousTable, + OperationalError, ForeignKey, Table, View, + NoTable, + NoView, ) -from sqlite_utils.utils import sqlite3 +from sqlite_utils.utils import hash_record, sqlite3 import collections import datetime import decimal @@ -17,12 +21,10 @@ import pathlib import pytest import uuid -from .utils import collapse_whitespace - try: - import pandas as pd + import pandas as pd # type: ignore except ImportError: - pd = None + pd = None # type: ignore def test_create_table(fresh_db): @@ -41,20 +43,20 @@ def test_create_table(fresh_db): assert ["test_table"] == fresh_db.table_names() assert [ {"name": "text_col", "type": "TEXT"}, - {"name": "float_col", "type": "FLOAT"}, + {"name": "float_col", "type": "REAL"}, {"name": "int_col", "type": "INTEGER"}, {"name": "bool_col", "type": "INTEGER"}, {"name": "bytes_col", "type": "BLOB"}, {"name": "datetime_col", "type": "TEXT"}, ] == [{"name": col.name, "type": col.type} for col in table.columns] assert ( - "CREATE TABLE [test_table] (\n" - " [text_col] TEXT,\n" - " [float_col] FLOAT,\n" - " [int_col] INTEGER,\n" - " [bool_col] INTEGER,\n" - " [bytes_col] BLOB,\n" - " [datetime_col] TEXT\n" + 'CREATE TABLE "test_table" (\n' + ' "text_col" TEXT,\n' + ' "float_col" REAL,\n' + ' "int_col" INTEGER,\n' + ' "bool_col" INTEGER,\n' + ' "bytes_col" BLOB,\n' + ' "datetime_col" TEXT\n' ")" ) == table.schema @@ -64,11 +66,11 @@ def test_create_table_compound_primary_key(fresh_db): "test_table", {"id1": str, "id2": str, "value": int}, pk=("id1", "id2") ) assert ( - "CREATE TABLE [test_table] (\n" - " [id1] TEXT,\n" - " [id2] TEXT,\n" - " [value] INTEGER,\n" - " PRIMARY KEY ([id1], [id2])\n" + 'CREATE TABLE "test_table" (\n' + ' "id1" TEXT,\n' + ' "id2" TEXT,\n' + ' "value" INTEGER,\n' + ' PRIMARY KEY ("id1", "id2")\n' ")" ) == table.schema assert ["id1", "id2"] == table.pks @@ -78,13 +80,17 @@ def test_create_table_compound_primary_key(fresh_db): def test_create_table_with_single_primary_key(fresh_db, pk): fresh_db["foo"].insert({"id": 1}, pk=pk) assert ( - fresh_db["foo"].schema == "CREATE TABLE [foo] (\n [id] INTEGER PRIMARY KEY\n)" + fresh_db["foo"].schema == 'CREATE TABLE "foo" (\n "id" INTEGER PRIMARY KEY\n)' ) -def test_create_table_with_invalid_column_characters(fresh_db): - with pytest.raises(AssertionError): - fresh_db.create_table("players", {"name[foo]": str}) +def test_create_table_with_special_column_characters(fresh_db): + # With double-quote escaping, columns with special characters are now valid + table = fresh_db.create_table("players", {"name[foo]": str}) + assert ["players"] == fresh_db.table_names() + assert [{"name": "name[foo]", "type": "TEXT"}] == [ + {"name": col.name, "type": col.type} for col in table.columns + ] def test_create_table_with_defaults(fresh_db): @@ -98,12 +104,12 @@ def test_create_table_with_defaults(fresh_db): {"name": col.name, "type": col.type} for col in table.columns ] assert ( - "CREATE TABLE [players] (\n [name] TEXT DEFAULT 'bob''''bob',\n [score] INTEGER DEFAULT 1\n)" + "CREATE TABLE \"players\" (\n \"name\" TEXT DEFAULT 'bob''''bob',\n \"score\" INTEGER DEFAULT 1\n)" ) == table.schema def test_create_table_with_bad_not_null(fresh_db): - with pytest.raises(AssertionError): + with pytest.raises(ValueError): fresh_db.create_table( "players", {"name": str, "score": int}, not_null={"mouse"} ) @@ -121,7 +127,7 @@ def test_create_table_with_not_null(fresh_db): {"name": col.name, "type": col.type} for col in table.columns ] assert ( - "CREATE TABLE [players] (\n [name] TEXT NOT NULL,\n [score] INTEGER NOT NULL DEFAULT 3\n)" + 'CREATE TABLE "players" (\n "name" TEXT NOT NULL,\n "score" INTEGER NOT NULL DEFAULT 3\n)' ) == table.schema @@ -137,12 +143,17 @@ def test_create_table_with_not_null(fresh_db): [{"name": "create", "type": "TEXT"}, {"name": "table", "type": "TEXT"}], ), ({"day": datetime.time(11, 0)}, [{"name": "day", "type": "TEXT"}]), - ({"decimal": decimal.Decimal("1.2")}, [{"name": "decimal", "type": "FLOAT"}]), + ({"decimal": decimal.Decimal("1.2")}, [{"name": "decimal", "type": "REAL"}]), ( {"memoryview": memoryview(b"hello")}, [{"name": "memoryview", "type": "BLOB"}], ), ({"uuid": uuid.uuid4()}, [{"name": "uuid", "type": "TEXT"}]), + ({"foo[bar]": 1}, [{"name": "foo[bar]", "type": "INTEGER"}]), + ( + {"timedelta": datetime.timedelta(hours=1)}, + [{"name": "timedelta", "type": "TEXT"}], + ), ), ) def test_create_table_from_example(fresh_db, example, expected_columns): @@ -150,8 +161,8 @@ def test_create_table_from_example(fresh_db, example, expected_columns): assert people_table.last_rowid is None assert people_table.last_pk is None people_table.insert(example) - assert 1 == people_table.last_rowid - assert 1 == people_table.last_pk + assert people_table.last_rowid == 1 + assert people_table.last_pk == 1 assert ["people"] == fresh_db.table_names() assert expected_columns == [ {"name": col.name, "type": col.type} for col in fresh_db["people"].columns @@ -168,19 +179,21 @@ def test_create_table_from_example_with_compound_primary_keys(fresh_db): @pytest.mark.parametrize( "method_name", ("insert", "upsert", "insert_all", "upsert_all") ) -def test_create_table_with_custom_columns(fresh_db, method_name): - table = fresh_db["dogs"] +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_create_table_with_custom_columns(method_name, use_old_upsert): + db = Database(memory=True, use_old_upsert=use_old_upsert) + table = db["dogs"] method = getattr(table, method_name) record = {"id": 1, "name": "Cleo", "age": "5"} if method_name.endswith("_all"): record = [record] method(record, pk="id", columns={"age": int, "weight": float}) - assert ["dogs"] == fresh_db.table_names() + assert ["dogs"] == db.table_names() expected_columns = [ {"name": "id", "type": "INTEGER"}, {"name": "name", "type": "TEXT"}, {"name": "age", "type": "INTEGER"}, - {"name": "weight", "type": "FLOAT"}, + {"name": "weight", "type": "REAL"}, ] assert expected_columns == [ {"name": col.name, "type": col.type} for col in table.columns @@ -231,11 +244,11 @@ def test_create_table_column_order(fresh_db, use_table_factory): # If you specify a column that doesn't point to a table, you get an error: (("one_id", "two_id", "three_id"), NoObviousTable), # Tuples of the wrong length get an error: - ((("one_id", "one", "id", "five"), ("two_id", "two", "id")), AssertionError), + ((("one_id", "one", "id", "five"), ("two_id", "two", "id")), ValueError), # Likewise a bad column: ((("one_id", "one", "id2"),), AlterError), # Or a list of dicts - (({"one_id": "one"},), AssertionError), + (({"one_id": "one"},), ValueError), ), ) @pytest.mark.parametrize("use_table_factory", [True, False]) @@ -286,6 +299,25 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( ) +def test_self_referential_foreign_key(fresh_db): + assert [] == fresh_db.table_names() + table = fresh_db.create_table( + "test_table", + columns={ + "id": int, + "ref": int, + }, + pk="id", + foreign_keys=(("ref", "test_table", "id"),), + ) + assert ( + 'CREATE TABLE "test_table" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "ref" INTEGER REFERENCES "test_table"("id")\n' + ")" + ) == table.schema + + def test_create_error_if_invalid_foreign_keys(fresh_db): with pytest.raises(AlterError): fresh_db["one"].insert( @@ -295,49 +327,77 @@ def test_create_error_if_invalid_foreign_keys(fresh_db): ) +def test_create_error_if_invalid_self_referential_foreign_keys(fresh_db): + with pytest.raises(AlterError) as ex: + fresh_db["one"].insert( + {"id": 1, "ref_id": 3}, + pk="id", + foreign_keys=(("ref_id", "one", "bad_column"),), + ) + assert ex.value.args == ("No such column: one.bad_column",) + + @pytest.mark.parametrize( "col_name,col_type,not_null_default,expected_schema", ( - ("nickname", str, None, "CREATE TABLE [dogs] ( [name] TEXT , [nickname] TEXT)"), - ("dob", datetime.date, None, "CREATE TABLE [dogs] ( [name] TEXT , [dob] TEXT)"), - ("age", int, None, "CREATE TABLE [dogs] ( [name] TEXT , [age] INTEGER)"), - ("weight", float, None, "CREATE TABLE [dogs] ( [name] TEXT , [weight] FLOAT)"), - ("text", "TEXT", None, "CREATE TABLE [dogs] ( [name] TEXT , [text] TEXT)"), + ( + "nickname", + str, + None, + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "nickname" TEXT)', + ), + ( + "dob", + datetime.date, + None, + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "dob" TEXT)', + ), + ("age", int, None, 'CREATE TABLE "dogs" (\n "name" TEXT\n, "age" INTEGER)'), + ( + "weight", + float, + None, + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "weight" REAL)', + ), + ("text", "TEXT", None, 'CREATE TABLE "dogs" (\n "name" TEXT\n, "text" TEXT)'), ( "integer", "INTEGER", None, - "CREATE TABLE [dogs] ( [name] TEXT , [integer] INTEGER)", + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "integer" INTEGER)', ), - ("float", "FLOAT", None, "CREATE TABLE [dogs] ( [name] TEXT , [float] FLOAT)"), - ("blob", "blob", None, "CREATE TABLE [dogs] ( [name] TEXT , [blob] BLOB)"), + ( + "float", + "FLOAT", + None, + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "float" FLOAT)', + ), + ("blob", "blob", None, 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), ( "default_str", None, None, - "CREATE TABLE [dogs] ( [name] TEXT , [default_str] TEXT)", + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "default_str" TEXT)', ), ( "nickname", str, "", - "CREATE TABLE [dogs] ( [name] TEXT , [nickname] TEXT NOT NULL DEFAULT '')", + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "nickname" TEXT NOT NULL DEFAULT \'\')', ), ( "nickname", str, "dawg's dawg", - "CREATE TABLE [dogs] ( [name] TEXT , [nickname] TEXT NOT NULL DEFAULT 'dawg''s dawg')", + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "nickname" TEXT NOT NULL DEFAULT \'dawg\'\'s dawg\')', ), ), ) def test_add_column(fresh_db, col_name, col_type, not_null_default, expected_schema): fresh_db.create_table("dogs", {"name": str}) - assert "CREATE TABLE [dogs] ( [name] TEXT )" == collapse_whitespace( - fresh_db["dogs"].schema - ) + assert fresh_db["dogs"].schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' fresh_db["dogs"].add_column(col_name, col_type, not_null_default=not_null_default) - assert expected_schema == collapse_whitespace(fresh_db["dogs"].schema) + assert fresh_db["dogs"].schema == expected_schema def test_add_foreign_key(fresh_db): @@ -438,18 +498,21 @@ def test_add_column_foreign_key(fresh_db): fresh_db.create_table("dogs", {"name": str}) fresh_db.create_table("breeds", {"name": str}) fresh_db["dogs"].add_column("breed_id", fk="breeds") - assert ( - "CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY([breed_id]) REFERENCES [breeds]([rowid]) )" - == collapse_whitespace(fresh_db["dogs"].schema) + assert fresh_db["dogs"].schema == ( + 'CREATE TABLE "dogs" (\n' + ' "name" TEXT,\n' + ' "breed_id" INTEGER REFERENCES "breeds"("rowid")\n' + ")" ) # And again with an explicit primary key column fresh_db.create_table("subbreeds", {"name": str, "primkey": str}, pk="primkey") fresh_db["dogs"].add_column("subbreed_id", fk="subbreeds") - assert ( - "CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, [subbreed_id] TEXT, " - "FOREIGN KEY([breed_id]) REFERENCES [breeds]([rowid]), " - "FOREIGN KEY([subbreed_id]) REFERENCES [subbreeds]([primkey]) )" - == collapse_whitespace(fresh_db["dogs"].schema) + assert fresh_db["dogs"].schema == ( + 'CREATE TABLE "dogs" (\n' + ' "name" TEXT,\n' + ' "breed_id" INTEGER REFERENCES "breeds"("rowid"),\n' + ' "subbreed_id" TEXT REFERENCES "subbreeds"("primkey")\n' + ")" ) @@ -458,9 +521,11 @@ def test_add_foreign_key_guess_table(fresh_db): fresh_db.create_table("breeds", {"name": str, "id": int}, pk="id") fresh_db["dogs"].add_column("breed_id", int) fresh_db["dogs"].add_foreign_key("breed_id") - assert ( - "CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY([breed_id]) REFERENCES [breeds]([id]) )" - == collapse_whitespace(fresh_db["dogs"].schema) + assert fresh_db["dogs"].schema == ( + 'CREATE TABLE "dogs" (\n' + ' "name" TEXT,\n' + ' "breed_id" INTEGER REFERENCES "breeds"("id")\n' + ")" ) @@ -474,6 +539,18 @@ def test_index_foreign_keys(fresh_db): assert [["breed_id"]] == [i.columns for i in fresh_db["dogs"].indexes] +def test_index_foreign_keys_if_index_name_is_already_used(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/335 + test_add_foreign_key_guess_table(fresh_db) + # Add index with a name that will conflict with index_foreign_keys() + fresh_db["dogs"].create_index(["name"], index_name="idx_dogs_breed_id") + fresh_db.index_foreign_keys() + assert {(idx.name, tuple(idx.columns)) for idx in fresh_db["dogs"].indexes} == { + ("idx_dogs_breed_id_2", ("breed_id",)), + ("idx_dogs_breed_id", ("name",)), + } + + @pytest.mark.parametrize( "extra_data,expected_new_columns", [ @@ -484,7 +561,7 @@ def test_index_foreign_keys(fresh_db): ), ( {"hats": 5, "rating": 3.5}, - [{"name": "hats", "type": "INTEGER"}, {"name": "rating", "type": "FLOAT"}], + [{"name": "hats", "type": "INTEGER"}, {"name": "rating", "type": "REAL"}], ), ], ) @@ -512,20 +589,13 @@ def test_insert_row_alter_table( ] -def test_insert_row_alter_table_invalid_column_characters(fresh_db): - table = fresh_db["table"] - table.insert({"foo": "bar"}).last_pk - with pytest.raises(AssertionError): - table.insert({"foo": "baz", "new_col[abc]": 1.2}, alter=True) - - def test_add_missing_columns_case_insensitive(fresh_db): table = fresh_db["foo"] table.insert({"id": 1, "name": "Cleo"}, pk="id") table.add_missing_columns([{"Name": ".", "age": 4}]) assert ( table.schema - == "CREATE TABLE [foo] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n, [age] INTEGER)" + == 'CREATE TABLE "foo" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n, "age" INTEGER)' ) @@ -622,7 +692,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): ), pk="id", ) - assert 100 == fresh_db["big"].count + assert fresh_db["big"].count == 100 @pytest.mark.parametrize( @@ -631,7 +701,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): record = dict([("c{}".format(i), i) for i in range(num_columns)]) if should_error: - with pytest.raises(AssertionError): + with pytest.raises(ValueError): fresh_db["big"].insert(record) else: fresh_db["big"].insert(record) @@ -733,12 +803,40 @@ def test_create_index_if_not_exists(fresh_db): dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) assert [] == dogs.indexes dogs.create_index(["name"]) - assert 1 == len(dogs.indexes) + assert len(dogs.indexes) == 1 with pytest.raises(Exception, match="index idx_dogs_name already exists"): dogs.create_index(["name"]) dogs.create_index(["name"], if_not_exists=True) +def test_drop_index(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) + dogs.create_index(["name"]) + assert [index.name for index in dogs.indexes] == ["idx_dogs_name"] + dogs.drop_index("idx_dogs_name") + assert dogs.indexes == [] + + +def test_drop_index_ignore(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"name": "Cleo"}) + with pytest.raises(OperationalError, match="No index named idx_dogs_name"): + dogs.drop_index("idx_dogs_name") + dogs.drop_index("idx_dogs_name", ignore=True) + + +def test_drop_index_wrong_table(fresh_db): + dogs = fresh_db["dogs"] + cats = fresh_db["cats"] + dogs.insert({"name": "Cleo"}) + cats.insert({"name": "Misty"}) + dogs.create_index(["name"]) + with pytest.raises(OperationalError, match="No index named idx_dogs_name"): + cats.drop_index("idx_dogs_name") + assert [index.name for index in dogs.indexes] == ["idx_dogs_name"] + + def test_create_index_desc(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) @@ -748,10 +846,36 @@ def test_create_index_desc(fresh_db): "select sql from sqlite_master where name='idx_dogs_age_name'" ).fetchone()[0] assert sql == ( - "CREATE INDEX [idx_dogs_age_name]\n" " ON [dogs] ([age] desc, [name])" + 'CREATE INDEX "idx_dogs_age_name"\n' ' ON "dogs" ("age" desc, "name")' ) +def test_create_index_find_unique_name(fresh_db): + table = fresh_db["t"] + table.insert({"id": 1}) + table.create_index(["id"]) + # Without find_unique_name should error + with pytest.raises(OperationalError, match="index idx_t_id already exists"): + table.create_index(["id"]) + # With find_unique_name=True it should work + table.create_index(["id"], find_unique_name=True) + table.create_index(["id"], find_unique_name=True) + # Should have three now + index_names = {idx.name for idx in table.indexes} + assert index_names == {"idx_t_id", "idx_t_id_2", "idx_t_id_3"} + + +def test_create_index_analyze(fresh_db): + dogs = fresh_db["dogs"] + assert "sqlite_stat1" not in fresh_db.table_names() + dogs.insert({"name": "Cleo", "twitter": "cleopaws"}) + dogs.create_index(["name"], analyze=True) + assert "sqlite_stat1" in fresh_db.table_names() + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "dogs", "idx": "idx_dogs_name", "stat": "1 1"} + ] + + @pytest.mark.parametrize( "data_structure", ( @@ -807,7 +931,7 @@ def test_insert_thousands_using_generator(fresh_db): assert [{"name": "i", "type": "INTEGER"}, {"name": "word", "type": "TEXT"}] == [ {"name": col.name, "type": col.type} for col in fresh_db["test"].columns ] - assert 10000 == fresh_db["test"].count + assert fresh_db["test"].count == 10000 def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fresh_db): @@ -830,6 +954,58 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db assert rows == [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] +@pytest.mark.parametrize("num_rows", (0, 1, 2, 3, 10)) +def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): + # https://github.com/simonw/sqlite-utils/issues/732 + fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") + rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + + with pytest.raises(InvalidColumns) as ex: + fresh_db["t"].insert_all(rows, pk="not_a_column") + + assert ex.value.args == ( + "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", + ) + assert fresh_db["t"].count == 0 + + +@pytest.mark.parametrize("num_rows", (1, 2, 3, 10)) +def test_insert_all_pk_not_in_records_alter_raises(fresh_db, num_rows): + # With alter=True the check is deferred until the record keys are + # known - a pk column that is in neither the table nor the records + # still raises + fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") + rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + + with pytest.raises(InvalidColumns) as ex: + fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) + + assert ex.value.args == ( + "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", + ) + assert fresh_db["t"].count == 0 + + +def test_insert_pk_in_records_with_alter_adds_column(fresh_db): + # 3.x allowed insert(pk=..., alter=True) to add the pk column from the + # records - the InvalidColumns check must not fire in that case + fresh_db["t"].insert({"a": 1}) + fresh_db["t"].insert({"id": 5, "a": 2}, pk="id", alter=True) + assert fresh_db["t"].columns_dict.keys() == {"a", "id"} + assert list(fresh_db.query("select * from t order by a")) == [ + {"a": 1, "id": None}, + {"a": 2, "id": 5}, + ] + + +def test_insert_all_invalid_pk_alter_empty_records_is_noop(fresh_db): + # With alter=True the pk check needs record keys, so an empty iterator + # returns without error - matching the 3.x no-op for empty inserts + fresh_db.conn.execute("CREATE TABLE t (a TEXT)") + fresh_db["t"].insert_all([], pk="not_a_column", alter=True) + assert fresh_db["t"].count == 0 + + def test_insert_ignore(fresh_db): fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again @@ -842,17 +1018,151 @@ def test_insert_ignore(fresh_db): assert rows == [{"id": 1, "bar": 2}] +def test_insert_ignore_reports_existing_row(fresh_db): + # An ignored insert (row already exists) should point last_rowid and + # last_pk at the existing conflicting row - see the Datasette insert API + fresh_db["docs"].insert({"id": 1, "title": "Exists"}, pk="id") + # Insert a conflicting row with ignore=True and no explicit pk= + table = fresh_db["docs"].insert({"id": 1, "title": "One"}, ignore=True) + assert table.last_rowid == 1 + assert table.last_pk == 1 + assert list(fresh_db["docs"].rows_where("rowid = ?", [table.last_rowid])) == [ + {"id": 1, "title": "Exists"} + ] + + +@pytest.mark.parametrize("rowid_alias", ("rowid", "_rowid_", "oid")) +@pytest.mark.parametrize("method", ("upsert", "insert_replace", "insert_ignore")) +def test_pk_rowid_alias_on_rowid_table(fresh_db, rowid_alias, method): + # rowid and its aliases are valid primary keys for a rowid table even + # though they are not listed among the table's columns - see the Datasette + # upsert API against tables without an explicit primary key + fresh_db["t"].insert({"title": "Hello"}) + assert fresh_db["t"].pks == ["rowid"] + record = {rowid_alias: 1, "title": "Updated"} + if method == "upsert": + table = fresh_db["t"].upsert(record, pk=rowid_alias) + elif method == "insert_replace": + table = fresh_db["t"].insert(record, pk=rowid_alias, replace=True) + else: + table = fresh_db["t"].insert(record, pk=rowid_alias, ignore=True) + assert table.last_pk == 1 + expected_title = "Hello" if method == "insert_ignore" else "Updated" + assert list(fresh_db["t"].rows) == [{"title": expected_title}] + + +def test_insert_ignore_reports_existing_row_compound_pk(fresh_db): + # Compound primary key variant of the ignored-insert lookup + fresh_db["t"].insert_all([{"a": 1, "b": 2, "note": "first"}], pk=("a", "b")) + table = fresh_db["t"].insert( + {"a": 1, "b": 2, "note": "second"}, pk=("a", "b"), ignore=True + ) + assert table.last_pk == (1, 2) + assert list(fresh_db["t"].rows_where("rowid = ?", [table.last_rowid])) == [ + {"a": 1, "b": 2, "note": "first"} + ] + + +def test_insert_ignore_reports_existing_row_list_mode(fresh_db): + # List-based iteration variant of the ignored-insert lookup + fresh_db["t"].insert_all([["id", "title"], [1, "first"]], pk="id") + table = fresh_db["t"].insert_all( + [["id", "title"], [1, "second"]], pk="id", ignore=True + ) + assert table.last_pk == 1 + assert table.last_rowid == 1 + assert list(fresh_db["t"].rows) == [{"id": 1, "title": "first"}] + + +def test_insert_ignore_hash_id_reports_pk(fresh_db): + # With hash_id the pk is the computed hash; the original record has no id + # column to look up so last_rowid is left unset + first = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id") + table = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id", ignore=True) + assert table.last_pk == first.last_pk + assert table.last_rowid is None + assert fresh_db["dogs"].count == 1 + + +def test_insert_ignore_unresolvable_conflict_leaves_pk_unset(fresh_db): + # When the conflict cannot be resolved to a primary key lookup, last_pk and + # last_rowid are left unset rather than reporting a misleading value + + # rowid table with a UNIQUE column and no primary key: no pk to look up + fresh_db["u"].db.execute("create table u (title text unique)") + fresh_db["u"].insert({"title": "x"}) + table = fresh_db["u"].insert({"title": "x"}, ignore=True) + assert table.last_pk is None + assert table.last_rowid is None + assert fresh_db["u"].count == 1 + + # Conflict on a UNIQUE column other than the primary key: the pk value from + # the record does not match the existing row, so the lookup finds nothing + fresh_db["docs"].db.execute( + "create table docs (id integer primary key, email text unique)" + ) + fresh_db["docs"].insert({"id": 1, "email": "a"}, pk="id") + table = fresh_db["docs"].insert({"id": 2, "email": "a"}, ignore=True) + assert table.last_pk is None + assert table.last_rowid is None + assert fresh_db["docs"].count == 1 + + +def test_insert_ignore_with_pk_after_other_table_insert(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/554 + user = {"id": "abc", "name": "david"} + + fresh_db["users"].insert(user, pk="id") + fresh_db["comments"].insert_all( + [ + {"id": "def", "text": "ok"}, + {"id": "ghi", "text": "great"}, + ], + ) + + table = fresh_db["users"].insert(user, pk="id", ignore=True) + + assert table.last_pk == "abc" + assert list(fresh_db["users"].rows) == [user] + + def test_insert_hash_id(fresh_db): dogs = fresh_db["dogs"] id = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id - assert 1 == dogs.count + assert dogs.count == 1 # Insert replacing a second time should not create a new row id2 = dogs.insert( {"name": "Cleo", "twitter": "cleopaws"}, hash_id="id", replace=True ).last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id2 - assert 1 == dogs.count + assert dogs.count == 1 + + +@pytest.mark.parametrize("use_table_factory", [True, False]) +def test_insert_hash_id_columns(fresh_db, use_table_factory): + if use_table_factory: + dogs = fresh_db.table("dogs", hash_id_columns=("name", "twitter")) + insert_kwargs = {} + else: + dogs = fresh_db["dogs"] + insert_kwargs = dict(hash_id_columns=("name", "twitter")) + + id = dogs.insert( + {"name": "Cleo", "twitter": "cleopaws", "age": 5}, + **insert_kwargs, + ).last_pk + expected_hash = hash_record({"name": "Cleo", "twitter": "cleopaws"}) + assert id == expected_hash + assert dogs.count == 1 + # Insert replacing a second time should not create a new row + id2 = dogs.insert( + {"name": "Cleo", "twitter": "cleopaws", "age": 6}, + **insert_kwargs, + replace=True, + ).last_pk + assert id2 == expected_hash + assert dogs.count == 1 def test_vacuum(fresh_db): @@ -864,7 +1174,7 @@ def test_works_with_pathlib_path(tmpdir): path = pathlib.Path(tmpdir / "test.db") db = Database(path) db["demo"].insert_all([{"foo": 1}]) - assert 1 == db["demo"].count + assert db["demo"].count == 1 @pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed") @@ -940,7 +1250,7 @@ def test_create_table_numpy(fresh_db): def test_cannot_provide_both_filename_and_memory(): with pytest.raises( - AssertionError, match="Either specify a filename_or_conn or pass memory=True" + ValueError, match="Either specify a filename_or_conn or pass memory=True" ): Database("/tmp/foo.db", memory=True) @@ -981,11 +1291,11 @@ def test_drop_ignore(fresh_db): def test_insert_all_empty_list(fresh_db): fresh_db["t"].insert({"foo": 1}) - assert 1 == fresh_db["t"].count + assert fresh_db["t"].count == 1 fresh_db["t"].insert_all([]) - assert 1 == fresh_db["t"].count + assert fresh_db["t"].count == 1 fresh_db["t"].insert_all([], replace=True) - assert 1 == fresh_db["t"].count + assert fresh_db["t"].count == 1 def test_insert_all_single_column(fresh_db): @@ -995,6 +1305,23 @@ def test_insert_all_single_column(fresh_db): assert table.pks == ["name"] +@pytest.mark.parametrize("method_name", ("insert_all", "upsert_all")) +def test_insert_all_analyze(fresh_db, method_name): + table = fresh_db["table"] + table.insert_all([{"id": 1, "name": "Cleo"}], pk="id") + assert "sqlite_stat1" not in fresh_db.table_names() + table.create_index(["name"], analyze=True) + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_name", "stat": "1 1"} + ] + method = getattr(table, method_name) + method([{"id": 2, "name": "Suna"}], pk="id", analyze=True) + assert "sqlite_stat1" in fresh_db.table_names() + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_name", "stat": "2 1"} + ] + + def test_create_with_a_null_column(fresh_db): record = {"name": "Name", "description": None} fresh_db["t"].insert(record) @@ -1012,3 +1339,337 @@ def test_create_with_nested_bytes(fresh_db): ) def test_quote(fresh_db, input, expected): assert fresh_db.quote(input) == expected + + +@pytest.mark.parametrize( + "columns,expected_sql_middle", + ( + ( + {"id": int}, + '"id" INTEGER', + ), + ( + {"col": dict}, + '"col" TEXT', + ), + ( + {"col": tuple}, + '"col" TEXT', + ), + ( + {"col": list}, + '"col" TEXT', + ), + ), +) +def test_create_table_sql(fresh_db, columns, expected_sql_middle): + sql = fresh_db.create_table_sql("t", columns) + middle = sql.split("(")[1].split(")")[0].strip() + assert middle == expected_sql_middle + + +def test_create(fresh_db): + fresh_db["t"].create( + { + "id": int, + "text": str, + "float": float, + "integer": int, + "bytes": bytes, + }, + pk="id", + column_order=("id", "float"), + not_null=("float", "integer"), + defaults={"integer": 0}, + ) + assert fresh_db["t"].schema == ( + 'CREATE TABLE "t" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "float" REAL NOT NULL,\n' + ' "text" TEXT,\n' + ' "integer" INTEGER NOT NULL DEFAULT 0,\n' + ' "bytes" BLOB\n' + ")" + ) + + +def test_create_if_not_exists(fresh_db): + fresh_db["t"].create({"id": int}) + # This should error + with pytest.raises(sqlite3.OperationalError): + fresh_db["t"].create({"id": int}) + # This should not + fresh_db["t"].create({"id": int}, if_not_exists=True) + + +def test_create_if_no_columns(fresh_db): + with pytest.raises(ValueError) as error: + fresh_db["t"].create({}) + assert error.value.args[0] == "Tables must have at least one column" + + +def test_create_ignore(fresh_db): + fresh_db["t"].create({"id": int}) + # This should error + with pytest.raises(sqlite3.OperationalError): + fresh_db["t"].create({"id": int}) + # This should not + fresh_db["t"].create({"id": int}, ignore=True) + + +def test_create_replace(fresh_db): + fresh_db["t"].create({"id": int}) + # This should error + with pytest.raises(sqlite3.OperationalError): + fresh_db["t"].create({"id": int}) + # This should not + fresh_db["t"].create({"name": str}, replace=True) + assert fresh_db["t"].schema == ('CREATE TABLE "t" (\n' ' "name" TEXT\n' ")") + + +@pytest.mark.parametrize( + "cols,kwargs,expected_schema,should_transform", + ( + # Change nothing + ( + {"id": int, "name": str}, + {"pk": "id"}, + 'CREATE TABLE "demo" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n)', + False, + ), + # Drop name column, remove primary key + ({"id": int}, {}, 'CREATE TABLE "demo" (\n "id" INTEGER\n)', True), + # Add a new column + ( + {"id": int, "name": str, "age": int}, + {"pk": "id"}, + 'CREATE TABLE "demo" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n)', + True, + ), + # Change a column type + ( + {"id": int, "name": bytes}, + {"pk": "id"}, + 'CREATE TABLE "demo" (\n "id" INTEGER PRIMARY KEY,\n "name" BLOB\n)', + True, + ), + # Change the primary key + ( + {"id": int, "name": str}, + {"pk": "name"}, + 'CREATE TABLE "demo" (\n "id" INTEGER,\n "name" TEXT PRIMARY KEY\n)', + True, + ), + # Change in column order + ( + {"id": int, "name": str}, + {"pk": "id", "column_order": ["name"]}, + 'CREATE TABLE "demo" (\n "name" TEXT,\n "id" INTEGER PRIMARY KEY\n)', + True, + ), + # Same column order is ignored + ( + {"id": int, "name": str}, + {"pk": "id", "column_order": ["id", "name"]}, + 'CREATE TABLE "demo" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n)', + False, + ), + # Change not null + ( + {"id": int, "name": str}, + {"pk": "id", "not_null": {"name"}}, + 'CREATE TABLE "demo" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT NOT NULL\n)', + True, + ), + # Change default values + ( + {"id": int, "name": str}, + {"pk": "id", "defaults": {"id": 0, "name": "Bob"}}, + 'CREATE TABLE "demo" (\n "id" INTEGER PRIMARY KEY DEFAULT 0,\n "name" TEXT DEFAULT \'Bob\'\n)', + True, + ), + ), +) +def test_create_transform(fresh_db, cols, kwargs, expected_schema, should_transform): + fresh_db.create_table("demo", {"id": int, "name": str}, pk="id") + fresh_db["demo"].insert({"id": 1, "name": "Cleo"}) + traces = [] + with fresh_db.tracer(lambda sql, parameters: traces.append((sql, parameters))): + fresh_db["demo"].create(cols, **kwargs, transform=True) + at_least_one_create_table = any(sql.startswith("CREATE TABLE") for sql, _ in traces) + assert should_transform == at_least_one_create_table + new_schema = fresh_db["demo"].schema + assert new_schema == expected_schema, repr(new_schema) + assert fresh_db["demo"].count == 1 + + +def test_rename_table(fresh_db): + fresh_db["t"].insert({"foo": "bar"}) + assert ["t"] == fresh_db.table_names() + fresh_db.rename_table("t", "renamed") + assert ["renamed"] == fresh_db.table_names() + assert [{"foo": "bar"}] == list(fresh_db["renamed"].rows) + # Should error if table does not exist: + with pytest.raises(sqlite3.OperationalError): + fresh_db.rename_table("does_not_exist", "renamed") + + +@pytest.mark.parametrize("strict", (False, True)) +def test_database_strict(strict): + db = Database(memory=True, strict=strict) + table = db.table("t", columns={"id": int}) + table.insert({"id": 1}) + assert table.strict == strict or not db.supports_strict + + +@pytest.mark.parametrize("strict", (False, True)) +def test_database_strict_override(strict): + db = Database(memory=True, strict=strict) + table = db.table("t", columns={"id": int}, strict=not strict) + table.insert({"id": 1}) + assert table.strict != strict or not db.supports_strict + + +@pytest.mark.parametrize( + "method_name", ("insert", "upsert", "insert_all", "upsert_all") +) +@pytest.mark.parametrize("strict", (False, True)) +def test_insert_upsert_strict(fresh_db, method_name, strict): + table = fresh_db["t"] + method = getattr(table, method_name) + record = {"id": 1} + if method_name.endswith("_all"): + record = [record] + method(record, pk="id", strict=strict) + assert table.strict == strict or not fresh_db.supports_strict + + +@pytest.mark.parametrize("strict", (False, True)) +def test_create_table_strict(fresh_db, strict): + table = fresh_db.create_table("t", {"id": int, "f": float}, strict=strict) + assert table.strict == strict or not fresh_db.supports_strict + expected_schema = 'CREATE TABLE "t" (\n' ' "id" INTEGER,\n' ' "f" REAL\n' ")" + if strict and not fresh_db.supports_strict: + return + if strict: + expected_schema = 'CREATE TABLE "t" (\n "id" INTEGER,\n "f" REAL\n) STRICT' + assert table.schema == expected_schema + + +@pytest.mark.parametrize("strict", (False, True)) +def test_create_strict(fresh_db, strict): + table = fresh_db["t"] + table.create({"id": int}, strict=strict) + assert table.strict == strict or not fresh_db.supports_strict + + +def test_bad_table_and_view_exceptions(fresh_db): + fresh_db.table("t").insert({"id": 1}, pk="id") + fresh_db.create_view("v", "select * from t") + with pytest.raises(NoTable) as ex: + fresh_db.table("v") + assert ex.value.args[0] == "Table v is actually a view" + with pytest.raises(NoView) as ex2: + fresh_db.view("t") + assert ex2.value.args[0] == "View t does not exist - t is a table" + with pytest.raises(NoView) as ex3: + fresh_db.view("missing") + assert ex3.value.args[0] == "View missing does not exist" + + +# Tests for issue #655: Table configuration should be stored in _defaults +# after table creation, so subsequent operations use the same settings. + + +def test_pk_persists_after_insert_655(fresh_db): + """When pk is passed to insert(), subsequent inserts should use it.""" + table = fresh_db["users"] + table.insert({"id": 1, "name": "Alice"}, pk="id") + # Second insert should use pk="id" from _defaults + table.insert({"id": 2, "name": "Bob"}) + assert table.pks == ["id"] + # Verify both rows exist (not overwritten due to missing pk) + assert table.count == 2 + + +def test_pk_persists_after_insert_all_655(fresh_db): + """When pk is passed to insert_all(), subsequent inserts should use it.""" + table = fresh_db["users"] + table.insert_all([{"id": 1, "name": "Alice"}], pk="id") + # Second insert_all should use pk="id" from _defaults + table.insert_all([{"id": 2, "name": "Bob"}]) + assert table.pks == ["id"] + assert table.count == 2 + + +def test_pk_persists_after_create_655(fresh_db): + """When pk is passed to create(), it should be stored in _defaults.""" + table = fresh_db["users"] + table.create({"id": int, "name": str}, pk="id") + assert table._defaults["pk"] == "id" + # Subsequent insert should use the pk + table.insert({"id": 1, "name": "Alice"}) + table.insert({"id": 2, "name": "Bob"}) + assert table.count == 2 + + +def test_foreign_keys_persist_after_create_655(fresh_db): + """When foreign_keys is passed to create(), it should be stored in _defaults.""" + fresh_db["authors"].insert({"id": 1, "name": "Alice"}, pk="id") + table = fresh_db["books"] + table.create( + {"id": int, "title": str, "author_id": int}, + pk="id", + foreign_keys=[("author_id", "authors", "id")], + ) + assert table._defaults["pk"] == "id" + assert table._defaults["foreign_keys"] == [("author_id", "authors", "id")] + + +def test_not_null_persists_after_create_655(fresh_db): + """When not_null is passed to create(), it should be stored in _defaults.""" + table = fresh_db["users"] + table.create({"id": int, "name": str}, pk="id", not_null=["name"]) + assert table._defaults["not_null"] == ["name"] + + +def test_defaults_persist_after_create_655(fresh_db): + """When defaults is passed to create(), it should be stored in _defaults.""" + table = fresh_db["users"] + table.create({"id": int, "score": int}, pk="id", defaults={"score": 0}) + assert table._defaults["defaults"] == {"score": 0} + + +def test_strict_persists_after_create_655(fresh_db): + """When strict is passed to create(), it should be stored in _defaults.""" + table = fresh_db["users"] + table.create({"id": int, "name": str}, pk="id", strict=True) + assert table._defaults["strict"] is True + + +def test_upsert_uses_pk_from_prior_insert_655(fresh_db): + """After insert with pk, upsert should use the same pk.""" + table = fresh_db["users"] + table.insert({"id": 1, "name": "Alice"}, pk="id") + # Upsert should work without specifying pk again + table.upsert({"id": 1, "name": "Alice Updated"}) + assert table.count == 1 + assert list(table.rows)[0]["name"] == "Alice Updated" + + +def test_upsert_all_uses_pk_from_prior_insert_655(fresh_db): + """After insert with pk, upsert_all should use the same pk.""" + table = fresh_db["users"] + table.insert({"id": 1, "name": "Alice"}, pk="id") + # Upsert_all should work without specifying pk again + table.upsert_all([{"id": 1, "name": "Alice Updated"}, {"id": 2, "name": "Bob"}]) + assert table.count == 2 + rows = {row["id"]: row["name"] for row in table.rows} + assert rows == {1: "Alice Updated", 2: "Bob"} + + +def test_chained_create_sets_pks(fresh_db): + table = fresh_db.table("dogs3", pk="id").create( + {"id": int, "name": str, "color": str} + ) + assert table.pks == ["id"] diff --git a/tests/test_create_view.py b/tests/test_create_view.py index f0e2855..056e246 100644 --- a/tests/test_create_view.py +++ b/tests/test_create_view.py @@ -15,7 +15,7 @@ def test_create_view_error(fresh_db): def test_create_view_only_arrow_one_param(fresh_db): - with pytest.raises(AssertionError): + with pytest.raises(ValueError): fresh_db.create_view("bar", "select 1 + 2", ignore=True, replace=True) diff --git a/tests/test_default_value.py b/tests/test_default_value.py new file mode 100644 index 0000000..3724d99 --- /dev/null +++ b/tests/test_default_value.py @@ -0,0 +1,59 @@ +import pytest + +EXAMPLES = [ + ("TEXT DEFAULT 'foo'", "'foo'", "'foo'"), + ("TEXT DEFAULT 'foo)'", "'foo)'", "'foo)'"), + ("INTEGER DEFAULT '1'", "'1'", "'1'"), + ("INTEGER DEFAULT 1", "1", "'1'"), + ("INTEGER DEFAULT (1)", "1", "'1'"), + # Expressions + ( + "TEXT DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))", + "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", + "(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))", + ), + # Special values + ("TEXT DEFAULT CURRENT_TIME", "CURRENT_TIME", "CURRENT_TIME"), + ("TEXT DEFAULT CURRENT_DATE", "CURRENT_DATE", "CURRENT_DATE"), + ("TEXT DEFAULT CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"), + ("TEXT DEFAULT current_timestamp", "current_timestamp", "current_timestamp"), + ("TEXT DEFAULT (CURRENT_TIMESTAMP)", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"), + # Strings + ("TEXT DEFAULT 'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'"), + ('TEXT DEFAULT "CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"'), + # Boolean and null keyword literals must stay unquoted + ("INTEGER DEFAULT TRUE", "TRUE", "TRUE"), + ("INTEGER DEFAULT FALSE", "FALSE", "FALSE"), + ("INTEGER DEFAULT true", "true", "true"), + ("TEXT DEFAULT NULL", "NULL", "NULL"), +] + + +@pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES) +def test_quote_default_value(fresh_db, column_def, initial_value, expected_value): + fresh_db.execute("create table foo (col {})".format(column_def)) + assert initial_value == fresh_db["foo"].columns[0].default_value + assert expected_value == fresh_db.quote_default_value( + fresh_db["foo"].columns[0].default_value + ) + + +def test_insert_empty_record_uses_default_values(fresh_db): + fresh_db.execute(""" + CREATE TABLE has_defaults ( + id INTEGER PRIMARY KEY, + name TEXT, + timestamp TEXT DEFAULT CURRENT_TIMESTAMP, + is_active INTEGER NOT NULL DEFAULT 1 + ) + """) + + table = fresh_db["has_defaults"] + table.insert({}) + + rows = list(table.rows) + assert len(rows) == 1 + assert rows[0]["id"] == 1 + assert rows[0]["name"] is None + assert rows[0]["timestamp"] is not None + assert rows[0]["is_active"] == 1 diff --git a/tests/test_delete.py b/tests/test_delete.py index 1198d06..a2d93aa 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -1,3 +1,6 @@ +import sqlite_utils + + def test_delete_rowid_table(fresh_db): table = fresh_db["table"] table.insert({"foo": 1}).last_pk @@ -18,15 +21,44 @@ def test_delete_where(fresh_db): table = fresh_db["table"] for i in range(1, 11): table.insert({"id": i}, pk="id") - assert 10 == table.count + assert table.count == 10 table.delete_where("id > ?", [5]) - assert 5 == table.count + assert table.count == 5 def test_delete_where_all(fresh_db): table = fresh_db["table"] for i in range(1, 11): table.insert({"id": i}, pk="id") - assert 10 == table.count + assert table.count == 10 table.delete_where() - assert 0 == table.count + assert table.count == 0 + + +def test_delete_where_commits(tmpdir): + path = str(tmpdir / "test.db") + db = sqlite_utils.Database(path) + db["table"].insert_all([{"id": i} for i in range(5)], pk="id") + db["table"].delete_where("id > ?", [2]) + # The connection must not be left inside an open transaction, + # otherwise subsequent atomic() blocks never commit either + assert not db.conn.in_transaction + db["table"].insert({"id": 100}) + db.close() + db2 = sqlite_utils.Database(path) + assert [r["id"] for r in db2["table"].rows] == [0, 1, 2, 100] + db2.close() + + +def test_delete_where_analyze(fresh_db): + table = fresh_db["table"] + table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id") + table.create_index(["i"], analyze=True) + assert "sqlite_stat1" in fresh_db.table_names() + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_i", "stat": "10 1"} + ] + table.delete_where("id > ?", [5], analyze=True) + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_i", "stat": "6 1"} + ] diff --git a/tests/test_docs.py b/tests/test_docs.py index d760629..f657416 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -5,13 +5,15 @@ import pytest import re docs_path = Path(__file__).parent.parent / "docs" -commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+) ") +commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+)") recipes_re = re.compile(r"r\.(\w+)\(") @pytest.fixture(scope="session") def documented_commands(): - rst = (docs_path / "cli.rst").read_text() + rst = "" + for doc in ("cli.rst", "plugins.rst"): + rst += (docs_path / doc).read_text() return { command for command in commands_re.findall(rst) @@ -39,16 +41,22 @@ def test_convert_help(): result = CliRunner().invoke(cli.cli, ["convert", "--help"]) assert result.exit_code == 0 for expected in ( - "r.jsonsplit(value, ", - "r.parsedate(value, ", - "r.parsedatetime(value, ", + "r.jsonsplit(value:", + "r.parsedate(value:", + "r.parsedatetime(value:", ): assert expected in result.output @pytest.mark.parametrize( "recipe", - [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")], + [ + n + for n in dir(recipes) + if not n.startswith("_") + and n not in ("json", "parser", "Callable", "Optional") + and callable(getattr(recipes, n)) + ], ) def test_recipes_are_documented(documented_recipes, recipe): assert recipe in documented_recipes diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py new file mode 100644 index 0000000..28961d2 --- /dev/null +++ b/tests/test_duplicate.py @@ -0,0 +1,41 @@ +from sqlite_utils.db import NoTable +import datetime +import pytest + + +def test_duplicate(fresh_db): + # Create table using native Sqlite statement: + fresh_db.execute("""CREATE TABLE "table1" ( + "text_col" TEXT, + "real_col" REAL, + "int_col" INTEGER, + "bool_col" INTEGER, + "datetime_col" TEXT)""") + # Insert one row of mock data: + dt = datetime.datetime.now() + data = { + "text_col": "Cleo", + "real_col": 3.14, + "int_col": -255, + "bool_col": True, + "datetime_col": str(dt), + } + table1 = fresh_db["table1"] + row_id = table1.insert(data).last_rowid + # Duplicate table: + table2 = table1.duplicate("table2") + # Ensure data integrity: + assert data == table2.get(row_id) + # Ensure schema integrity: + assert [ + {"name": "text_col", "type": "TEXT"}, + {"name": "real_col", "type": "REAL"}, + {"name": "int_col", "type": "INT"}, + {"name": "bool_col", "type": "INT"}, + {"name": "datetime_col", "type": "TEXT"}, + ] == [{"name": col.name, "type": col.type} for col in table2.columns] + + +def test_duplicate_fails_if_table_does_not_exist(fresh_db): + with pytest.raises(NoTable): + fresh_db["not_a_table"].duplicate("duplicated") diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index d724e80..2f6b0db 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -15,25 +15,25 @@ def test_enable_counts_specific_table(fresh_db): foo.enable_counts() assert foo.triggers_dict == { "foo_counts_insert": ( - "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\n" + 'CREATE TRIGGER "foo_counts_insert" AFTER INSERT ON "foo"\n' "BEGIN\n" - " INSERT OR REPLACE INTO [_counts]\n" + ' INSERT OR REPLACE INTO "_counts"\n' " VALUES (\n 'foo',\n" " COALESCE(\n" - " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" + ' (SELECT count FROM "_counts" WHERE "table" = \'foo\'),\n' " 0\n" " ) + 1\n" " );\n" "END" ), "foo_counts_delete": ( - "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\n" + 'CREATE TRIGGER "foo_counts_delete" AFTER DELETE ON "foo"\n' "BEGIN\n" - " INSERT OR REPLACE INTO [_counts]\n" + ' INSERT OR REPLACE INTO "_counts"\n' " VALUES (\n" " 'foo',\n" " COALESCE(\n" - " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" + ' (SELECT count FROM "_counts" WHERE "table" = \'foo\'),\n' " 0\n" " ) - 1\n" " );\n" @@ -129,19 +129,19 @@ def test_uses_counts_after_enable_counts(counts_db_path): db = Database(counts_db_path) logged = [] with db.tracer(lambda sql, parameters: logged.append((sql, parameters))): - assert db["foo"].count == 1 + assert db.table("foo").count == 1 assert logged == [ ("select name from sqlite_master where type = 'view'", None), - ("select count(*) from [foo]", []), + ('select count(*) from "foo"', []), ] logged.clear() assert not db.use_counts_table db.enable_counts() assert db.use_counts_table - assert db["foo"].count == 1 + assert db.table("foo").count == 1 assert logged == [ ( - "CREATE TABLE IF NOT EXISTS [_counts](\n [table] TEXT PRIMARY KEY,\n count INTEGER DEFAULT 0\n);", + 'CREATE TABLE IF NOT EXISTS "_counts"(\n "table" TEXT PRIMARY KEY,\n count INTEGER DEFAULT 0\n);', None, ), ("select name from sqlite_master where type = 'table'", None), @@ -157,7 +157,7 @@ def test_uses_counts_after_enable_counts(counts_db_path): ("SELECT quote(:value)", {"value": "baz"}), ("select sql from sqlite_master where name = ?", ("_counts",)), ("select name from sqlite_master where type = 'view'", None), - ("select [table], count from _counts where [table] in (?)", ["foo"]), + ('select "table", count from _counts where "table" in (?)', ["foo"]), ] diff --git a/tests/test_extract.py b/tests/test_extract.py index 10b5b09..c73ee7a 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -24,19 +24,16 @@ def test_extract_single_column(fresh_db, table, fk_column): fresh_db["tree"].extract("species", table=table, fk_column=fk_column) assert fresh_db["tree"].schema == ( 'CREATE TABLE "tree" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT,\n" - " [{}] INTEGER,\n".format(expected_fk) - + " [end] INTEGER,\n" - + " FOREIGN KEY([{}]) REFERENCES [{}]([id])\n".format( - expected_fk, expected_table - ) + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT,\n' + ' "{}" INTEGER REFERENCES "{}"("id"),\n'.format(expected_fk, expected_table) + + ' "end" INTEGER\n' + ")" ) assert fresh_db[expected_table].schema == ( - "CREATE TABLE [{}] (\n".format(expected_table) - + " [id] INTEGER PRIMARY KEY,\n" - " [species] TEXT\n" + 'CREATE TABLE "{}" (\n'.format(expected_table) + + ' "id" INTEGER PRIMARY KEY,\n' + ' "species" TEXT\n' ")" ) assert list(fresh_db[expected_table].rows) == [ @@ -74,17 +71,16 @@ def test_extract_multiple_columns_with_rename(fresh_db): ) assert fresh_db["tree"].schema == ( 'CREATE TABLE "tree" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT,\n" - " [common_name_latin_name_id] INTEGER,\n" - " FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n" + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT,\n' + ' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n' ")" ) assert fresh_db["common_name_latin_name"].schema == ( - "CREATE TABLE [common_name_latin_name] (\n" - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT,\n" - " [latin_name] TEXT\n" + 'CREATE TABLE "common_name_latin_name" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT,\n' + ' "latin_name" TEXT\n' ")" ) assert list(fresh_db["common_name_latin_name"].rows) == [ @@ -126,14 +122,11 @@ def test_extract_rowid_table(fresh_db): fresh_db["tree"].extract(["common_name", "latin_name"]) assert fresh_db["tree"].schema == ( 'CREATE TABLE "tree" (\n' - " [name] TEXT,\n" - " [common_name_latin_name_id] INTEGER,\n" - " FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n" + ' "name" TEXT,\n' + ' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n' ")" ) - assert ( - fresh_db.execute( - """ + assert fresh_db.execute(""" select tree.name, common_name_latin_name.common_name, @@ -141,10 +134,7 @@ def test_extract_rowid_table(fresh_db): from tree join common_name_latin_name on tree.common_name_latin_name_id = common_name_latin_name.id - """ - ).fetchall() - == [("Tree 1", "Palm", "Arecaceae")] - ) + """).fetchall() == [("Tree 1", "Palm", "Arecaceae")] def test_reuse_lookup_table(fresh_db): @@ -157,17 +147,15 @@ def test_reuse_lookup_table(fresh_db): fresh_db["individuals"].extract("species", rename={"species": "name"}) assert fresh_db["sightings"].schema == ( 'CREATE TABLE "sightings" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [species_id] INTEGER,\n" - " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ' "id" INTEGER PRIMARY KEY,\n' + ' "species_id" INTEGER REFERENCES "species"("id")\n' ")" ) assert fresh_db["individuals"].schema == ( 'CREATE TABLE "individuals" (\n' - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT,\n" - " [species_id] INTEGER,\n" - " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ' "id" INTEGER PRIMARY KEY,\n' + ' "name" TEXT,\n' + ' "species_id" INTEGER REFERENCES "species"("id")\n' ")" ) assert list(fresh_db["species"].rows) == [ @@ -186,3 +174,131 @@ def test_extract_error_on_incompatible_existing_lookup_table(fresh_db): fresh_db["species2"].insert({"id": 1, "common_name": 3.5}) with pytest.raises(InvalidColumns): fresh_db["tree"].extract("common_name", table="species2") + + +def test_extract_works_with_null_values(fresh_db): + fresh_db["listens"].insert_all( + [ + {"id": 1, "track_title": "foo", "album_title": "bar"}, + {"id": 2, "track_title": "baz", "album_title": None}, + ], + pk="id", + ) + fresh_db["listens"].extract( + columns=["album_title"], table="albums", fk_column="album_id" + ) + assert list(fresh_db["listens"].rows) == [ + {"id": 1, "track_title": "foo", "album_id": 1}, + {"id": 2, "track_title": "baz", "album_id": None}, + ] + assert list(fresh_db["albums"].rows) == [ + {"id": 1, "album_title": "bar"}, + ] + + +def test_extract_null_values_single_column(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/186 + fresh_db["species"].insert({"id": 1, "species": "Wolf"}, pk="id") + fresh_db["individuals"].insert_all( + [ + {"id": 10, "name": "Terriana", "species": "Fox"}, + {"id": 11, "name": "Spenidorm", "species": None}, + {"id": 12, "name": "Grantheim", "species": "Wolf"}, + {"id": 13, "name": "Turnutopia", "species": None}, + {"id": 14, "name": "Wargal", "species": "Wolf"}, + ], + pk="id", + ) + fresh_db["individuals"].extract("species") + # No null row should have been added to species + assert list(fresh_db["species"].rows) == [ + {"id": 1, "species": "Wolf"}, + {"id": 2, "species": "Fox"}, + ] + assert list(fresh_db["individuals"].rows) == [ + {"id": 10, "name": "Terriana", "species_id": 2}, + {"id": 11, "name": "Spenidorm", "species_id": None}, + {"id": 12, "name": "Grantheim", "species_id": 1}, + {"id": 13, "name": "Turnutopia", "species_id": None}, + {"id": 14, "name": "Wargal", "species_id": 1}, + ] + + +def test_extract_null_values_multiple_columns(fresh_db): + # A row should be extracted if at least one column is not null - + # only rows where ALL extracted columns are null are left alone + fresh_db["circulation"].insert_all( + [ + {"id": 1, "title": "title one", "creator": "creator one", "year": 2018}, + {"id": 2, "title": "title two", "creator": None, "year": 2019}, + {"id": 3, "title": None, "creator": None, "year": 2020}, + {"id": 4, "title": None, "creator": None, "year": 2021}, + ], + pk="id", + ) + fresh_db["circulation"].extract( + ["title", "creator"], table="books", fk_column="book_id" + ) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "title one", "creator": "creator one"}, + {"id": 2, "title": "title two", "creator": None}, + ] + assert list(fresh_db["circulation"].rows) == [ + {"id": 1, "book_id": 1, "year": 2018}, + {"id": 2, "book_id": 2, "year": 2019}, + {"id": 3, "book_id": None, "year": 2020}, + {"id": 4, "book_id": None, "year": 2021}, + ] + + +def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db): + # Even if the lookup table already contains an all-null row, rows where + # every extracted column is null should keep a null foreign key + fresh_db["species"].insert({"id": 1, "species": None}, pk="id") + fresh_db["individuals"].insert_all( + [ + {"id": 10, "name": "Terriana", "species": "Fox"}, + {"id": 11, "name": "Spenidorm", "species": None}, + ], + pk="id", + ) + fresh_db["individuals"].extract("species") + assert list(fresh_db["species"].rows) == [ + {"id": 1, "species": None}, + {"id": 2, "species": "Fox"}, + ] + assert list(fresh_db["individuals"].rows) == [ + {"id": 10, "name": "Terriana", "species_id": 2}, + {"id": 11, "name": "Spenidorm", "species_id": None}, + ] + + +def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db): + # Unique indexes treat NULLs as distinct, so INSERT OR IGNORE alone + # cannot dedupe NULL-containing rows against the existing lookup + # table - extracting a second table into the same lookup previously + # inserted duplicate rows that nothing pointed to + fresh_db["t1"].insert_all( + [ + {"id": 1, "species": None, "common": "X"}, + {"id": 2, "species": "Oak", "common": "Oak"}, + ], + pk="id", + ) + fresh_db["t2"].insert_all([{"id": 1, "species": None, "common": "X"}], pk="id") + fresh_db["t1"].extract(["species", "common"], table="lk") + fresh_db["t2"].extract(["species", "common"], table="lk") + assert fresh_db["lk"].count == 2 + # Both tables point at the same lookup row + t1_fk = fresh_db.execute("select lk_id from t1 where id = 1").fetchone()[0] + t2_fk = fresh_db.execute("select lk_id from t2 where id = 1").fetchone()[0] + assert t1_fk == t2_fk + + +def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db): + # Non-NULL rows were already deduped by the unique index - keep it so + fresh_db["t1"].insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db["t2"].insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db["t1"].extract(["species"], table="lk") + fresh_db["t2"].extract(["species"], table="lk") + assert fresh_db["lk"].count == 1 diff --git a/tests/test_extracts.py b/tests/test_extracts.py index cca16ba..7add79a 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -25,18 +25,18 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): {"id": 2, "species_id": "Oak"}, {"id": 3, "species_id": "Palm"}, ], - **insert_kwargs + **insert_kwargs, ) # Should now have two tables: Trees and Species assert {expected_table, "Trees"} == set(fresh_db.table_names()) assert ( - "CREATE TABLE [{}] (\n [id] INTEGER PRIMARY KEY,\n [value] TEXT\n)".format( + 'CREATE TABLE "{}" (\n "id" INTEGER PRIMARY KEY,\n "value" TEXT\n)'.format( expected_table ) == fresh_db[expected_table].schema ) assert ( - "CREATE TABLE [Trees] (\n [id] INTEGER,\n [species_id] INTEGER REFERENCES [{}]([id])\n)".format( + 'CREATE TABLE "Trees" (\n "id" INTEGER,\n "species_id" INTEGER REFERENCES "{}"("id")\n)'.format( expected_table ) == fresh_db["Trees"].schema @@ -67,3 +67,51 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): {"id": 2, "species_id": 1}, {"id": 3, "species_id": 2}, ] == list(fresh_db["Trees"].rows) + + +def test_extracts_null_values(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/186 + # Null values should stay null, not be extracted into the lookup table + fresh_db["Trees"].insert_all( + [ + {"id": 1, "species_id": "Oak"}, + {"id": 2, "species_id": None}, + {"id": 3, "species_id": "Palm"}, + {"id": 4, "species_id": None}, + ], + extracts={"species_id": "Species"}, + ) + assert list(fresh_db["Species"].rows) == [ + {"id": 1, "value": "Oak"}, + {"id": 2, "value": "Palm"}, + ] + assert list(fresh_db["Trees"].rows) == [ + {"id": 1, "species_id": 1}, + {"id": 2, "species_id": None}, + {"id": 3, "species_id": 2}, + {"id": 4, "species_id": None}, + ] + + +def test_extracts_null_values_list_mode(fresh_db): + # Same as test_extracts_null_values but for list-based records + fresh_db["Trees"].insert_all( + [ + ["id", "species_id"], + [1, "Oak"], + [2, None], + [3, "Palm"], + [4, None], + ], + extracts={"species_id": "Species"}, + ) + assert list(fresh_db["Species"].rows) == [ + {"id": 1, "value": "Oak"}, + {"id": 2, "value": "Palm"}, + ] + assert list(fresh_db["Trees"].rows) == [ + {"id": 1, "species_id": 1}, + {"id": 2, "species_id": None}, + {"id": 3, "species_id": 2}, + {"id": 4, "species_id": None}, + ] diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py new file mode 100644 index 0000000..b37d374 --- /dev/null +++ b/tests/test_foreign_keys.py @@ -0,0 +1,691 @@ +"""Tests for compound (multi-column) foreign keys - issue #594.""" + +import pytest +from sqlite_utils import Database +from sqlite_utils.db import AlterError, ForeignKey +from sqlite_utils.utils import sqlite3 + +COMPOUND_SCHEMA = """ +CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + dept_name TEXT, + PRIMARY KEY (campus_name, dept_code) +); +CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + course_name TEXT, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + FOREIGN KEY (campus_name, dept_code) + REFERENCES departments(campus_name, dept_code) +); +""" + + +@pytest.fixture +def compound_db(): + db = Database(memory=True) + db.executescript(COMPOUND_SCHEMA) + return db + + +def test_compound_foreign_key(compound_db): + fks = compound_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.table == "courses" + assert fk.other_table == "departments" + assert fk.columns == ("campus_name", "dept_code") + assert fk.other_columns == ("campus_name", "dept_code") + # Scalar column/other_column can't sensibly hold a compound key + assert fk.column is None + assert fk.other_column is None + + +def test_single_foreign_key_gets_columns_fields(fresh_db): + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fk = fresh_db["books"].foreign_keys[0] + assert fk.is_compound is False + assert fk.column == "author_id" + assert fk.other_column == "id" + assert fk.columns == ("author_id",) + assert fk.other_columns == ("id",) + + +def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): + # Clean break in 4.0: ForeignKey is a dataclass, not a namedtuple, so the + # old tuple unpacking and indexing patterns now fail hard. + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fk = fresh_db["books"].foreign_keys[0] + with pytest.raises(TypeError): + table, column, other_table, other_column = fk + with pytest.raises(TypeError): + fk[0] + + +def test_foreign_keys_are_sortable(fresh_db): + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["categories"].insert({"id": 1, "name": "Wildlife"}, pk="id") + fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1, "category_id": 1}) + fresh_db.add_foreign_keys( + [ + ("books", "author_id", "authors", "id"), + ("books", "category_id", "categories", "id"), + ] + ) + fks = sorted(fresh_db["books"].foreign_keys) + assert fks[0].column == "author_id" + assert fks[1].column == "category_id" + + +def test_mixed_compound_and_single_foreign_keys_are_sortable(): + # compound FKs have column=None, which must not break sorting + # against single-column FKs (None < str raises TypeError) + db = Database(memory=True) + db.executescript(""" + CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + PRIMARY KEY (campus_name, dept_code) + ); + CREATE TABLE accreditations (id INTEGER PRIMARY KEY); + CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + accreditation_id INTEGER REFERENCES accreditations(id), + FOREIGN KEY (campus_name, dept_code) + REFERENCES departments(campus_name, dept_code) + ); + """) + fks = db["courses"].foreign_keys + assert len(fks) == 2 + assert {fk.is_compound for fk in fks} == {True, False} + fks_sorted = sorted(fks) + assert fks_sorted[0].other_table == "accreditations" + assert fks_sorted[1].other_table == "departments" + + +@pytest.fixture +def departments_db(): + db = Database(memory=True) + db.create_table( + "departments", + {"campus_name": str, "dept_code": str, "dept_name": str}, + pk=("campus_name", "dept_code"), + ) + return db + + +EXPECTED_COURSES_SCHEMA = ( + 'CREATE TABLE "courses" (\n' + ' "course_code" TEXT PRIMARY KEY,\n' + ' "campus_name" TEXT,\n' + ' "dept_code" TEXT,\n' + ' FOREIGN KEY ("campus_name", "dept_code") ' + 'REFERENCES "departments"("campus_name", "dept_code")\n' + ")" +) + + +@pytest.mark.parametrize( + "foreign_keys", + ( + [ + ForeignKey( + table="courses", + column=None, + other_table="departments", + other_column=None, + columns=("campus_name", "dept_code"), + other_columns=("campus_name", "dept_code"), + is_compound=True, + ) + ], + [(("campus_name", "dept_code"), "departments", ("campus_name", "dept_code"))], + # Two-item form guesses the other table's primary key: + [(("campus_name", "dept_code"), "departments")], + # Lists work too, though tuples are the documented form: + [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])], + ), +) +def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + foreign_keys=foreign_keys, + ) + assert departments_db["courses"].schema == EXPECTED_COURSES_SCHEMA + fks = departments_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ("campus_name", "dept_code") + assert fk.other_table == "departments" + assert fk.other_columns == ("campus_name", "dept_code") + + +def test_create_table_compound_foreign_key_enforced(departments_db): + departments_db.execute("PRAGMA foreign_keys = ON") + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + foreign_keys=[(("campus_name", "dept_code"), "departments")], + ) + departments_db["departments"].insert( + {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"} + ) + departments_db["courses"].insert( + {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"} + ) + with pytest.raises(sqlite3.IntegrityError): + departments_db.execute( + "insert into courses (course_code, campus_name, dept_code) " + "values ('X1', 'Nowhere', 'NOPE')" + ) + + +def test_create_table_compound_foreign_key_missing_other_column(departments_db): + with pytest.raises(AlterError): + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + foreign_keys=[ + (("campus_name", "dept_code"), "departments", ("campus_name", "nope")) + ], + ) + + +def test_transform_preserves_compound_foreign_key(compound_db): + compound_db["courses"].transform(rename={"course_name": "title"}) + fks = compound_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ("campus_name", "dept_code") + assert fk.other_table == "departments" + assert fk.other_columns == ("campus_name", "dept_code") + + +def test_transform_rename_member_column_updates_compound_foreign_key(compound_db): + compound_db["courses"].transform(rename={"campus_name": "campus"}) + fks = compound_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ("campus", "dept_code") + # Referenced columns in the other table are unchanged + assert fk.other_columns == ("campus_name", "dept_code") + + +def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): + # Matches single-column behavior: dropping the column silently + # drops the foreign key that used it + compound_db["courses"].transform(drop={"dept_code"}) + assert compound_db["courses"].foreign_keys == [] + assert "FOREIGN KEY" not in compound_db["courses"].schema + + +@pytest.mark.parametrize( + "drop_foreign_keys", + ( + # A bare column name matches any foreign key it participates in: + ["campus_name"], + # A tuple must match the full compound key: + [("campus_name", "dept_code")], + ), +) +def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys): + compound_db["courses"].transform(drop_foreign_keys=drop_foreign_keys) + assert compound_db["courses"].foreign_keys == [] + # The columns themselves survive + assert {"campus_name", "dept_code"} <= set( + compound_db["courses"].columns_dict.keys() + ) + + +@pytest.fixture +def courses_db(departments_db): + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + ) + return departments_db + + +def test_add_compound_foreign_key(courses_db): + t = courses_db["courses"].add_foreign_key( + ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") + ) + # Returns self + assert t.name == "courses" + fks = courses_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ("campus_name", "dept_code") + assert fk.other_table == "departments" + assert fk.other_columns == ("campus_name", "dept_code") + + +def test_add_compound_foreign_key_guesses_other_columns(courses_db): + # Lists work here too, though tuples are the documented form + courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") + fk = courses_db["courses"].foreign_keys[0] + assert fk.other_columns == ("campus_name", "dept_code") + + +def test_add_compound_foreign_key_error_if_already_exists(courses_db): + courses_db["courses"].add_foreign_key(("campus_name", "dept_code"), "departments") + with pytest.raises(AlterError) as ex: + courses_db["courses"].add_foreign_key( + ("campus_name", "dept_code"), "departments" + ) + assert "already exists" in ex.value.args[0] + # ignore=True should not raise + courses_db["courses"].add_foreign_key( + ("campus_name", "dept_code"), "departments", ignore=True + ) + + +def test_add_compound_foreign_key_error_if_column_missing(courses_db): + with pytest.raises(AlterError): + courses_db["courses"].add_foreign_key(("campus_name", "nope"), "departments") + + +def test_db_add_foreign_keys_compound(courses_db): + courses_db.add_foreign_keys( + [ + ( + "courses", + ("campus_name", "dept_code"), + "departments", + ("campus_name", "dept_code"), + ) + ] + ) + fk = courses_db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.columns == ("campus_name", "dept_code") + + +def test_index_foreign_keys_compound_creates_composite_index(compound_db): + compound_db.index_foreign_keys() + index_columns = [i.columns for i in compound_db["courses"].indexes] + assert ["campus_name", "dept_code"] in index_columns + # No separate single-column indexes for the members + assert ["campus_name"] not in index_columns + assert ["dept_code"] not in index_columns + + +def test_foreign_key_captures_on_delete_and_on_update(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + author_id INTEGER REFERENCES authors(id) + ON DELETE CASCADE ON UPDATE RESTRICT + ); + """) + fk = db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert fk.on_update == "RESTRICT" + + +def test_foreign_key_on_delete_defaults_to_no_action(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fk = fresh_db["books"].foreign_keys[0] + assert fk.on_delete == "NO ACTION" + assert fk.on_update == "NO ACTION" + + +def test_create_table_foreign_key_with_on_delete(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db.create_table( + "books", + {"id": int, "author_id": int}, + pk="id", + foreign_keys=[ + ForeignKey( + table="books", + column="author_id", + other_table="authors", + other_column="id", + on_delete="CASCADE", + ) + ], + ) + assert "ON DELETE CASCADE" in fresh_db["books"].schema + assert fresh_db["books"].foreign_keys[0].on_delete == "CASCADE" + + +def test_transform_preserves_on_delete_cascade(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + db["books"].transform(rename={"title": "book_title"}) + fk = db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert fk.on_update == "NO ACTION" + assert "ON DELETE CASCADE" in db["books"].schema + + +def test_transform_preserves_compound_foreign_key_on_delete(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + PRIMARY KEY (campus_name, dept_code) + ); + CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + FOREIGN KEY (campus_name, dept_code) + REFERENCES departments(campus_name, dept_code) ON DELETE CASCADE + ); + """) + db["courses"].transform(rename={"course_code": "code"}) + fk = db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.on_delete == "CASCADE" + assert "ON DELETE CASCADE" in db["courses"].schema + + +def test_implicit_primary_key_reference_is_resolved(): + # REFERENCES authors (no column) has "to" of None in the pragma - + # it should be resolved to the primary key of the other table + db = Database(memory=True) + db.executescript(""" + CREATE TABLE authors (author_id INTEGER PRIMARY KEY); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + author_id INTEGER REFERENCES authors + ); + """) + fk = db["books"].foreign_keys[0] + assert fk.is_compound is False + assert fk.other_column == "author_id" + assert fk.other_columns == ("author_id",) + + +def test_implicit_compound_primary_key_reference_is_resolved(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + PRIMARY KEY (campus_name, dept_code) + ); + CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + FOREIGN KEY (campus_name, dept_code) REFERENCES departments + ); + """) + fk = db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.other_columns == ("campus_name", "dept_code") + + +def test_foreign_key_normalizes_list_columns_to_tuples(): + # Compound columns passed as lists are normalized to tuples, so they + # compare equal to introspected ForeignKeys + fk = ForeignKey( + table="courses", + column=None, + other_table="departments", + other_column=None, + columns=["campus_name", "dept_code"], + other_columns=["campus_name", "dept_code"], + is_compound=True, + ) + assert fk.columns == ("campus_name", "dept_code") + assert fk.other_columns == ("campus_name", "dept_code") + + +def test_add_foreign_keys_preserves_actions(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/594 review finding: + # ForeignKey objects passed to db.add_foreign_keys() were flattened + # to plain tuples, losing on_delete/on_update + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.add_foreign_keys( + [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] + ) + fk = fresh_db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert "ON DELETE CASCADE" in fresh_db["books"].schema + + +def test_add_foreign_keys_preserves_actions_compound(courses_db): + courses_db.add_foreign_keys( + [ + ForeignKey( + table="courses", + column=None, + other_table="departments", + other_column=None, + columns=("campus_name", "dept_code"), + other_columns=("campus_name", "dept_code"), + is_compound=True, + on_delete="CASCADE", + ) + ] + ) + fk = courses_db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.on_delete == "CASCADE" + assert "ON DELETE CASCADE" in courses_db["courses"].schema + + +def test_add_foreign_key_on_delete_on_update(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db["books"].add_foreign_key( + "author_id", "authors", "id", on_delete="CASCADE", on_update="RESTRICT" + ) + fk = fresh_db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert fk.on_update == "RESTRICT" + assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db["books"].schema + # The cascade should actually fire + fresh_db.execute("PRAGMA foreign_keys = ON") + fresh_db.execute("delete from authors where id = 1") + assert fresh_db["books"].count == 0 + + +def test_add_compound_foreign_key_on_delete(courses_db): + courses_db["courses"].add_foreign_key( + ("campus_name", "dept_code"), "departments", on_delete="SET NULL" + ) + fk = courses_db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.on_delete == "SET NULL" + assert "ON DELETE SET NULL" in courses_db["courses"].schema + + +def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db): + # The other table's PRIMARY KEY declares its columns in a different + # order to the table's column order. SQLite resolves the implicit + # "REFERENCES other" using PRIMARY KEY declaration order, so the + # introspected other_columns must too + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db.execute( + "create table child (x text, y text, foreign key (x, y) references other)" + ) + fk = fresh_db["child"].foreign_keys[0] + assert fk.other_columns == ("a", "b") + + +def test_transform_implicit_compound_foreign_key_stays_valid(fresh_db): + # transform() rewrites the implicit FK with explicit columns - they + # must be in PRIMARY KEY declaration order or valid data fails the + # foreign key check with an IntegrityError + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db.execute( + "create table child (x text, y text, foreign key (x, y) references other)" + ) + fresh_db.execute("PRAGMA foreign_keys = ON") + fresh_db["other"].insert({"a": "A", "b": "B"}) + fresh_db["child"].insert({"x": "A", "y": "B"}) + fresh_db["child"].transform(types={"x": str}) + assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + # The constraint still points the right way around + fresh_db["child"].insert({"x": "A", "y": "B"}) + with pytest.raises(sqlite3.IntegrityError): + fresh_db["child"].insert({"x": "B", "y": "A"}) + + +def test_create_compound_foreign_key_guesses_pk_declaration_order(fresh_db): + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db["other"].insert({"a": "A", "b": "B"}) + fresh_db["child"].create( + {"id": int, "x": str, "y": str}, + pk="id", + foreign_keys=[(("x", "y"), "other")], + ) + assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + fresh_db.execute("PRAGMA foreign_keys = ON") + fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}) + with pytest.raises(sqlite3.IntegrityError): + fresh_db["child"].insert({"id": 2, "x": "B", "y": "A"}) + + +def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db): + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id") + fresh_db["child"].add_foreign_key(("x", "y"), "other") + assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + + +def test_foreign_keys_are_hashable(fresh_db): + # set() over foreign_keys worked with the 3.x namedtuple and must + # keep working with the dataclass + fresh_db["p"].insert({"id": 1}, pk="id") + fresh_db["c"].insert( + {"id": 1, "pid": 1}, pk="id", foreign_keys=[("pid", "p", "id")] + ) + fks = set(fresh_db["c"].foreign_keys) + assert len(fks) == 1 + assert ForeignKey("c", "pid", "p", "id") in fks + # Usable as dict keys too + assert {fk: True for fk in fks} + + +def test_foreign_key_is_immutable(): + import dataclasses + + fk = ForeignKey("c", "pid", "p", "id") + with pytest.raises(dataclasses.FrozenInstanceError): + fk.table = "other" + + +def test_foreign_key_equality_and_hash_include_actions(): + # Two foreign keys differing only in ON DELETE behavior are different + # constraints - they compare unequal and hash separately + plain = ForeignKey("c", "pid", "p", "id") + cascade = ForeignKey("c", "pid", "p", "id", on_delete="CASCADE") + assert plain != cascade + assert len({plain, cascade}) == 2 + assert plain == ForeignKey("c", "pid", "p", "id") + + +def test_create_table_mixed_foreign_keys_list(fresh_db): + # 3.x accepted a mix of ForeignKey objects, tuples and bare column + # strings in foreign_keys= (ForeignKey was a namedtuple, so it passed + # the tuple check) - keep accepting the mix + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["publishers"].insert({"id": 1}, pk="id") + fresh_db["books"].create( + {"id": int, "author_id": int, "publisher_id": int}, + pk="id", + foreign_keys=[ + ForeignKey("books", "author_id", "authors", "id"), + ("publisher_id", "publishers", "id"), + ], + ) + fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + assert fks == {"author_id": "authors", "publisher_id": "publishers"} + + +def test_create_table_mixed_foreign_keys_with_string(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["publishers"].insert({"id": 1}, pk="id") + fresh_db["books"].create( + {"id": int, "author_id": int, "publisher_id": int}, + pk="id", + foreign_keys=[ + "author_id", # bare column, table and column guessed + ("publisher_id", "publishers", "id"), + ], + ) + fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + assert fks == {"author_id": "authors", "publisher_id": "publishers"} + + +def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db): + # Requesting an existing foreign key with different ON DELETE/ON UPDATE + # actions was silently skipped, dropping the requested change + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert( + {"id": 1, "author_id": 1}, + pk="id", + foreign_keys=[("author_id", "authors", "id")], + ) + with pytest.raises(AlterError) as ex: + fresh_db.add_foreign_keys( + [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] + ) + assert "ON DELETE" in str(ex.value) + assert fresh_db["books"].foreign_keys[0].on_delete == "NO ACTION" + + +def test_add_foreign_keys_identical_existing_is_noop(fresh_db): + # An exact match, including actions, is silently skipped so repeated + # calls stay idempotent + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db["books"].add_foreign_key("author_id", "authors", "id", on_delete="CASCADE") + fresh_db.add_foreign_keys( + [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] + ) + fks = fresh_db["books"].foreign_keys + assert len(fks) == 1 + assert fks[0].on_delete == "CASCADE" + + +def test_add_foreign_keys_compound_column_count_mismatch_errors(fresh_db): + # Previously the extra other-column was silently discarded, creating + # a single-column foreign key to just ("id") + fresh_db["departments"].insert( + {"campus": "north", "code": "cs"}, pk=("campus", "code") + ) + fresh_db["courses"].insert({"id": 1, "campus": "north"}, pk="id") + with pytest.raises(ValueError) as ex: + fresh_db.add_foreign_keys( + [("courses", ("campus",), "departments", ("campus", "code"))] + ) + assert "same number of columns" in str(ex.value) + assert fresh_db["courses"].foreign_keys == [] diff --git a/tests/test_fts.py b/tests/test_fts.py index a3efa54..64ec645 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -1,6 +1,7 @@ import pytest from sqlite_utils import Database from sqlite_utils.utils import sqlite3 +from unittest.mock import ANY search_records = [ { @@ -82,6 +83,20 @@ def test_enable_fts_escape_table_names(fresh_db): assert [] == list(table.search("bar")) +def test_search_duplicate_columns_are_deduped(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/624 + table = fresh_db["t"] + table.insert_all(search_records) + table.enable_fts(["text", "country"], fts_version="FTS4") + rows = list(table.search("tanuki", columns=["text", "text"])) + assert rows == [ + { + "text": "tanuki are running tricksters", + "text_2": "tanuki are running tricksters", + } + ] + + def test_search_limit_offset(fresh_db): table = fresh_db["t"] table.insert_all(search_records) @@ -94,6 +109,64 @@ def test_search_limit_offset(fresh_db): ) +@pytest.mark.parametrize("fts_version", ("FTS4", "FTS5")) +def test_search_where(fresh_db, fts_version): + table = fresh_db["t"] + table.insert_all(search_records) + table.enable_fts(["text", "country"], fts_version=fts_version) + results = list( + table.search("are", where="country = :country", where_args={"country": "Japan"}) + ) + assert results == [ + { + "rowid": 1, + "text": "tanuki are running tricksters", + "country": "Japan", + "not_searchable": "foo", + } + ] + + +def test_search_where_args_disallows_query(fresh_db): + table = fresh_db["t"] + with pytest.raises(ValueError) as ex: + list( + table.search( + "x", where="author = :query", where_args={"query": "not allowed"} + ) + ) + assert ( + ex.value.args[0] + == "'query' is a reserved key and cannot be passed to where_args for .search()" + ) + + +def test_search_include_rank(fresh_db): + table = fresh_db["t"] + table.insert_all(search_records) + table.enable_fts(["text", "country"], fts_version="FTS5") + results = list(table.search("are", include_rank=True)) + assert results == [ + { + "rowid": 1, + "text": "tanuki are running tricksters", + "country": "Japan", + "not_searchable": "foo", + "rank": ANY, + }, + { + "rowid": 2, + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + "rank": ANY, + }, + ] + assert isinstance(results[0]["rank"], float) + assert isinstance(results[1]["rank"], float) + assert results[0]["rank"] < results[1]["rank"] + + def test_enable_fts_table_names_containing_spaces(fresh_db): table = fresh_db["test"] table.insert({"column with spaces": "in its name"}) @@ -148,32 +221,32 @@ def test_populate_fts_escape_table_names(fresh_db): ] == list(table.search("usa")) -def test_fts_tokenize(fresh_db): - for fts_version in ("4", "5"): - table_name = "searchable_{}".format(fts_version) - table = fresh_db[table_name] - table.insert_all(search_records) - # Test without porter stemming - table.enable_fts( - ["text", "country"], - fts_version="FTS{}".format(fts_version), - ) - assert [] == list(table.search("bite")) - # Test WITH stemming - table.disable_fts() - table.enable_fts( - ["text", "country"], - fts_version="FTS{}".format(fts_version), - tokenize="porter", - ) - rows = list(table.search("bite", order_by="rowid")) - assert len(rows) == 1 - assert { - "rowid": 2, - "text": "racoons are biting trash pandas", - "country": "USA", - "not_searchable": "bar", - }.items() <= rows[0].items() +@pytest.mark.parametrize("fts_version", ("4", "5")) +def test_fts_tokenize(fresh_db, fts_version): + table_name = "searchable_{}".format(fts_version) + table = fresh_db[table_name] + table.insert_all(search_records) + # Test without porter stemming + table.enable_fts( + ["text", "country"], + fts_version="FTS{}".format(fts_version), + ) + assert [] == list(table.search("bite")) + # Test WITH stemming + table.disable_fts() + table.enable_fts( + ["text", "country"], + fts_version="FTS{}".format(fts_version), + tokenize="porter", + ) + rows = list(table.search("bite", order_by="rowid")) + assert len(rows) == 1 + assert { + "rowid": 2, + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + }.items() <= rows[0].items() def test_optimize_fts(fresh_db): @@ -254,13 +327,12 @@ def test_disable_fts(fresh_db, create_triggers): assert ["searchable"] == fresh_db.table_names() -@pytest.mark.parametrize("table_to_fix", ["searchable", "searchable_fts"]) -def test_rebuild_fts(fresh_db, table_to_fix): +def test_rebuild_fts(fresh_db): table = fresh_db["searchable"] table.insert(search_records[0]) table.enable_fts(["text", "country"]) # Run a search - rows = list(table.search("tanuki")) + rows = list(table.search("are")) assert len(rows) == 1 assert { "rowid": 1, @@ -268,21 +340,32 @@ def test_rebuild_fts(fresh_db, table_to_fix): "country": "Japan", "not_searchable": "foo", }.items() <= rows[0].items() - # Delete from searchable_fts_data - fresh_db["searchable_fts_data"].delete_where() - # This should have broken the index - with pytest.raises(sqlite3.DatabaseError): - list(table.search("tanuki")) + # Insert another record + table.insert(search_records[1]) + # This should NOT show up in searches + assert len(list(table.search("are"))) == 1 # Running rebuild_fts() should fix it - fresh_db[table_to_fix].rebuild_fts() - rows2 = list(table.search("tanuki")) - assert len(rows2) == 1 - assert { - "rowid": 1, - "text": "tanuki are running tricksters", - "country": "Japan", - "not_searchable": "foo", - }.items() <= rows2[0].items() + table.rebuild_fts() + rows2 = list(table.search("are")) + assert len(rows2) == 2 + + +@pytest.mark.parametrize("method", ["optimize", "rebuild_fts"]) +def test_optimize_and_rebuild_fts_commit(tmpdir, method): + path = str(tmpdir / "test.db") + db = Database(path) + table = db["searchable"] + table.insert(search_records[0]) + table.enable_fts(["text", "country"]) + getattr(table, method)() + # The connection must not be left inside an open transaction, + # otherwise this and all subsequent writes are lost on close + assert not db.conn.in_transaction + table.insert(search_records[1]) + db.close() + db2 = Database(path) + assert db2["searchable"].count == 2 + db2.close() @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"]) @@ -369,12 +452,35 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): assert all(q[0].startswith("select ") for q in queries) -def test_enable_fts_error_message_on_views(): +def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table(): + db = Database(memory=True) + db["books"].insert( + { + "id": 1, + "title": "Habits of Australian Marsupials", + "author": "Marlee Hawkins", + }, + pk="id", + ) + db.executescript(""" + CREATE VIRTUAL TABLE [books_fts] USING FTS5 ( + [title], + content=[books] + ); + """) + + db["books"].enable_fts(["title", "author"], replace=True) + + assert db["books_fts"].columns_dict.keys() == {"title", "author"} + assert 'content="books"' in db["books_fts"].schema + + +def test_view_has_no_enable_fts(): db = Database(memory=True) db.create_view("hello", "select 1 + 1") - with pytest.raises(NotImplementedError) as e: - db["hello"].enable_fts() - assert e.value.args[0] == "enable_fts() is supported on tables but not on views" + # Views deliberately do not have an enable_fts() method + with pytest.raises(AttributeError): + db["hello"].enable_fts() # type: ignore[union-attr] @pytest.mark.parametrize( @@ -384,85 +490,107 @@ def test_enable_fts_error_message_on_views(): {}, "FTS5", ( - "with original as (\n" + 'with "original" as (\n' " select\n" " rowid,\n" " *\n" - " from [books]\n" + ' from "books"\n' ")\n" "select\n" - " [original].*\n" + ' "original".*\n' "from\n" - " [original]\n" - " join [books_fts] on [original].rowid = [books_fts].rowid\n" + ' "original"\n' + ' join "books_fts" on "original".rowid = "books_fts".rowid\n' "where\n" - " [books_fts] match :query\n" + ' "books_fts" match :query\n' "order by\n" - " [books_fts].rank" + ' "books_fts".rank' ), ), ( {"columns": ["title"], "order_by": "rowid", "limit": 10}, "FTS5", ( - "with original as (\n" + 'with "original" as (\n' " select\n" " rowid,\n" - " [title]\n" - " from [books]\n" + ' "title"\n' + ' from "books"\n' ")\n" "select\n" - " [original].[title]\n" + ' "original"."title"\n' "from\n" - " [original]\n" - " join [books_fts] on [original].rowid = [books_fts].rowid\n" + ' "original"\n' + ' join "books_fts" on "original".rowid = "books_fts".rowid\n' "where\n" - " [books_fts] match :query\n" + ' "books_fts" match :query\n' "order by\n" " rowid\n" "limit 10" ), ), + ( + {"where": "author = :author"}, + "FTS5", + ( + 'with "original" as (\n' + " select\n" + " rowid,\n" + " *\n" + ' from "books"\n' + " where author = :author\n" + ")\n" + "select\n" + ' "original".*\n' + "from\n" + ' "original"\n' + ' join "books_fts" on "original".rowid = "books_fts".rowid\n' + "where\n" + ' "books_fts" match :query\n' + "order by\n" + ' "books_fts".rank' + ), + ), ( {"columns": ["title"]}, "FTS4", ( - "with original as (\n" + 'with "original" as (\n' " select\n" " rowid,\n" - " [title]\n" - " from [books]\n" + ' "title"\n' + ' from "books"\n' ")\n" "select\n" - " [original].[title]\n" + ' "original"."title"\n' "from\n" - " [original]\n" - " join [books_fts] on [original].rowid = [books_fts].rowid\n" + ' "original"\n' + ' join "books_fts" on "original".rowid = "books_fts".rowid\n' "where\n" - " [books_fts] match :query\n" + ' "books_fts" match :query\n' "order by\n" - " rank_bm25(matchinfo([books_fts], 'pcnalx'))" + " rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))" ), ), ( {"offset": 1, "limit": 1}, "FTS4", ( - "with original as (\n" + 'with "original" as (\n' " select\n" " rowid,\n" " *\n" - " from [books]\n" + ' from "books"\n' ")\n" "select\n" - " [original].*\n" + ' "original".*\n' "from\n" - " [original]\n" - " join [books_fts] on [original].rowid = [books_fts].rowid\n" + ' "original"\n' + ' join "books_fts" on "original".rowid = "books_fts".rowid\n' "where\n" - " [books_fts] match :query\n" + ' "books_fts" match :query\n' "order by\n" - " rank_bm25(matchinfo([books_fts], 'pcnalx'))\n" + " rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))\n" "limit 1 offset 1" ), ), @@ -470,24 +598,90 @@ def test_enable_fts_error_message_on_views(): {"limit": 2}, "FTS4", ( - "with original as (\n" + 'with "original" as (\n' " select\n" " rowid,\n" " *\n" - " from [books]\n" + ' from "books"\n' ")\n" "select\n" - " [original].*\n" + ' "original".*\n' "from\n" - " [original]\n" - " join [books_fts] on [original].rowid = [books_fts].rowid\n" + ' "original"\n' + ' join "books_fts" on "original".rowid = "books_fts".rowid\n' "where\n" - " [books_fts] match :query\n" + ' "books_fts" match :query\n' "order by\n" - " rank_bm25(matchinfo([books_fts], 'pcnalx'))\n" + " rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))\n" "limit 2" ), ), + ( + {"where": "author = :author"}, + "FTS4", + ( + 'with "original" as (\n' + " select\n" + " rowid,\n" + " *\n" + ' from "books"\n' + " where author = :author\n" + ")\n" + "select\n" + ' "original".*\n' + "from\n" + ' "original"\n' + ' join "books_fts" on "original".rowid = "books_fts".rowid\n' + "where\n" + ' "books_fts" match :query\n' + "order by\n" + " rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))" + ), + ), + ( + {"include_rank": True}, + "FTS5", + ( + 'with "original" as (\n' + " select\n" + " rowid,\n" + " *\n" + ' from "books"\n' + ")\n" + "select\n" + ' "original".*,\n' + ' "books_fts".rank rank\n' + "from\n" + ' "original"\n' + ' join "books_fts" on "original".rowid = "books_fts".rowid\n' + "where\n" + ' "books_fts" match :query\n' + "order by\n" + ' "books_fts".rank' + ), + ), + ( + {"include_rank": True}, + "FTS4", + ( + 'with "original" as (\n' + " select\n" + " rowid,\n" + " *\n" + ' from "books"\n' + ")\n" + "select\n" + ' "original".*,\n' + " rank_bm25(matchinfo(\"books_fts\", 'pcnalx')) rank\n" + "from\n" + ' "original"\n' + ' join "books_fts" on "original".rowid = "books_fts".rowid\n' + "where\n" + ' "books_fts" match :query\n' + "order by\n" + " rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))" + ), + ), ], ) def test_search_sql(kwargs, fts, expected): @@ -501,3 +695,54 @@ def test_search_sql(kwargs, fts, expected): db["books"].enable_fts(["title", "author"], fts_version=fts) sql = db["books"].search_sql(**kwargs) assert sql == expected + + +@pytest.mark.parametrize( + "input,expected", + ( + ("dog", '"dog"'), + ("cat,", '"cat,"'), + ("cat's", '"cat\'s"'), + ("dog.", '"dog."'), + ("cat dog", '"cat" "dog"'), + # If a phrase is already double quoted, leave it so + ('"cat dog"', '"cat dog"'), + ('"cat dog" fish', '"cat dog" "fish"'), + # Sensibly handle unbalanced double quotes + ('cat"', '"cat"'), + ('"cat dog" "fish', '"cat dog" "fish"'), + ), +) +def test_quote_fts_query(fresh_db, input, expected): + table = fresh_db["searchable"] + table.insert_all(search_records) + table.enable_fts(["text", "country"]) + quoted = fresh_db.quote_fts(input) + assert quoted == expected + # Executing query does not crash. + list(table.search(quoted)) + + +def test_search_quote(fresh_db): + table = fresh_db["searchable"] + table.insert_all(search_records) + table.enable_fts(["text", "country"]) + query = "cat's" + with pytest.raises(sqlite3.OperationalError): + list(table.search(query)) + # No exception with quote=True + list(table.search(query, quote=True)) + + +def test_enable_fts_cli_on_view_errors(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["t"].insert({"text": "hello"}) + db.create_view("v", "select * from t") + db.close() + from click.testing import CliRunner + from sqlite_utils import cli as cli_module + + result = CliRunner().invoke(cli_module.cli, ["enable-fts", db_path, "v", "text"]) + assert result.exit_code == 1 + assert result.output.strip() == "Error: Table v is actually a view" diff --git a/tests/test_gis.py b/tests/test_gis.py new file mode 100644 index 0000000..f39554e --- /dev/null +++ b/tests/test_gis.py @@ -0,0 +1,236 @@ +import json +import pytest + +from click.testing import CliRunner +from sqlite_utils.cli import cli +from sqlite_utils.db import Database +from sqlite_utils.utils import find_spatialite, sqlite3 + +pytestmark = [ + pytest.mark.skipif( + not find_spatialite(), reason="Could not find SpatiaLite extension" + ), + pytest.mark.skipif( + not hasattr(sqlite3.Connection, "enable_load_extension"), + reason="sqlite3.Connection missing enable_load_extension", + ), +] + + +# python API tests +def test_find_spatialite(): + spatialite = find_spatialite() + assert spatialite is None or isinstance(spatialite, str) + + +def test_init_spatialite(): + db = Database(memory=True) + spatialite = find_spatialite() + db.init_spatialite(spatialite) + assert "spatial_ref_sys" in db.table_names() + + +def test_add_geometry_column(): + db = Database(memory=True) + spatialite = find_spatialite() + db.init_spatialite(spatialite) + + # create a table first + table = db.create_table("locations", {"id": str, "properties": str}) + table.add_geometry_column( + column_name="geometry", + geometry_type="Point", + srid=4326, + coord_dimension="XY", + ) + + assert db["geometry_columns"].get(["locations", "geometry"]) == { + "f_table_name": "locations", + "f_geometry_column": "geometry", + "geometry_type": 1, # point + "coord_dimension": 2, + "srid": 4326, + "spatial_index_enabled": 0, + } + + +def test_create_spatial_index(): + db = Database(memory=True) + spatialite = find_spatialite() + assert db.init_spatialite(spatialite) + + # create a table, add a geometry column with default values + table = db.create_table("locations", {"id": str, "properties": str}) + assert table.add_geometry_column("geometry", "Point") + + # index it + assert table.create_spatial_index("geometry") + + assert "idx_locations_geometry" in db.table_names() + + +def test_double_create_spatial_index(): + db = Database(memory=True) + spatialite = find_spatialite() + db.init_spatialite(spatialite) + + # create a table, add a geometry column with default values + table = db.create_table("locations", {"id": str, "properties": str}) + table.add_geometry_column("geometry", "Point") + + # index it, return True + assert table.create_spatial_index("geometry") + + assert "idx_locations_geometry" in db.table_names() + + # call it again, return False + assert not table.create_spatial_index("geometry") + + +# cli tests +@pytest.mark.parametrize("use_spatialite_shortcut", [True, False]) +def test_query_load_extension(use_spatialite_shortcut): + # Without --load-extension: + result = CliRunner().invoke(cli, [":memory:", "select spatialite_version()"]) + assert result.exit_code == 1 + assert "no such function: spatialite_version" in result.output + # With --load-extension: + if use_spatialite_shortcut: + load_extension = "spatialite" + else: + load_extension = find_spatialite() + result = CliRunner().invoke( + cli, + [ + ":memory:", + "select spatialite_version()", + "--load-extension={}".format(load_extension), + ], + ) + assert result.exit_code == 0, result.stdout + assert ["spatialite_version()"] == list(json.loads(result.output)[0].keys()) + + +def test_cli_create_spatialite(tmpdir): + # sqlite-utils create test.db --init-spatialite + db_path = tmpdir / "created.db" + result = CliRunner().invoke( + cli, ["create-database", str(db_path), "--init-spatialite"] + ) + + assert result.exit_code == 0 + assert db_path.exists() + assert db_path.read_binary()[:16] == b"SQLite format 3\x00" + + db = Database(str(db_path)) + assert "spatial_ref_sys" in db.table_names() + + +def test_cli_add_geometry_column(tmpdir): + # create a rowid table with one column + db_path = tmpdir / "spatial.db" + db = Database(str(db_path)) + db.init_spatialite() + + table = db["locations"].create({"name": str}) + + result = CliRunner().invoke( + cli, + [ + "add-geometry-column", + str(db_path), + table.name, + "geometry", + "--type", + "POINT", + ], + ) + + assert result.exit_code == 0 + + assert db["geometry_columns"].get(["locations", "geometry"]) == { + "f_table_name": "locations", + "f_geometry_column": "geometry", + "geometry_type": 1, # point + "coord_dimension": 2, + "srid": 4326, + "spatial_index_enabled": 0, + } + + +def test_cli_add_geometry_column_options(tmpdir): + # create a rowid table with one column + db_path = tmpdir / "spatial.db" + db = Database(str(db_path)) + db.init_spatialite() + table = db["locations"].create({"name": str}) + + result = CliRunner().invoke( + cli, + [ + "add-geometry-column", + str(db_path), + table.name, + "geometry", + "-t", + "POLYGON", + "--srid", + "3857", # https://epsg.io/3857 + "--not-null", + ], + ) + + assert result.exit_code == 0 + + assert db["geometry_columns"].get(["locations", "geometry"]) == { + "f_table_name": "locations", + "f_geometry_column": "geometry", + "geometry_type": 3, # polygon + "coord_dimension": 2, + "srid": 3857, + "spatial_index_enabled": 0, + } + + column = table.columns[1] + assert column.notnull + + +def test_cli_add_geometry_column_invalid_type(tmpdir): + # create a rowid table with one column + db_path = tmpdir / "spatial.db" + db = Database(str(db_path)) + db.init_spatialite() + + table = db["locations"].create({"name": str}) + + result = CliRunner().invoke( + cli, + [ + "add-geometry-column", + str(db_path), + table.name, + "geometry", + "--type", + "NOT-A-TYPE", + ], + ) + + assert 2 == result.exit_code + + +def test_cli_create_spatial_index(tmpdir): + # create a rowid table with one column + db_path = tmpdir / "spatial.db" + db = Database(str(db_path)) + db.init_spatialite() + + table = db["locations"].create({"name": str}) + table.add_geometry_column("geometry", "POINT") + + result = CliRunner().invoke( + cli, ["create-spatial-index", str(db_path), table.name, "geometry"] + ) + + assert result.exit_code == 0 + + assert "idx_locations_geometry" in db.table_names() diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 1e30a8d..88e49a8 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -3,10 +3,18 @@ from click.testing import CliRunner import os import pathlib import pytest +import sys @pytest.mark.parametrize("silent", (False, True)) -def test_insert_files(silent): +@pytest.mark.parametrize( + "pk_args,expected_pks", + ( + (["--pk", "path"], ["path"]), + (["--pk", "path", "--pk", "name"], ["path", "name"]), + ), +) +def test_insert_files(silent, pk_args, expected_pks): runner = CliRunner() with runner.isolated_filesystem(): tmpdir = pathlib.Path(".") @@ -14,7 +22,7 @@ def test_insert_files(silent): (tmpdir / "one.txt").write_text("This is file one", "utf-8") (tmpdir / "two.txt").write_text("Two is shorter", "utf-8") (tmpdir / "nested").mkdir() - (tmpdir / "nested" / "three.txt").write_text("Three is nested", "utf-8") + (tmpdir / "nested" / "three.zz.txt").write_text("Three is nested", "utf-8") coltypes = ( "name", "path", @@ -23,6 +31,7 @@ def test_insert_files(silent): "md5", "mode", "content", + "content_text", "mtime", "ctime", "mtime_int", @@ -30,6 +39,8 @@ def test_insert_files(silent): "mtime_iso", "ctime_iso", "size", + "suffix", + "stem", ) cols = [] for coltype in coltypes: @@ -38,7 +49,7 @@ def test_insert_files(silent): cli.cli, ["insert-files", db_path, "files", str(tmpdir)] + cols - + ["--pk", "path"] + + pk_args + (["--silent"] if silent else []), catch_exceptions=False, ) @@ -48,31 +59,40 @@ def test_insert_files(silent): one, two, three = ( rows_by_path["one.txt"], rows_by_path["two.txt"], - rows_by_path[os.path.join("nested", "three.txt")], + rows_by_path[os.path.join("nested", "three.zz.txt")], ) assert { "content": b"This is file one", + "content_text": "This is file one", "md5": "556dfb57fce9ca301f914e2273adf354", "name": "one.txt", "path": "one.txt", "sha256": "e34138f26b5f7368f298b4e736fea0aad87ddec69fbd04dc183b20f4d844bad5", "size": 16, + "stem": "one", + "suffix": ".txt", }.items() <= one.items() assert { "content": b"Two is shorter", + "content_text": "Two is shorter", "md5": "f86f067b083af1911043eb215e74ac70", "name": "two.txt", "path": "two.txt", "sha256": "9368988ed16d4a2da0af9db9b686d385b942cb3ffd4e013f43aed2ec041183d9", "size": 14, + "stem": "two", + "suffix": ".txt", }.items() <= two.items() assert { "content": b"Three is nested", + "content_text": "Three is nested", "md5": "12580f341781f5a5b589164d3cd39523", - "name": "three.txt", - "path": os.path.join("nested", "three.txt"), + "name": "three.zz.txt", + "path": os.path.join("nested", "three.zz.txt"), "sha256": "6dd45aaaaa6b9f96af19363a92c8fca5d34791d3c35c44eb19468a6a862cc8cd", "size": 15, + "stem": "three.zz", + "suffix": ".txt", }.items() <= three.items() # Assert the other int/str/float columns exist and are of the right types expected_types = { @@ -84,24 +104,68 @@ def test_insert_files(silent): "mtime_iso": str, "mode": int, "fullpath": str, + "content": bytes, + "content_text": str, + "stem": str, + "suffix": str, } for colname, expected_type in expected_types.items(): for row in (one, two, three): assert isinstance(row[colname], expected_type) + assert set(db["files"].pks) == set(expected_pks) -def test_insert_files_stdin(): +@pytest.mark.parametrize( + "use_text,encoding,input,expected", + ( + (False, None, "hello world", b"hello world"), + (True, None, "hello world", "hello world"), + (False, None, b"S\xe3o Paulo", b"S\xe3o Paulo"), + (True, "latin-1", b"S\xe3o Paulo", "S\xe3o Paulo"), + ), +) +def test_insert_files_stdin(use_text, encoding, input, expected): runner = CliRunner() with runner.isolated_filesystem(): tmpdir = pathlib.Path(".") db_path = str(tmpdir / "files.db") + args = ["insert-files", db_path, "files", "-", "--name", "stdin-name"] + if use_text: + args += ["--text"] + if encoding is not None: + args += ["--encoding", encoding] result = runner.invoke( cli.cli, - ["insert-files", db_path, "files", "-", "--name", "stdin-name"], + args, catch_exceptions=False, - input="hello world", + input=input, ) assert result.exit_code == 0, result.stdout db = Database(db_path) row = list(db["files"].rows)[0] - assert {"path": "stdin-name", "content": b"hello world", "size": 11} == row + key = "content" + if use_text: + key = "content_text" + assert {"path": "stdin-name", key: expected}.items() <= row.items() + + +@pytest.mark.skipif( + sys.platform.startswith("win"), + reason="Windows has a different way of handling default encodings", +) +def test_insert_files_bad_text_encoding_error(): + runner = CliRunner() + with runner.isolated_filesystem(): + tmpdir = pathlib.Path(".") + latin = tmpdir / "latin.txt" + latin.write_bytes(b"S\xe3o Paulo") + db_path = str(tmpdir / "files.db") + result = runner.invoke( + cli.cli, + ["insert-files", db_path, "files", str(latin), "--text"], + catch_exceptions=False, + ) + assert result.exit_code == 1, result.output + assert result.output.strip().startswith( + "Error: Could not read file '{}' as text".format(str(latin.resolve())) + ) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index dce8afc..8b6765d 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -2,6 +2,14 @@ from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn import pytest +def _check_supports_strict(): + """Check if SQLite supports strict tables without leaking the database.""" + db = Database(memory=True) + result = db.supports_strict + db.close() + return result + + def test_table_names(existing_db): assert ["foo"] == existing_db.table_names() @@ -36,19 +44,36 @@ def test_detect_fts(existing_db): assert existing_db["foo"].detect_fts() is None +@pytest.mark.parametrize("reverse_order", (True, False)) +def test_detect_fts_similar_tables(fresh_db, reverse_order): + # https://github.com/simonw/sqlite-utils/issues/434 + table1, table2 = ("demo", "demo2") + if reverse_order: + table1, table2 = table2, table1 + + fresh_db[table1].insert({"title": "Hello"}).enable_fts( + ["title"], fts_version="FTS4" + ) + fresh_db[table2].insert({"title": "Hello"}).enable_fts( + ["title"], fts_version="FTS4" + ) + assert fresh_db[table1].detect_fts() == "{}_fts".format(table1) + assert fresh_db[table2].detect_fts() == "{}_fts".format(table2) + + def test_tables(existing_db): - assert 1 == len(existing_db.tables) - assert "foo" == existing_db.tables[0].name + assert len(existing_db.tables) == 1 + assert existing_db.tables[0].name == "foo" def test_views(fresh_db): fresh_db.create_view("foo_view", "select 1") - assert 1 == len(fresh_db.views) + assert len(fresh_db.views) == 1 view = fresh_db.views[0] assert isinstance(view, View) - assert "foo_view" == view.name - assert "" == repr(view) - assert {"1": str} == view.columns_dict + assert view.name == "foo_view" + assert repr(view) == "" + assert view.columns_dict == {"1": str} def test_count(existing_db): @@ -84,13 +109,11 @@ def test_table_repr(fresh_db): def test_indexes(fresh_db): - fresh_db.executescript( - """ + fresh_db.executescript(""" create table Gosh (c1 text, c2 text, c3 text); create index Gosh_c1 on Gosh(c1); create index Gosh_c2c3 on Gosh(c2, c3); - """ - ) + """) assert [ Index( seq=0, @@ -105,13 +128,11 @@ def test_indexes(fresh_db): def test_xindexes(fresh_db): - fresh_db.executescript( - """ + fresh_db.executescript(""" create table Gosh (c1 text, c2 text, c3 text); create index Gosh_c1 on Gosh(c1); create index Gosh_c2c3 on Gosh(c2, c3 desc); - """ - ) + """) assert fresh_db["Gosh"].xindexes == [ XIndex( name="Gosh_c2c3", @@ -183,19 +204,19 @@ def test_triggers_and_triggers_dict(fresh_db): } expected_triggers = { "authors_ai": ( - "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n" - " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\n" + 'CREATE TRIGGER "authors_ai" AFTER INSERT ON "authors" BEGIN\n' + ' INSERT INTO "authors_fts" (rowid, "name", "famous_works") VALUES (new.rowid, new."name", new."famous_works");\n' "END" ), "authors_ad": ( - "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n" - " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" + 'CREATE TRIGGER "authors_ad" AFTER DELETE ON "authors" BEGIN\n' + ' INSERT INTO "authors_fts" ("authors_fts", rowid, "name", "famous_works") VALUES(\'delete\', old.rowid, old."name", old."famous_works");\n' "END" ), "authors_au": ( - "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n" - " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" - " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND" + 'CREATE TRIGGER "authors_au" AFTER UPDATE ON "authors" BEGIN\n' + ' INSERT INTO "authors_fts" ("authors_fts", rowid, "name", "famous_works") VALUES(\'delete\', old.rowid, old."name", old."famous_works");\n' + ' INSERT INTO "authors_fts" (rowid, "name", "famous_works") VALUES (new.rowid, new."name", new."famous_works");\nEND' ), } assert authors.triggers_dict == expected_triggers @@ -252,15 +273,66 @@ def test_has_counts_triggers(fresh_db): ), ], ) -def test_virtual_table_using(sql, expected_name, expected_using): - db = Database(memory=True) - db.execute(sql) - assert db[expected_name].virtual_table_using == expected_using +def test_virtual_table_using(fresh_db, sql, expected_name, expected_using): + fresh_db.execute(sql) + assert fresh_db[expected_name].virtual_table_using == expected_using -def test_use_rowid(): - db = Database(memory=True) - db["rowid_table"].insert({"name": "Cleo"}) - db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") - assert db["rowid_table"].use_rowid - assert not db["regular_table"].use_rowid +def test_use_rowid(fresh_db): + fresh_db["rowid_table"].insert({"name": "Cleo"}) + fresh_db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") + assert fresh_db["rowid_table"].use_rowid + assert not fresh_db["regular_table"].use_rowid + + +@pytest.mark.skipif( + not _check_supports_strict(), + reason="Needs SQLite version that supports strict", +) +@pytest.mark.parametrize( + "create_table,expected_strict", + ( + ("create table t (id integer) strict", True), + ("create table t (id integer) STRICT", True), + ("create table t (id integer primary key) StriCt, WITHOUT ROWID", True), + ("create table t (id integer primary key) WITHOUT ROWID", False), + ("create table t (id integer)", False), + ), +) +def test_table_strict(fresh_db, create_table, expected_strict): + fresh_db.execute(create_table) + table = fresh_db["t"] + assert table.strict == expected_strict + + +@pytest.mark.parametrize( + "value", + ( + 1, + 1.3, + "foo", + True, + b"binary", + ), +) +def test_table_default_values(fresh_db, value): + fresh_db["default_values"].insert( + {"nodefault": 1, "value": value}, defaults={"value": value} + ) + default_values = fresh_db["default_values"].default_values + assert default_values == {"value": value} + + +def test_pks_use_primary_key_declaration_order(fresh_db): + # PRIMARY KEY (a, b) declared against columns stored in order (b, a) - + # pks must follow the declaration order, which is what SQLite uses to + # resolve implicit foreign key references and compound pk lookups + fresh_db.execute("create table t (b text, a text, primary key (a, b))") + assert fresh_db["t"].pks == ["a", "b"] + + +def test_transform_preserves_compound_pk_declaration_order(fresh_db): + fresh_db.execute("create table t (a text, b text, c text, primary key (b, a))") + fresh_db["t"].transform(drop={"c"}) + assert fresh_db["t"].pks == ["b", "a"] + assert 'PRIMARY KEY ("b", "a")' in fresh_db["t"].schema diff --git a/tests/test_list_mode.py b/tests/test_list_mode.py new file mode 100644 index 0000000..746c9c1 --- /dev/null +++ b/tests/test_list_mode.py @@ -0,0 +1,288 @@ +""" +Tests for list-based iteration in insert_all and upsert_all +""" + +import pytest +from sqlite_utils import Database + + +def test_insert_all_list_mode_basic(): + """Test basic insert_all with list-based iteration""" + db = Database(memory=True) + + def data_generator(): + # First yield column names + yield ["id", "name", "age"] + # Then yield data rows + yield [1, "Alice", 30] + yield [2, "Bob", 25] + yield [3, "Charlie", 35] + + db["people"].insert_all(data_generator()) + + rows = list(db["people"].rows) + assert len(rows) == 3 + assert rows[0] == {"id": 1, "name": "Alice", "age": 30} + assert rows[1] == {"id": 2, "name": "Bob", "age": 25} + assert rows[2] == {"id": 3, "name": "Charlie", "age": 35} + + +def test_insert_all_list_mode_with_pk(): + """Test insert_all with list mode and primary key""" + db = Database(memory=True) + + def data_generator(): + yield ["id", "name", "score"] + yield [1, "Alice", 95] + yield [2, "Bob", 87] + + db["scores"].insert_all(data_generator(), pk="id") + + assert db["scores"].pks == ["id"] + rows = list(db["scores"].rows) + assert len(rows) == 2 + + +def test_upsert_all_list_mode(): + """Test upsert_all with list-based iteration""" + db = Database(memory=True) + + # Initial insert + def initial_data(): + yield ["id", "name", "value"] + yield [1, "Alice", 100] + yield [2, "Bob", 200] + + db["data"].insert_all(initial_data(), pk="id") + + # Upsert with some updates and new records + def upsert_data(): + yield ["id", "name", "value"] + yield [1, "Alice", 150] # Update existing + yield [3, "Charlie", 300] # Insert new + + db["data"].upsert_all(upsert_data(), pk="id") + + rows = list(db["data"].rows_where(order_by="id")) + assert len(rows) == 3 + assert rows[0] == {"id": 1, "name": "Alice", "value": 150} + assert rows[1] == {"id": 2, "name": "Bob", "value": 200} + assert rows[2] == {"id": 3, "name": "Charlie", "value": 300} + + +def test_list_mode_with_various_types(): + """Test list mode with different data types""" + db = Database(memory=True) + + def data_generator(): + yield ["id", "name", "score", "active"] + yield [1, "Alice", 95.5, True] + yield [2, "Bob", 87.3, False] + yield [3, "Charlie", None, True] + + db["mixed"].insert_all(data_generator()) + + rows = list(db["mixed"].rows) + assert len(rows) == 3 + assert rows[0]["score"] == 95.5 + assert rows[1]["active"] == 0 # SQLite stores boolean as int + assert rows[2]["score"] is None + + +def test_list_mode_error_non_string_columns(): + """Test that non-string column names raise an error""" + db = Database(memory=True) + + def bad_data(): + yield [1, 2, 3] # Non-string column names + yield ["a", "b", "c"] + + with pytest.raises(ValueError, match="must be a list of column name strings"): + db["bad"].insert_all(bad_data()) + + +def test_list_mode_error_mixed_types(): + """Test that mixing list and dict raises an error""" + db = Database(memory=True) + + def bad_data(): + yield ["id", "name"] + yield {"id": 1, "name": "Alice"} # Should be a list, not dict + + with pytest.raises(ValueError, match="must also be lists"): + db["bad"].insert_all(bad_data()) + + +def test_list_mode_empty_after_headers(): + """Test that only headers without data works gracefully""" + db = Database(memory=True) + + def data_generator(): + yield ["id", "name", "age"] + # No data rows + + result = db["people"].insert_all(data_generator()) + assert result is not None + assert not db["people"].exists() + + +def test_list_mode_batch_processing(): + """Test list mode with large dataset requiring batching""" + db = Database(memory=True) + + def large_data(): + yield ["id", "value"] + for i in range(1000): + yield [i, f"value_{i}"] + + db["large"].insert_all(large_data(), batch_size=100) + + count = db.execute("SELECT COUNT(*) as c FROM large").fetchone()[0] + assert count == 1000 + + +def test_list_mode_shorter_rows(): + """Test that rows shorter than column list get NULL values""" + db = Database(memory=True) + + def data_generator(): + yield ["id", "name", "age", "city"] + yield [1, "Alice", 30, "NYC"] + yield [2, "Bob"] # Missing age and city + yield [3, "Charlie", 35] # Missing city + + db["people"].insert_all(data_generator()) + + rows = list(db["people"].rows_where(order_by="id")) + assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"} + assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None} + assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None} + + +def test_backwards_compatibility_dict_mode(): + """Ensure dict mode still works (backward compatibility)""" + db = Database(memory=True) + + # Traditional dict-based insert + data = [ + {"id": 1, "name": "Alice", "age": 30}, + {"id": 2, "name": "Bob", "age": 25}, + ] + + db["people"].insert_all(data) + + rows = list(db["people"].rows) + assert len(rows) == 2 + assert rows[0] == {"id": 1, "name": "Alice", "age": 30} + + +def test_insert_all_tuple_mode_basic(): + """Test basic insert_all with tuple-based iteration""" + db = Database(memory=True) + + def data_generator(): + # First yield column names as tuple + yield ("id", "name", "age") + # Then yield data rows as tuples + yield (1, "Alice", 30) + yield (2, "Bob", 25) + yield (3, "Charlie", 35) + + db["people"].insert_all(data_generator()) + + rows = list(db["people"].rows) + assert len(rows) == 3 + assert rows[0] == {"id": 1, "name": "Alice", "age": 30} + assert rows[1] == {"id": 2, "name": "Bob", "age": 25} + assert rows[2] == {"id": 3, "name": "Charlie", "age": 35} + + +def test_insert_all_mixed_list_tuple(): + """Test insert_all with mixed lists and tuples for data rows""" + db = Database(memory=True) + + def data_generator(): + # Column names as list + yield ["id", "name", "age"] + # Mix of list and tuple data rows + yield [1, "Alice", 30] + yield (2, "Bob", 25) + yield [3, "Charlie", 35] + yield (4, "Diana", 40) + + db["people"].insert_all(data_generator()) + + rows = list(db["people"].rows) + assert len(rows) == 4 + assert rows[0] == {"id": 1, "name": "Alice", "age": 30} + assert rows[1] == {"id": 2, "name": "Bob", "age": 25} + assert rows[2] == {"id": 3, "name": "Charlie", "age": 35} + assert rows[3] == {"id": 4, "name": "Diana", "age": 40} + + +def test_upsert_all_tuple_mode(): + """Test upsert_all with tuple-based iteration""" + db = Database(memory=True) + + # Initial insert with tuples + def initial_data(): + yield ("id", "name", "value") + yield (1, "Alice", 100) + yield (2, "Bob", 200) + + db["data"].insert_all(initial_data(), pk="id") + + # Upsert with tuples + def upsert_data(): + yield ("id", "name", "value") + yield (1, "Alice", 150) # Update existing + yield (3, "Charlie", 300) # Insert new + + db["data"].upsert_all(upsert_data(), pk="id") + + rows = list(db["data"].rows_where(order_by="id")) + assert len(rows) == 3 + assert rows[0] == {"id": 1, "name": "Alice", "value": 150} + assert rows[1] == {"id": 2, "name": "Bob", "value": 200} + assert rows[2] == {"id": 3, "name": "Charlie", "value": 300} + + +def test_tuple_mode_shorter_rows(): + """Test that tuple rows shorter than column list get NULL values""" + db = Database(memory=True) + + def data_generator(): + yield "id", "name", "age", "city" + yield 1, "Alice", 30, "NYC" + yield 2, "Bob" # Missing age and city + yield 3, "Charlie", 35 # Missing city + + db["people"].insert_all(data_generator()) + + rows = list(db["people"].rows_where(order_by="id")) + assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"} + assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None} + assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None} + + +def test_list_mode_single_record_upsert_last_pk(): + """Test that last_pk is populated correctly for single-record upserts in list mode""" + db = Database(memory=True) + + # Create table first + db["data"].insert({"id": 1, "name": "Alice", "value": 100}, pk="id") + + # Now upsert a single record using list mode + def upsert_data(): + yield ["id", "name", "value"] + yield [1, "Alice", 150] # Update existing + + table = db["data"] + table.upsert_all(upsert_data(), pk="id") + + # Verify the data was updated + rows = list(db["data"].rows) + assert rows == [{"id": 1, "name": "Alice", "value": 150}] + + # Verify last_pk is populated correctly + assert table.last_pk == 1 diff --git a/tests/test_lookup.py b/tests/test_lookup.py index ac8e1c2..da4f18b 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -66,3 +66,118 @@ def test_lookup_fails_if_constraint_cannot_be_added(fresh_db): # This will fail because the name column is not unique with pytest.raises(Exception, match="UNIQUE constraint failed"): species.lookup({"name": "Palm"}) + + +def test_lookup_with_extra_values(fresh_db): + species = fresh_db["species"] + id = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2020-01-01"}) + assert species.get(id) == { + "id": 1, + "name": "Palm", + "type": "Tree", + "first_seen": "2020-01-01", + } + # A subsequent lookup() should ignore the second dictionary + id2 = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2021-02-02"}) + assert id2 == id + assert species.get(id2) == { + "id": 1, + "name": "Palm", + "type": "Tree", + "first_seen": "2020-01-01", + } + + +def test_lookup_with_extra_insert_parameters(fresh_db): + other_table = fresh_db["other_table"] + other_table.insert({"id": 1, "name": "Name"}, pk="id") + species = fresh_db["species"] + id = species.lookup( + {"name": "Palm", "type": "Tree"}, + { + "first_seen": "2020-01-01", + "make_not_null": 1, + "fk_to_other": 1, + "default_is_dog": "cat", + "extract_this": "This is extracted", + "convert_to_upper": "upper", + "make_this_integer": "2", + "this_at_front": 1, + }, + pk="renamed_id", + foreign_keys=(("fk_to_other", "other_table", "id"),), + column_order=("this_at_front",), + not_null={"make_not_null"}, + defaults={"default_is_dog": "dog"}, + extracts=["extract_this"], + conversions={"convert_to_upper": "upper(?)"}, + columns={"make_this_integer": int}, + ) + assert species.schema == ( + 'CREATE TABLE "species" (\n' + ' "renamed_id" INTEGER PRIMARY KEY,\n' + ' "this_at_front" INTEGER,\n' + ' "name" TEXT,\n' + ' "type" TEXT,\n' + ' "first_seen" TEXT,\n' + ' "make_not_null" INTEGER NOT NULL,\n' + ' "fk_to_other" INTEGER REFERENCES "other_table"("id"),\n' + " \"default_is_dog\" TEXT DEFAULT 'dog',\n" + ' "extract_this" INTEGER REFERENCES "extract_this"("id"),\n' + ' "convert_to_upper" TEXT,\n' + ' "make_this_integer" INTEGER\n' + ")" + ) + assert species.get(id) == { + "renamed_id": id, + "this_at_front": 1, + "name": "Palm", + "type": "Tree", + "first_seen": "2020-01-01", + "make_not_null": 1, + "fk_to_other": 1, + "default_is_dog": "cat", + "extract_this": 1, + "convert_to_upper": "UPPER", + "make_this_integer": 2, + } + assert species.indexes == [ + Index( + seq=0, + name="idx_species_name_type", + unique=1, + origin="c", + partial=0, + columns=["name", "type"], + ) + ] + + +@pytest.mark.parametrize("strict", (False, True)) +def test_lookup_new_table_strict(fresh_db, strict): + fresh_db["species"].lookup({"name": "Palm"}, strict=strict) + assert fresh_db["species"].strict == strict or not fresh_db.supports_strict + + +def test_lookup_null_value_idempotent(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/186 + # Repeated lookups of a null value should return the same row, + # not insert a duplicate row each time + species = fresh_db["species"] + first_id = species.lookup({"name": None}) + second_id = species.lookup({"name": None}) + assert first_id == second_id + assert list(species.rows) == [{"id": first_id, "name": None}] + + +def test_lookup_compound_key_with_null_idempotent(fresh_db): + species = fresh_db["species"] + palm_id = species.lookup({"name": "Palm", "type": None}) + oak_id = species.lookup({"name": "Oak", "type": "Tree"}) + assert palm_id == species.lookup({"name": "Palm", "type": None}) + assert oak_id == species.lookup({"name": "Oak", "type": "Tree"}) + assert palm_id != oak_id + assert list(species.rows) == [ + {"id": palm_id, "name": "Palm", "type": None}, + {"id": oak_id, "name": "Oak", "type": "Tree"}, + ] diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 5423d2b..d613bb9 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -109,9 +109,9 @@ def test_m2m_with_table_objects(fresh_db): ) expected_tables = {"dogs", "humans", "dogs_humans"} assert expected_tables == set(fresh_db.table_names()) - assert 1 == dogs.count - assert 2 == humans.count - assert 2 == fresh_db["dogs_humans"].count + assert dogs.count == 1 + assert humans.count == 2 + assert fresh_db["dogs_humans"].count == 2 def test_m2m_lookup(fresh_db): @@ -139,9 +139,9 @@ def test_m2m_lookup(fresh_db): def test_m2m_requires_either_records_or_lookup(fresh_db): people = fresh_db.table("people", pk="id").insert({"name": "Wahyu"}) - with pytest.raises(AssertionError): + with pytest.raises(ValueError): people.m2m("tags") - with pytest.raises(AssertionError): + with pytest.raises(ValueError): people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"}) diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 0000000..04185fc --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,246 @@ +import pytest +import sqlite_utils +from sqlite_utils import Migrations + + +@pytest.fixture +def migrations(): + migrations = Migrations("test") + + @migrations() + def m001(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations() + def m002(db): + db["cats"].create({"name": str}) + db.execute("insert into dogs (name) values ('Pancakes')") + + return migrations + + +@pytest.fixture +def migrations_not_ordered_alphabetically(): + # Names order alphabetically in the wrong direction but this + # should still be applied correctly. + migrations = Migrations("test") + + @migrations() + def m002(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations() + def m001(db): + db["cats"].create({"name": str}) + db.execute("insert into dogs (name) values ('Pancakes')") + + return migrations + + +@pytest.fixture +def migrations2(): + migrations = Migrations("test2") + + @migrations() + def m001(db): + db["dogs2"].insert({"name": "Cleo"}) + + return migrations + + +def test_basic(migrations): + db = sqlite_utils.Database(memory=True) + assert db.table_names() == [] + migrations.apply(db) + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} + + +def test_stop_before(migrations): + db = sqlite_utils.Database(memory=True) + assert db.table_names() == [] + migrations.apply(db, stop_before="m002") + assert set(db.table_names()) == {"_sqlite_migrations", "dogs"} + migrations.apply(db) + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} + + +def test_two_migration_sets(migrations, migrations2): + db = sqlite_utils.Database(memory=True) + assert db.table_names() == [] + migrations.apply(db) + migrations2.apply(db) + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats", "dogs2"} + + +def test_order_does_not_matter(migrations, migrations_not_ordered_alphabetically): + db1 = sqlite_utils.Database(memory=True) + db2 = sqlite_utils.Database(memory=True) + migrations.apply(db1) + migrations_not_ordered_alphabetically.apply(db2) + assert db1.schema == db2.schema + + +def test_applied_at_is_a_string(migrations): + db = sqlite_utils.Database(memory=True) + migrations.apply(db) + applied = migrations.applied(db) + assert len(applied) == 2 + for migration in applied: + # applied_at is the TEXT timestamp straight from the + # _sqlite_migrations table, e.g. "2026-07-04 12:00:00.000000+00:00" + assert isinstance(migration.applied_at, str) + assert migration.applied_at.endswith("+00:00") + + +def test_failing_migration_rolls_back(migrations): + @migrations() + def m003(db): + db["birds"].create({"name": str}) + db.execute("insert into dogs (name) values ('Dozer')") + raise ValueError("boom") + + db = sqlite_utils.Database(memory=True) + with pytest.raises(ValueError): + migrations.apply(db) + # m001 and m002 committed before the failure and stay applied + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} + assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] + # Everything m003 did was rolled back and it is still pending + assert [m.name for m in migrations.pending(db)] == ["m003"] + + +def test_rerun_after_failure_applies_each_migration_once(): + state = {"fail": True} + migrations = Migrations("test") + + @migrations() + def m001(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations() + def m002(db): + db["dogs"].insert({"name": "Pancakes"}) + if state["fail"]: + raise ValueError("boom") + + db = sqlite_utils.Database(memory=True) + with pytest.raises(ValueError): + migrations.apply(db) + state["fail"] = False + migrations.apply(db) + # m001 must not have been re-applied, m002 applied exactly once + assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] + + +def test_non_transactional_migration_allows_vacuum(tmpdir): + path = str(tmpdir / "test.db") + db = sqlite_utils.Database(path) + migrations = Migrations("test") + + @migrations() + def m001(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations(transactional=False) + def m002(db): + db.execute("VACUUM") + + migrations.apply(db) + assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] + db.close() + + +def test_apply_composes_inside_outer_transaction(migrations): + db = sqlite_utils.Database(memory=True) + with pytest.raises(ZeroDivisionError): + with db.atomic(): + migrations.apply(db) + raise ZeroDivisionError + # The outer transaction rolled back, taking the migrations with it + assert db.table_names() == [] + + +@pytest.mark.parametrize( + "create_table,pk", + ( + ( + { + "migration_set": str, + "name": str, + "applied_at": str, + }, + "name", + ), + ( + { + "migration_set": str, + "name": str, + "applied_at": str, + }, + ("migration_set", "name"), + ), + ), +) +def test_upgrades_sqlite_migrations(migrations, create_table, pk): + db = sqlite_utils.Database(memory=True) + db["_sqlite_migrations"].create(create_table, pk=pk) + assert db.table_names() == ["_sqlite_migrations"] + assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk)) + migrations.apply(db) + assert db["_sqlite_migrations"].pks == ["id"] + + +def test_pending_and_applied_are_read_only(migrations): + db = sqlite_utils.Database(memory=True) + assert [m.name for m in migrations.pending(db)] == ["m001", "m002"] + assert migrations.applied(db) == [] + # Neither call should have created the tracking table + assert db.table_names() == [] + + +def test_duplicate_migration_name_errors(): + migrations = Migrations("test") + + @migrations() + def m001(db): + pass + + with pytest.raises(ValueError) as ex: + + @migrations(name="m001") + def m001_again(db): + pass + + assert "m001" in str(ex.value) + + +def test_stop_before_applied_migration_errors(migrations): + # Stopping before a migration that has already been applied is + # impossible to honor - previously the stop name was only checked + # against pending migrations, so everything after it was applied + db = sqlite_utils.Database(memory=True) + migrations.apply(db, stop_before="m002") # applies m001 only + with pytest.raises(ValueError) as ex: + migrations.apply(db, stop_before="m001") + assert "m001" in str(ex.value) + assert "already been applied" in str(ex.value) + # Nothing else was applied + assert not db["cats"].exists() + + +def test_stop_before_applied_migration_errors_before_any_apply(migrations): + # The error fires before any pending migration runs, even those that + # come before the already-applied stop target in registration order + db = sqlite_utils.Database(memory=True) + only_second = Migrations("test") + + @only_second() + def m002(db): + db["cats"].create({"name": str}) + + only_second.apply(db) # m002 applied, m001 still pending + with pytest.raises(ValueError): + migrations.apply(db, stop_before="m002") + assert not db["dogs"].exists() diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 0000000..c793e32 --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,128 @@ +from click.testing import CliRunner +import click +import importlib +import pytest +import sys +from sqlite_utils import cli, Database, hookimpl, plugins + + +def _supports_pragma_function_list(): + db = Database(memory=True) + try: + db.execute("select * from pragma_function_list()") + return True + except Exception: + return False + finally: + db.close() + + +def test_get_plugins_loads_setuptools_entrypoints_once(monkeypatch): + calls = [] + monkeypatch.delattr(sys, "_called_from_test", raising=False) + monkeypatch.setattr(plugins, "_plugins_loaded", False) + monkeypatch.setattr( + plugins.pm, + "load_setuptools_entrypoints", + lambda group: calls.append(group) or 0, + ) + + plugins.get_plugins() + plugins.get_plugins() + + assert calls == ["sqlite_utils"] + + +def test_get_plugins_does_not_load_setuptools_entrypoints_in_tests(monkeypatch): + calls = [] + monkeypatch.setattr(sys, "_called_from_test", True, raising=False) + monkeypatch.setattr(plugins, "_plugins_loaded", False) + monkeypatch.setattr( + plugins.pm, + "load_setuptools_entrypoints", + lambda group: calls.append(group) or 0, + ) + + assert plugins.get_plugins() == [] + assert calls == [] + + +def test_register_commands(): + importlib.reload(cli) + assert plugins.get_plugins() == [] + + class HelloWorldPlugin: + __name__ = "HelloWorldPlugin" + + @hookimpl + def register_commands(self, cli): + @cli.command(name="hello-world") + def hello_world(): + "Print hello world" + click.echo("Hello world!") + + try: + plugins.pm.register(HelloWorldPlugin(), name="HelloWorldPlugin") + importlib.reload(cli) + + assert plugins.get_plugins() == [ + {"name": "HelloWorldPlugin", "hooks": ["register_commands"]} + ] + + runner = CliRunner() + result = runner.invoke(cli.cli, ["hello-world"]) + assert result.exit_code == 0 + assert result.output == "Hello world!\n" + + finally: + plugins.pm.unregister(name="HelloWorldPlugin") + importlib.reload(cli) + assert plugins.get_plugins() == [] + + +@pytest.mark.skipif( + not _supports_pragma_function_list(), + reason="Needs SQLite version that supports pragma_function_list()", +) +def test_prepare_connection(): + importlib.reload(cli) + assert plugins.get_plugins() == [] + + class HelloFunctionPlugin: + __name__ = "HelloFunctionPlugin" + + @hookimpl + def prepare_connection(self, conn): + conn.create_function("hello", 1, lambda name: f"Hello, {name}!") + + db = Database(memory=True) + + def _functions(db): + return [ + row[0] + for row in db.execute( + "select distinct name from pragma_function_list() order by 1" + ).fetchall() + ] + + assert "hello" not in _functions(db) + + try: + plugins.pm.register(HelloFunctionPlugin(), name="HelloFunctionPlugin") + + assert plugins.get_plugins() == [ + {"name": "HelloFunctionPlugin", "hooks": ["prepare_connection"]} + ] + + db = Database(memory=True) + assert "hello" in _functions(db) + result = db.execute('select hello("world")').fetchone()[0] + assert result == "Hello, world!" + + # Test execute_plugins=False + db2 = Database(memory=True, execute_plugins=False) + assert "hello" not in _functions(db2) + + finally: + plugins.pm.unregister(name="HelloFunctionPlugin") + assert plugins.get_plugins() == [] diff --git a/tests/test_query.py b/tests/test_query.py index fe79cc0..06847da 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1,5 +1,8 @@ +import pytest import types +from sqlite_utils.utils import sqlite3 + def test_query(fresh_db): fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) @@ -8,6 +11,268 @@ def test_query(fresh_db): assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}] +def test_query_executes_eagerly(fresh_db): + # The SQL runs when query() is called, not when the result is iterated, + # so errors are raised at the call site + with pytest.raises(sqlite3.OperationalError): + fresh_db.query("select * from missing_table") + + +def test_query_rejects_statements_that_return_no_rows(fresh_db): + fresh_db["dogs"].insert({"name": "Cleo"}) + with pytest.raises(ValueError) as ex: + fresh_db.query("update dogs set name = 'Cleopaws'") + assert "execute()" in str(ex.value) + # The rejected update was rolled back, and no transaction is left open + assert not fresh_db.conn.in_transaction + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + +def test_query_rejected_ddl_is_rolled_back(fresh_db): + with pytest.raises(ValueError): + fresh_db.query("create table dogs (id integer primary key)") + assert not fresh_db.conn.in_transaction + assert fresh_db.table_names() == [] + + +def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + fresh_db.execute("insert into dogs (name) values ('Pancakes')") + with pytest.raises(ValueError): + fresh_db.query("update dogs set name = 'Cleopaws'") + # The transaction is still open and the earlier insert is intact + assert fresh_db.conn.in_transaction + fresh_db.commit() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo", "Pancakes"] + + +@pytest.mark.parametrize( + "sql", + [ + "begin", + "commit", + "rollback", + "vacuum", + "detach database foo", + "/* comment */ commit", + "-- comment\nbegin", + "/* multi\nline */ -- and another\n vacuum", + "\t /* a */ /* b */ savepoint s1", + "; commit", + ";;\n ; rollback", + "; /* comment */ vacuum", + "\ufeffbegin", + ], +) +def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): + with pytest.raises(ValueError) as ex: + fresh_db.query(sql) + assert "execute()" in str(ex.value) + assert not fresh_db.conn.in_transaction + + +def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): + # A COMMIT hidden behind a leading comment must not slip past the + # keyword check - previously it committed the caller's open + # transaction before the ValueError was raised + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + fresh_db.execute("insert into dogs (name) values ('Pancakes')") + with pytest.raises(ValueError): + fresh_db.query("/* comment */ COMMIT") + # The explicit transaction is still open and can still be rolled back + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + +@pytest.mark.parametrize("sql", ["; COMMIT", "\ufeffCOMMIT"]) +def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql): + # sqlite3 tolerates empty statements and a UTF-8 BOM before the first + # real token, so the keyword scanner must skip them too - previously + # '; COMMIT' slipped past the check and committed the caller's open + # transaction before raising OperationalError + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + fresh_db.execute("insert into dogs (name) values ('Pancakes')") + with pytest.raises(ValueError): + fresh_db.query(sql) + # The explicit transaction is still open and can still be rolled back + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + +def test_query_error_leaves_no_transaction_open(fresh_db): + with pytest.raises(sqlite3.OperationalError): + fresh_db.query("select * from missing_table") + assert not fresh_db.conn.in_transaction + + +def test_query_pragma(tmpdir): + from sqlite_utils import Database + + db = Database(str(tmpdir / "test.db")) + # A row-returning PRAGMA works, including one that cannot run in a transaction + assert list(db.query("pragma journal_mode = wal")) == [{"journal_mode": "wal"}] + # A PRAGMA that returns no rows raises ValueError + with pytest.raises(ValueError): + db.query("pragma user_version = 5") + db.close() + + +def test_query_rejected_pragma_still_takes_effect(fresh_db): + # Documented limitation: PRAGMAs run outside the savepoint guard, + # because some of them refuse to run inside a transaction - so a + # row-less PRAGMA takes effect even though it raises ValueError. + # If this test starts failing because the pragma was rolled back, + # the limitation has been fixed - update the docs in python-api.rst + # and the query() docstring to remove the carve-out + with pytest.raises(ValueError): + fresh_db.query("pragma user_version = 5") + assert fresh_db.execute("pragma user_version").fetchone()[0] == 5 + + +def test_query_comment_prefixed_pragma(tmpdir): + from sqlite_utils import Database + + db = Database(str(tmpdir / "test.db")) + # A leading comment must not stop a PRAGMA being recognized as one - + # previously it was executed inside the savepoint guard, where + # journal mode changes are refused + assert list(db.query("-- set WAL mode\npragma journal_mode = wal")) == [ + {"journal_mode": "wal"} + ] + db.close() + + +def test_query_comment_prefixed_pragma_inside_transaction(fresh_db): + fresh_db.begin() + assert list(fresh_db.query("-- check version\npragma user_version")) == [ + {"user_version": 0} + ] + assert fresh_db.conn.in_transaction + fresh_db.rollback() + + +@pytest.mark.parametrize( + "sql,expected", + [ + ("select 1", "SELECT"), + (" \t\n select 1", "SELECT"), + ("-- comment\nbegin", "BEGIN"), + ("/* one */ /* two */ pragma user_version", "PRAGMA"), + ("/* multi\nline */vacuum", "VACUUM"), + ("insert into t values (1)", "INSERT"), + ("-- only a comment", ""), + ("/* unterminated", ""), + ("", ""), + (" ", ""), + ("123", ""), + ("; commit", "COMMIT"), + (";;\n ; rollback", "ROLLBACK"), + ("; -- comment\n begin", "BEGIN"), + ("\ufeffcommit", "COMMIT"), + ("\ufeff ; select 1", "SELECT"), + (";", ""), + ], +) +def test_first_keyword(sql, expected): + from sqlite_utils.db import _first_keyword + + assert _first_keyword(sql) == expected + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) +def test_query_insert_returning(fresh_db): + fresh_db["dogs"].insert({"name": "Cleo"}) + rows = list( + fresh_db.query("insert into dogs (name) values ('Pancakes') returning name") + ) + assert rows == [{"name": "Pancakes"}] + assert fresh_db["dogs"].count == 2 + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) +def test_query_insert_returning_commits_without_iteration(tmpdir): + from sqlite_utils import Database + + path = str(tmpdir / "test.db") + db = Database(path) + db["dogs"].insert({"name": "Cleo"}) + # Never iterate over the results + db.query("insert into dogs (name) values ('Pancakes') returning name") + assert not db.conn.in_transaction + # A completely separate connection sees the new row straight away + other = sqlite3.connect(path) + assert other.execute("select count(*) from dogs").fetchone()[0] == 2 + other.close() + db.close() + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) +def test_query_insert_returning_partial_iteration_still_commits(tmpdir): + from sqlite_utils import Database + + path = str(tmpdir / "test.db") + db = Database(path) + db["dogs"].insert({"name": "Cleo"}) + row = next( + db.query( + "insert into dogs (name) values ('Pancakes'), ('Marnie') returning name" + ) + ) + assert row == {"name": "Pancakes"} + assert not db.conn.in_transaction + other = sqlite3.connect(path) + assert other.execute("select count(*) from dogs").fetchone()[0] == 3 + other.close() + db.close() + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) +def test_query_insert_returning_respects_explicit_transaction(fresh_db): + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + rows = list( + fresh_db.query("insert into dogs (name) values ('Pancakes') returning name") + ) + assert rows == [{"name": "Pancakes"}] + # Still inside the explicit transaction - not committed + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + +def test_query_duplicate_column_names_are_deduped(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/624 + fresh_db["one"].insert({"id": 1, "value": "left"}) + fresh_db["two"].insert({"id": 2, "value": "right"}) + rows = list( + fresh_db.query("select one.id, two.id, one.value, two.value from one, two") + ) + assert rows == [{"id": 1, "id_2": 2, "value": "left", "value_2": "right"}] + + +def test_query_deduped_column_avoids_existing_names(fresh_db): + # The renamed duplicate must not overwrite a real column called id_2 + rows = list(fresh_db.query("select 1 as id, 2 as id, 3 as id_2")) + assert rows == [{"id": 1, "id_3": 2, "id_2": 3}] + + def test_execute_returning_dicts(fresh_db): # Like db.query() but returns a list, included for backwards compatibility # see https://github.com/simonw/sqlite-utils/issues/290 @@ -15,3 +280,24 @@ def test_execute_returning_dicts(fresh_db): assert fresh_db.execute_returning_dicts("select * from test") == [ {"id": 1, "bar": 2} ] + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) +def test_query_preserves_error_from_transaction_destroying_trigger(fresh_db): + # RAISE(ROLLBACK) destroys the savepoint guard - the original + # IntegrityError must propagate, not "no such savepoint" + fresh_db.execute("create table t (id integer primary key, v text)") + fresh_db.execute(""" + create trigger no_bad before insert on t + when new.v = 'bad' + begin + select raise(rollback, 'trigger says no'); + end + """) + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + fresh_db.query("insert into t (id, v) values (1, 'bad') returning id") + assert not fresh_db.conn.in_transaction + assert fresh_db.execute("select count(*) from t").fetchone()[0] == 0 diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 89240a2..a7c7ef7 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1,4 +1,5 @@ from sqlite_utils import recipes +from sqlite_utils.utils import sqlite3 import json import pytest @@ -61,6 +62,39 @@ def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): ] +@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") +@pytest.mark.parametrize("fn", ("parsedate", "parsedatetime")) +def test_dateparse_errors_raises(fresh_db, fn): + """Test that invalid dates raise errors when errors=None""" + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "invalid"}, + ], + pk="id", + ) + # Exception in SQLite callback surfaces as OperationalError + with pytest.raises(sqlite3.OperationalError): + fresh_db["example"].convert("dt", lambda value: getattr(recipes, fn)(value)) + + +@pytest.mark.parametrize("fn", ("parsedate", "parsedatetime")) +@pytest.mark.parametrize("errors", (recipes.SET_NULL, recipes.IGNORE)) +def test_dateparse_errors_handled(fresh_db, fn, errors): + """Test error handling modes for invalid dates""" + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "invalid"}, + ], + pk="id", + ) + fresh_db["example"].convert( + "dt", lambda value: getattr(recipes, fn)(value, errors=errors) + ) + rows = list(fresh_db["example"].rows) + expected = [{"id": 1, "dt": None if errors is recipes.SET_NULL else "invalid"}] + assert rows == expected + + @pytest.mark.parametrize("delimiter", [None, ";", "-"]) def test_jsonsplit(fresh_db, delimiter): fresh_db["example"].insert_all( @@ -70,12 +104,14 @@ def test_jsonsplit(fresh_db, delimiter): ], pk="id", ) - fn = recipes.jsonsplit if delimiter is not None: def fn(value): return recipes.jsonsplit(value, delimiter=delimiter) + else: + fn = recipes.jsonsplit + fresh_db["example"].convert("tags", fn) assert list(fresh_db["example"].rows) == [ {"id": 1, "tags": '["foo", "bar"]'}, @@ -98,11 +134,13 @@ def test_jsonsplit_type(fresh_db, type, expected): ], pk="id", ) - fn = recipes.jsonsplit if type is not None: def fn(value): return recipes.jsonsplit(value, type=type) + else: + fn = recipes.jsonsplit + fresh_db["example"].convert("records", fn) assert json.loads(fresh_db["example"].get(1)["records"]) == expected diff --git a/tests/test_recreate.py b/tests/test_recreate.py index 504a0b8..bce53d5 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -14,19 +14,25 @@ def test_recreate_ignored_for_in_memory(): def test_recreate_not_allowed_for_connection(): conn = sqlite3.connect(":memory:") - with pytest.raises(AssertionError): - Database(conn, recreate=True) + try: + with pytest.raises(ValueError): + Database(conn, recreate=True) + finally: + conn.close() @pytest.mark.parametrize( - "use_path,file_exists", [(True, True), (True, False), (False, True), (False, False)] + "use_path,create_file_first", + [(True, True), (True, False), (False, True), (False, False)], ) -def test_recreate(tmpdir, use_path, file_exists): - filepath = str(tmpdir / "data.db") +def test_recreate(tmp_path, use_path, create_file_first): + filepath = str(tmp_path / "data.db") if use_path: filepath = pathlib.Path(filepath) - if file_exists: - Database(filepath)["t1"].insert({"foo": "bar"}) - assert ["t1"] == Database(filepath).table_names() + if create_file_first: + db = Database(filepath) + db["t1"].insert({"foo": "bar"}) + assert ["t1"] == db.table_names() + db.close() Database(filepath, recreate=True)["t2"].insert({"foo": "bar"}) assert ["t2"] == Database(filepath).table_names() diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 19af0b6..618bf1e 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -1,7 +1,8 @@ # flake8: noqa import pytest import sys -from unittest.mock import MagicMock +from unittest.mock import MagicMock, call +from sqlite_utils.utils import sqlite3 def test_register_function(fresh_db): @@ -13,6 +14,15 @@ def test_register_function(fresh_db): assert result == "olleh" +def test_register_function_custom_name(fresh_db): + @fresh_db.register_function(name="revstr") + def reverse_string(s): + return "".join(reversed(list(s))) + + result = fresh_db.execute('select revstr("hello")').fetchone()[0] + assert result == "olleh" + + def test_register_function_multiple_arguments(fresh_db): @fresh_db.register_function def a_times_b_plus_c(a, b, c): @@ -31,20 +41,47 @@ def test_register_function_deterministic(fresh_db): assert result == "bob" -@pytest.mark.skipif( - sys.version_info < (3, 8), reason="deterministic=True was added in Python 3.8" -) -def test_register_function_deterministic_registered(fresh_db): +def test_register_function_deterministic_tries_again_if_exception_raised(fresh_db): + # Save the original connection so we can close it later + original_conn = fresh_db.conn fresh_db.conn = MagicMock() fresh_db.conn.create_function = MagicMock() - @fresh_db.register_function(deterministic=True) - def to_lower_2(s): - return s.lower() + try: - fresh_db.conn.create_function.assert_called_with( - "to_lower_2", 1, to_lower_2, deterministic=True - ) + @fresh_db.register_function(deterministic=True) + def to_lower_2(s): + return s.lower() + + fresh_db.conn.create_function.assert_called_with( + "to_lower_2", 1, to_lower_2, deterministic=True + ) + + first = True + + def side_effect(*args, **kwargs): + # Raise exception only first time this is called + nonlocal first + if first: + first = False + raise sqlite3.NotSupportedError() + + # But if sqlite3.NotSupportedError is raised, it tries again + fresh_db.conn.create_function.reset_mock() + fresh_db.conn.create_function.side_effect = side_effect + + @fresh_db.register_function(deterministic=True) + def to_lower_3(s): + return s.lower() + + # Should have been called once with deterministic=True and once without + assert fresh_db.conn.create_function.call_args_list == [ + call("to_lower_3", 1, to_lower_3, deterministic=True), + call("to_lower_3", 1, to_lower_3), + ] + finally: + # Close the original connection that was replaced with the mock + original_conn.close() def test_register_function_replace(fresh_db): @@ -54,7 +91,7 @@ def test_register_function_replace(fresh_db): assert "one" == fresh_db.execute("select one()").fetchone()[0] - # This will fail to replace the function: + # This will silently fail to replaec the function @fresh_db.register_function() def one(): # noqa return "two" diff --git a/tests/test_rows.py b/tests/test_rows.py index a8a4ca0..46d4f53 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -104,3 +104,37 @@ def test_pks_and_rows_where_compound_pk(fresh_db): (("number", 1), {"type": "number", "number": 1, "plusone": 2}), (("number", 2), {"type": "number", "number": 2, "plusone": 3}), ] + + +def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/624 + fresh_db["t"].insert({"id": 1, "name": "Cleo"}) + rows = list(fresh_db["t"].rows_where(select="id, id, name")) + assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}] + + +def test_pks_and_rows_where_view(fresh_db): + # pks_and_rows_where() lives on Queryable so views expose it, but + # SQLite views have no rowid. Modern SQLite (3.36+) raises an + # OperationalError from the generated SQL; older versions returned + # NULL for a view's rowid. Either way it must not fail earlier with + # an AttributeError from View lacking Table-only properties + from sqlite_utils.utils import sqlite3 + + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.create_view("dog_names", "select name from dogs") + try: + result = list(fresh_db["dog_names"].pks_and_rows_where()) + except sqlite3.OperationalError: + pass # SQLite 3.36+: no such column: rowid + else: + # Older SQLite returns NULL rowids for views + assert result == [(None, {"rowid": None, "name": "Cleo"})] + + +def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db): + # Compound pks are returned in PRIMARY KEY declaration order + fresh_db.execute("create table t (b text, a text, primary key (a, b))") + fresh_db["t"].insert({"a": "A", "b": "B"}) + pks_and_rows = list(fresh_db["t"].pks_and_rows_where()) + assert pks_and_rows == [(("A", "B"), {"b": "B", "a": "A"})] diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py new file mode 100644 index 0000000..a19fed6 --- /dev/null +++ b/tests/test_rows_from_file.py @@ -0,0 +1,54 @@ +from sqlite_utils.utils import rows_from_file, Format, RowError +from io import BytesIO, StringIO +import pytest + + +@pytest.mark.parametrize( + "input,expected_format", + ( + (b"id,name\n1,Cleo", Format.CSV), + (b"id\tname\n1\tCleo", Format.TSV), + (b'[{"id": "1", "name": "Cleo"}]', Format.JSON), + ), +) +def test_rows_from_file_detect_format(input, expected_format): + rows, format = rows_from_file(BytesIO(input)) + assert format == expected_format + rows_list = list(rows) + assert rows_list == [{"id": "1", "name": "Cleo"}] + + +@pytest.mark.parametrize( + "ignore_extras,extras_key,expected", + ( + (True, None, [{"id": "1", "name": "Cleo"}]), + (False, "_rest", [{"id": "1", "name": "Cleo", "_rest": ["oops"]}]), + # expected of None means expect an error: + (False, False, None), + ), +) +def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expected): + try: + rows, format = rows_from_file( + BytesIO(b"id,name\r\n1,Cleo,oops"), + format=Format.CSV, + ignore_extras=ignore_extras, + extras_key=extras_key, + ) + list_rows = list(rows) + except RowError: + if expected is None: + # This is fine, + return + else: + # We did not expect an error + raise + assert list_rows == expected + + +def test_rows_from_file_error_on_string_io(): + with pytest.raises(TypeError) as ex: + rows_from_file(StringIO("id,name\r\n1,Cleo")) # type: ignore[arg-type] + assert ex.value.args == ( + "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO", + ) diff --git a/tests/test_sniff.py b/tests/test_sniff.py index 36cc471..4bbdb66 100644 --- a/tests/test_sniff.py +++ b/tests/test_sniff.py @@ -6,13 +6,13 @@ import pytest sniff_dir = pathlib.Path(__file__).parent / "sniff" -@pytest.mark.parametrize("filepath", sniff_dir.glob("example*")) +@pytest.mark.parametrize("filepath", sorted(sniff_dir.glob("example*"))) def test_sniff(tmpdir, filepath): db_path = str(tmpdir / "test.db") runner = CliRunner() result = runner.invoke( cli.cli, - ["insert", db_path, "creatures", str(filepath), "--sniff"], + ["insert", db_path, "creatures", str(filepath), "--sniff", "--no-detect-types"], catch_exceptions=False, ) assert result.exit_code == 0, result.stdout diff --git a/tests/test_tracer.py b/tests/test_tracer.py index d3ff22d..d14697d 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -6,26 +6,29 @@ def test_tracer(): db = Database( memory=True, tracer=lambda sql, params: collected.append((sql, params)) ) - db["dogs"].insert({"name": "Cleopaws"}) - db["dogs"].enable_fts(["name"]) - db["dogs"].search("Cleopaws") + dogs = db.table("dogs") + dogs.insert({"name": "Cleopaws"}) + dogs.enable_fts(["name"]) + dogs.search("Cleopaws") assert collected == [ ("PRAGMA recursive_triggers=on;", None), ("select name from sqlite_master where type = 'view'", None), ("select name from sqlite_master where type = 'table'", None), - ("CREATE TABLE [dogs] (\n [name] TEXT\n);\n ", None), ("select name from sqlite_master where type = 'view'", None), - ("INSERT INTO [dogs] ([name]) VALUES (?);", ["Cleopaws"]), ("select name from sqlite_master where type = 'view'", None), + ("select name from sqlite_master where type = 'table'", None), + ("select name from sqlite_master where type = 'view'", None), + ('CREATE TABLE "dogs" (\n "name" TEXT\n);\n ', None), + ("select name from sqlite_master where type = 'view'", None), + ('INSERT INTO "dogs" ("name") VALUES (?)', ["Cleopaws"]), ( - "CREATE VIRTUAL TABLE [dogs_fts] USING FTS5 (\n [name],\n content=[dogs]\n)", + 'CREATE VIRTUAL TABLE "dogs_fts" USING FTS5 (\n "name",\n content="dogs"\n)', None, ), ( - "INSERT INTO [dogs_fts] (rowid, [name])\n SELECT rowid, [name] FROM [dogs];", + 'INSERT INTO "dogs_fts" (rowid, "name")\n SELECT rowid, "name" FROM "dogs";', None, ), - ("select name from sqlite_master where type = 'view'", None), ] @@ -37,55 +40,57 @@ def test_with_tracer(): db = Database(memory=True) - db["dogs"].insert({"name": "Cleopaws"}) - db["dogs"].enable_fts(["name"]) + dogs = db.table("dogs") + + dogs.insert({"name": "Cleopaws"}) + dogs.enable_fts(["name"]) assert len(collected) == 0 with db.tracer(tracer): - list(db["dogs"].search("Cleopaws")) + list(dogs.search("Cleopaws")) - assert len(collected) == 5 + assert len(collected) == 4 assert collected == [ - ("select name from sqlite_master where type = 'view'", None), ( - ( - "SELECT name FROM sqlite_master\n" - " WHERE rootpage = 0\n" - " AND (\n" - " sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n" - " OR (\n" - ' tbl_name = "dogs"\n' - " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" - " )\n" - " )" - ), - None, + "SELECT name FROM sqlite_master\n" + " WHERE rootpage = 0\n" + " AND (\n" + " sql LIKE :like\n" + " OR sql LIKE :like2\n" + " OR (\n" + " tbl_name = :table\n" + " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" + " )\n" + " )", + { + "like": "%VIRTUAL TABLE%USING FTS%content=[dogs]%", + "like2": '%VIRTUAL TABLE%USING FTS%content="dogs"%', + "table": "dogs", + }, ), ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), ( - ( - "with original as (\n" - " select\n" - " rowid,\n" - " *\n" - " from [dogs]\n" - ")\n" - "select\n" - " [original].*\n" - "from\n" - " [original]\n" - " join [dogs_fts] on [original].rowid = [dogs_fts].rowid\n" - "where\n" - " [dogs_fts] match :query\n" - "order by\n" - " [dogs_fts].rank" - ), + 'with "original" as (\n' + " select\n" + " rowid,\n" + " *\n" + ' from "dogs"\n' + ")\n" + "select\n" + ' "original".*\n' + "from\n" + ' "original"\n' + ' join "dogs_fts" on "original".rowid = "dogs_fts".rowid\n' + "where\n" + ' "dogs_fts" match :query\n' + "order by\n" + ' "dogs_fts".rank', {"query": "Cleopaws"}, ), ] # Outside the with block collected should not be appended to - db["dogs"].insert({"name": "Cleopaws"}) - assert len(collected) == 5 + dogs.insert({"name": "Cleopaws"}) + assert len(collected) == 4 diff --git a/tests/test_transform.py b/tests/test_transform.py index 06e5729..362f1ca 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,4 +1,6 @@ -from sqlite_utils.db import ForeignKey +import sqlite3 + +from sqlite_utils.db import ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError import pytest @@ -10,80 +12,90 @@ import pytest ( {}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Change column type ( {"types": {"age": int}}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Rename a column ( {"rename": {"age": "dog_age"}}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" TEXT\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Drop a column ( {"drop": ["age"]}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name])\n SELECT [id], [name] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n SELECT "rowid", "id", "name" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Convert type AND rename column ( {"types": {"age": int}, "rename": {"age": "dog_age"}}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] INTEGER\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Change primary key ( {"pk": "age"}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT PRIMARY KEY\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT PRIMARY KEY\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Change primary key to a compound pk ( {"pk": ("age", "name")}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT,\n PRIMARY KEY ([age], [name])\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT,\n PRIMARY KEY ("age", "name")\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Remove primary key, creating a rowid table ( {"pk": None}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + ], + ), + # Keeping the table + ( + {"drop": ["age"], "keep_table": "kept_table"}, + [ + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n SELECT "rowid", "id", "name" FROM "dogs";', + 'ALTER TABLE "dogs" RENAME TO "kept_table";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), ], @@ -123,40 +135,40 @@ def test_transform_sql_table_with_primary_key( ( {}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Change column type ( {"types": {"age": int}}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] INTEGER\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" INTEGER\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Rename a column ( {"rename": {"age": "dog_age"}}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [dog_age] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "dog_age" TEXT\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), # Make ID a primary key ( {"pk": "id"}, [ - "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", - "DROP TABLE [dogs];", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n);', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'DROP TABLE "dogs";', + 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', ], ), ], @@ -194,13 +206,13 @@ def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db): dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) assert ( dogs.schema - == "CREATE TABLE [dogs] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n)" + == 'CREATE TABLE "dogs" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT\n)' ) dogs.transform(pk="id") # Slight oddity: [dogs] becomes "dogs" during the rename: assert ( dogs.schema - == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n)' + == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n)' ) @@ -210,17 +222,51 @@ def test_transform_rename_pk(fresh_db): dogs.transform(rename={"id": "pk"}) assert ( dogs.schema - == 'CREATE TABLE "dogs" (\n [pk] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n)' + == 'CREATE TABLE "dogs" (\n "pk" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n)' ) +def test_transform_preserves_keyword_literal_defaults(fresh_db): + # transform() used to requote keyword-literal defaults (DEFAULT TRUE became + # DEFAULT 'TRUE'), so a default insert stored the text 'TRUE' instead of the + # integer 1 -- silent value corruption on every rebuilt table. + fresh_db.execute( + "CREATE TABLE t (" + " id INTEGER PRIMARY KEY," + " is_active INTEGER DEFAULT TRUE," + " flag INTEGER DEFAULT FALSE," + " note TEXT DEFAULT NULL" + ")" + ) + table = fresh_db["t"] + table.insert({"id": 1}) + before = fresh_db.execute("SELECT is_active, flag, note FROM t").fetchone() + assert before == (1, 0, None) + + # Rebuild the table via an unrelated change. + table.transform(rename={"note": "note2"}) + + # The keyword literals stay unquoted in the schema ... + assert "DEFAULT TRUE" in table.schema + assert "DEFAULT FALSE" in table.schema + assert "DEFAULT NULL" in table.schema + assert "'TRUE'" not in table.schema + + # ... and a fresh default insert still yields 1 / 0 / NULL, not strings. + table.insert({"id": 2}) + after = fresh_db.execute( + "SELECT is_active, flag, note2 FROM t WHERE id = 2" + ).fetchone() + assert after == (1, 0, None) + + def test_transform_not_null(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") dogs.transform(not_null={"name"}) assert ( dogs.schema - == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT NOT NULL,\n [age] TEXT\n)' + == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT NOT NULL,\n "age" TEXT\n)' ) @@ -230,7 +276,7 @@ def test_transform_remove_a_not_null(fresh_db): dogs.transform(not_null={"name": True, "age": False}) assert ( dogs.schema - == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT NOT NULL,\n [age] TEXT\n)' + == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT NOT NULL,\n "age" TEXT\n)' ) @@ -241,7 +287,7 @@ def test_transform_add_not_null_with_rename(fresh_db, not_null): dogs.transform(not_null=not_null, rename={"age": "dog_age"}) assert ( dogs.schema - == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] TEXT NOT NULL\n)' + == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" TEXT NOT NULL\n)' ) @@ -251,7 +297,7 @@ def test_transform_defaults(fresh_db): dogs.transform(defaults={"age": 1}) assert ( dogs.schema - == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER DEFAULT 1\n)' + == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER DEFAULT 1\n)' ) @@ -261,7 +307,7 @@ def test_transform_defaults_and_rename_column(fresh_db): dogs.transform(rename={"age": "dog_age"}, defaults={"age": 1}) assert ( dogs.schema - == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] INTEGER DEFAULT 1\n)' + == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER DEFAULT 1\n)' ) @@ -271,7 +317,7 @@ def test_remove_defaults(fresh_db): dogs.transform(defaults={"age": None}) assert ( dogs.schema - == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER\n)' + == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n)' ) @@ -319,22 +365,29 @@ def test_transform_foreign_keys_survive_renamed_column( ] +def _add_country_city_continent(db): + db["country"].insert({"id": 1, "name": "France"}, pk="id") + db["continent"].insert({"id": 2, "name": "Europe"}, pk="id") + db["city"].insert({"id": 24, "name": "Paris"}, pk="id") + + +_CAVEAU = { + "id": 32, + "name": "Caveau de la Huchette", + "country": 1, + "continent": 2, + "city": 24, +} + + @pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys): if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") # Create table with three foreign keys so we can drop two of them - fresh_db["country"].insert({"id": 1, "name": "France"}, pk="id") - fresh_db["continent"].insert({"id": 2, "name": "Europe"}, pk="id") - fresh_db["city"].insert({"id": 24, "name": "Paris"}, pk="id") + _add_country_city_continent(fresh_db) fresh_db["places"].insert( - { - "id": 32, - "name": "Caveau de la Huchette", - "country": 1, - "continent": 2, - "city": 24, - }, + _CAVEAU, foreign_keys=("country", "continent", "city"), ) assert fresh_db["places"].foreign_keys == [ @@ -374,6 +427,480 @@ def test_transform_verify_foreign_keys(fresh_db): # This should have rolled us back assert ( fresh_db["authors"].schema - == "CREATE TABLE [authors] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)" + == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n)' ) assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_on_delete_cascade_does_not_delete_records( + fresh_db, use_pragma_foreign_keys +): + # Transforming a table drops and recreates it - if another table references + # it with ON DELETE CASCADE and PRAGMA foreign_keys is on, that drop must + # not cascade and delete the referencing records + if use_pragma_foreign_keys: + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + # Transform the table on the other end of the cascading foreign key + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["authors"].rows) == [ + {"id": 1, "author_name": "Ursula K. Le Guin"} + ] + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + # Transforming the table with the cascading foreign key should not + # delete its records either + fresh_db["books"].transform(rename={"title": "book_title"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "book_title": "The Dispossessed", "author_id": 1} + ] + if use_pragma_foreign_keys: + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +@pytest.mark.parametrize("on_delete", ["CASCADE", "SET NULL", "SET DEFAULT", "cascade"]) +def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_delete): + # PRAGMA foreign_keys is a no-op inside a transaction, so transforming a + # table referenced by ON DELETE CASCADE / SET NULL / SET DEFAULT foreign + # keys inside an open transaction would fire those actions when the old + # table is dropped - transform() should refuse instead + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE {} + ); + """.format(on_delete)) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + previous_schema = fresh_db["authors"].schema + with fresh_db.atomic(): + with pytest.raises(TransactionError) as excinfo: + fresh_db["authors"].transform(rename={"name": "author_name"}) + message = str(excinfo.value) + assert "books" in message + assert "ON DELETE {}".format(on_delete.upper()) in message + # Nothing should have changed + assert fresh_db["authors"].schema == previous_schema + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db): + # The copied table carries a foreign key referencing the original table + # name, so a self-referential cascade would wipe the copy too + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE categories ( + id INTEGER PRIMARY KEY, + name TEXT, + parent_id INTEGER REFERENCES categories(id) ON DELETE CASCADE + ); + """) + fresh_db["categories"].insert_all( + [ + {"id": 1, "name": "Fiction", "parent_id": None}, + {"id": 2, "name": "Science Fiction", "parent_id": 1}, + ] + ) + with fresh_db.atomic(): + with pytest.raises(TransactionError) as excinfo: + fresh_db["categories"].transform(rename={"name": "title"}) + assert "categories" in str(excinfo.value) + assert fresh_db["categories"].count == 2 + + +def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db): + # An inbound foreign key without a destructive ON DELETE action is safe + # inside a transaction thanks to PRAGMA defer_foreign_keys + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["authors"].rows) == [ + {"id": 1, "author_name": "Ursula K. Le Guin"} + ] + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +def test_transform_in_transaction_allowed_for_child_table(fresh_db): + # The table being transformed only has an outbound foreign key - dropping + # it fires no ON DELETE actions, so this is allowed inside a transaction + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + with fresh_db.atomic(): + fresh_db["books"].transform(rename={"title": "book_title"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "book_title": "The Dispossessed", "author_id": 1} + ] + + +def test_transform_in_transaction_allowed_with_foreign_keys_off(fresh_db): + # With PRAGMA foreign_keys off (the default) no cascades can fire, so + # transform inside a transaction is safe even with a CASCADE schema + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + + +def test_transform_add_foreign_keys_from_scratch(fresh_db): + _add_country_city_continent(fresh_db) + fresh_db["places"].insert(_CAVEAU) + # Should have no foreign keys + assert fresh_db["places"].foreign_keys == [] + # Now add them using .transform() + fresh_db["places"].transform(add_foreign_keys=("country", "continent", "city")) + # Should now have all three: + assert fresh_db["places"].foreign_keys == [ + ForeignKey( + table="places", column="city", other_table="city", other_column="id" + ), + ForeignKey( + table="places", + column="continent", + other_table="continent", + other_column="id", + ), + ForeignKey( + table="places", column="country", other_table="country", other_column="id" + ), + ] + assert fresh_db["places"].schema == ( + 'CREATE TABLE "places" (\n' + ' "id" INTEGER,\n' + ' "name" TEXT,\n' + ' "country" INTEGER REFERENCES "country"("id"),\n' + ' "continent" INTEGER REFERENCES "continent"("id"),\n' + ' "city" INTEGER REFERENCES "city"("id")\n' + ")" + ) + + +@pytest.mark.parametrize( + "add_foreign_keys", + ( + ("country", "continent"), + # Fully specified + ( + ("country", "country", "id"), + ("continent", "continent", "id"), + ), + ), +) +def test_transform_add_foreign_keys_from_partial(fresh_db, add_foreign_keys): + _add_country_city_continent(fresh_db) + fresh_db["places"].insert( + _CAVEAU, + foreign_keys=("city",), + ) + # Should have one foreign keys + assert fresh_db["places"].foreign_keys == [ + ForeignKey(table="places", column="city", other_table="city", other_column="id") + ] + # Now add three more using .transform() + fresh_db["places"].transform(add_foreign_keys=add_foreign_keys) + # Should now have all three: + assert fresh_db["places"].foreign_keys == [ + ForeignKey( + table="places", column="city", other_table="city", other_column="id" + ), + ForeignKey( + table="places", + column="continent", + other_table="continent", + other_column="id", + ), + ForeignKey( + table="places", column="country", other_table="country", other_column="id" + ), + ] + + +@pytest.mark.parametrize( + "foreign_keys", + ( + ("country", "continent"), + # Fully specified + ( + ("country", "country", "id"), + ("continent", "continent", "id"), + ), + ), +) +def test_transform_replace_foreign_keys(fresh_db, foreign_keys): + _add_country_city_continent(fresh_db) + fresh_db["places"].insert( + _CAVEAU, + foreign_keys=("city",), + ) + assert len(fresh_db["places"].foreign_keys) == 1 + # Replace with two different ones + fresh_db["places"].transform(foreign_keys=foreign_keys) + assert fresh_db["places"].schema == ( + 'CREATE TABLE "places" (\n' + ' "id" INTEGER,\n' + ' "name" TEXT,\n' + ' "country" INTEGER REFERENCES "country"("id"),\n' + ' "continent" INTEGER REFERENCES "continent"("id"),\n' + ' "city" INTEGER\n' + ")" + ) + + +@pytest.mark.parametrize("table_type", ("id_pk", "rowid", "compound_pk")) +def test_transform_preserves_rowids(fresh_db, table_type): + pk = None + if table_type == "id_pk": + pk = "id" + elif table_type == "compound_pk": + pk = ("id", "name") + elif table_type == "rowid": + pk = None + fresh_db["places"].insert_all( + [ + {"id": "1", "name": "Paris", "country": "France"}, + {"id": "2", "name": "London", "country": "UK"}, + {"id": "3", "name": "New York", "country": "USA"}, + ], + pk=pk, + ) + # Now delete and insert a row to mix up the `rowid` sequence + fresh_db["places"].delete_where("id = ?", ["2"]) + fresh_db["places"].insert({"id": "4", "name": "London", "country": "UK"}) + previous_rows = list( + tuple(row) for row in fresh_db.execute("select rowid, id, name from places") + ) + # Transform it + fresh_db["places"].transform(column_order=("country", "name")) + # Should be the same + next_rows = list( + tuple(row) for row in fresh_db.execute("select rowid, id, name from places") + ) + assert previous_rows == next_rows + + +@pytest.mark.parametrize( + "initial_strict,transform_strict,expected_strict", + ( + (False, None, False), + (True, None, True), + (False, True, True), + (True, False, False), + ), +) +def test_transform_strict(fresh_db, initial_strict, transform_strict, expected_strict): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + dogs = fresh_db.table("dogs", strict=initial_strict) + dogs.insert({"id": 1, "name": "Cleo"}) + assert dogs.strict is initial_strict + dogs.transform(strict=transform_strict) + assert dogs.strict is expected_strict + + +def test_transform_to_strict_with_invalid_data(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + dogs = fresh_db["dogs"] + dogs.create({"id": int}) + dogs.insert({"id": "not-an-integer"}) + + with pytest.raises(sqlite3.IntegrityError): + dogs.transform(strict=True) + + assert dogs.strict is False + assert list(dogs.rows) == [{"id": "not-an-integer"}] + assert fresh_db.table_names() == ["dogs"] + + +def test_transform_strict_updates_default(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + table = fresh_db.table("items", strict=True) + table.create({"id": int}) + + table.transform(strict=False) + assert table.strict is False + + table.create({"id": int}, replace=True) + assert table.strict is False + + +@pytest.mark.parametrize("method_name", ("transform", "transform_sql")) +def test_transform_to_strict_not_supported(fresh_db, method_name): + table = fresh_db["items"] + table.create({"id": int}) + fresh_db._supports_strict = False + + with pytest.raises(TransformError, match="SQLite does not support STRICT tables"): + getattr(table, method_name)(strict=True) + + assert table.strict is False + + +@pytest.mark.parametrize( + "indexes, transform_params", + [ + ([["name"]], {"types": {"age": str}}), + ([["name"], ["age", "breed"]], {"types": {"age": str}}), + ([], {"types": {"age": str}}), + ([["name"]], {"types": {"age": str}, "keep_table": "old_dogs"}), + ], +) +def test_transform_indexes(fresh_db, indexes, transform_params): + # https://github.com/simonw/sqlite-utils/issues/633 + # New table should have same indexes as old table after transformation + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": 5, "breed": "Labrador"}, pk="id") + + for index in indexes: + dogs.create_index(index) + + indexes_before_transform = dogs.indexes + + dogs.transform(**transform_params) + + assert sorted( + [ + {k: v for k, v in idx._asdict().items() if k != "seq"} + for idx in dogs.indexes + ], + key=lambda x: x["name"], + ) == sorted( + [ + {k: v for k, v in idx._asdict().items() if k != "seq"} + for idx in indexes_before_transform + ], + key=lambda x: x["name"], + ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}" + if "keep_table" in transform_params: + assert all( + index.origin == "pk" + for index in fresh_db[transform_params["keep_table"]].indexes + ) + + +def test_transform_retains_indexes_with_foreign_keys(fresh_db): + dogs = fresh_db["dogs"] + owners = fresh_db["owners"] + + dogs.insert({"id": 1, "name": "Cleo", "owner_id": 1}, pk="id") + owners.insert({"id": 1, "name": "Alice"}, pk="id") + + dogs.create_index(["name"]) + + indexes_before_transform = dogs.indexes + + fresh_db.add_foreign_keys([("dogs", "owner_id", "owners", "id")]) # calls transform + + assert sorted( + [ + {k: v for k, v in idx._asdict().items() if k != "seq"} + for idx in dogs.indexes + ], + key=lambda x: x["name"], + ) == sorted( + [ + {k: v for k, v in idx._asdict().items() if k != "seq"} + for idx in indexes_before_transform + ], + key=lambda x: x["name"], + ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}" + + +@pytest.mark.parametrize( + "transform_params", + [ + {"rename": {"age": "dog_age"}}, + {"drop": ["age"]}, + ], +) +def test_transform_with_indexes_errors(fresh_db, transform_params): + # Should error with a compound (name, age) index if age is renamed or dropped + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") + + dogs.create_index(["name", "age"]) + + with pytest.raises(TransformError) as excinfo: + dogs.transform(**transform_params) + + assert ( + "Index 'idx_dogs_name_age' column 'age' is not in updated table 'dogs'. " + "You must manually drop this index prior to running this transformation" + in str(excinfo.value) + ) + + +def test_transform_with_unique_constraint_implicit_index(fresh_db): + dogs = fresh_db["dogs"] + # Create a table with a UNIQUE constraint on 'name', which creates an implicit index + fresh_db.execute(""" + CREATE TABLE dogs ( + id INTEGER PRIMARY KEY, + name TEXT UNIQUE, + age INTEGER + ); + """) + dogs.insert({"id": 1, "name": "Cleo", "age": 5}) + + # Attempt to transform the table without modifying 'name' + with pytest.raises(TransformError) as excinfo: + dogs.transform(types={"age": str}) + + assert ( + "Index 'sqlite_autoindex_dogs_1' on table 'dogs' does not have a CREATE INDEX statement." + in str(excinfo.value) + ) + assert ( + "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation." + in str(excinfo.value) + ) diff --git a/tests/test_update.py b/tests/test_update.py index 7e3e0ab..03bec11 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -70,20 +70,21 @@ def test_update_alter(fresh_db): ] == list(table.rows) -def test_update_alter_with_invalid_column_characters(fresh_db): +def test_update_alter_with_special_column_characters(fresh_db): + # With double-quote escaping, columns with special characters are now valid table = fresh_db["table"] rowid = table.insert({"foo": "bar"}).last_pk - with pytest.raises(AssertionError): - table.update(rowid, {"new_col[abc]": 1.2}, alter=True) + table.update(rowid, {"new_col[abc]": 1.2}, alter=True) + assert list(table.rows) == [{"foo": "bar", "new_col[abc]": 1.2}] def test_update_with_no_values_sets_last_pk(fresh_db): table = fresh_db.table("dogs", pk="id") table.insert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Pancakes"}]) table.update(1) - assert 1 == table.last_pk + assert table.last_pk == 1 table.update(2) - assert 2 == table.last_pk + assert table.last_pk == 2 with pytest.raises(NotFoundError): table.update(3) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 09bdacc..a782b26 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -1,33 +1,46 @@ from sqlite_utils.db import PrimaryKeyRequired +from sqlite_utils import Database import pytest -def test_upsert(fresh_db): - table = fresh_db["table"] +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert(use_old_upsert): + db = Database(memory=True, use_old_upsert=use_old_upsert) + table = db["table"] table.insert({"id": 1, "name": "Cleo"}, pk="id") table.upsert({"id": 1, "age": 5}, pk="id", alter=True) - assert [{"id": 1, "name": "Cleo", "age": 5}] == list(table.rows) - assert 1 == table.last_pk + assert list(table.rows) == [{"id": 1, "name": "Cleo", "age": 5}] + assert table.last_pk == 1 def test_upsert_all(fresh_db): table = fresh_db["table"] table.upsert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Nixie"}], pk="id") table.upsert_all([{"id": 1, "age": 5}, {"id": 2, "age": 5}], pk="id", alter=True) - assert [ + assert list(table.rows) == [ {"id": 1, "name": "Cleo", "age": 5}, {"id": 2, "name": "Nixie", "age": 5}, - ] == list(table.rows) + ] assert table.last_pk is None def test_upsert_all_single_column(fresh_db): table = fresh_db["table"] table.upsert_all([{"name": "Cleo"}], pk="name") - assert [{"name": "Cleo"}] == list(table.rows) + assert list(table.rows) == [{"name": "Cleo"}] assert table.pks == ["name"] +def test_upsert_all_not_null(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/538 + fresh_db["comments"].upsert_all( + [{"id": 1, "name": "Cleo"}], + pk="id", + not_null=["name"], + ) + assert list(fresh_db["comments"].rows) == [{"id": 1, "name": "Cleo"}] + + def test_upsert_error_if_no_pk(fresh_db): table = fresh_db["table"] with pytest.raises(PrimaryKeyRequired): @@ -36,6 +49,89 @@ def test_upsert_error_if_no_pk(fresh_db): table.upsert({"id": 1, "name": "Cleo"}) +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_empty_record_errors(use_old_upsert): + db = Database(memory=True, use_old_upsert=use_old_upsert) + table = db["table"] + table.insert({"id": 1, "name": "Cleo"}, pk="id") + with pytest.raises(PrimaryKeyRequired): + table.upsert({}, pk="id") + with pytest.raises(PrimaryKeyRequired): + table.upsert_all([{}, {}], pk="id") + # No rows can have been inserted + assert table.count == 1 + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_missing_pk_value_errors(use_old_upsert): + db = Database(memory=True, use_old_upsert=use_old_upsert) + table = db["table"] + table.insert({"id": 1, "name": "Cleo"}, pk="id") + # Records that omit the pk column entirely + with pytest.raises(PrimaryKeyRequired): + table.upsert_all([{"name": "Pancakes"}, {"name": "Marnie"}], pk="id") + # A record with an explicit None pk value can never conflict + with pytest.raises(PrimaryKeyRequired): + table.upsert({"id": None, "name": "Pancakes"}, pk="id") + assert list(table.rows) == [{"id": 1, "name": "Cleo"}] + + +def test_upsert_missing_compound_pk_value_errors(fresh_db): + table = fresh_db["table"] + table.insert({"a": "x", "b": "y", "v": 1}, pk=("a", "b")) + # Missing one component of the detected compound primary key + with pytest.raises(PrimaryKeyRequired): + table.upsert({"a": "x", "v": 2}) + assert list(table.rows) == [{"a": "x", "b": "y", "v": 1}] + + +def test_upsert_error_if_existing_table_has_no_pk(fresh_db): + table = fresh_db.create_table("table", {"id": int, "name": str}) + with pytest.raises(PrimaryKeyRequired): + table.upsert({"id": 1, "name": "Cleo"}) + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_uses_compound_pk_from_existing_table(use_old_upsert): + # https://github.com/simonw/sqlite-utils/issues/629 + db = Database(memory=True, use_old_upsert=use_old_upsert) + db.execute(""" + create table summary ( + Source text, + Object text, + Category text, + Count integer, + primary key (Source, Object, Category) + ) + """) + table = db["summary"] + table.upsert( + { + "Source": "Client A", + "Object": "Accounts", + "Category": "All", + "Count": 3, + } + ) + assert table.last_pk == ("Client A", "Accounts", "All") + table.upsert( + { + "Source": "Client A", + "Object": "Accounts", + "Category": "All", + "Count": 4, + } + ) + assert list(table.rows) == [ + { + "Source": "Client A", + "Object": "Accounts", + "Category": "All", + "Count": 4, + } + ] + + def test_upsert_with_hash_id(fresh_db): table = fresh_db["table"] table.upsert({"foo": "bar"}, hash_id="pk") @@ -45,6 +141,30 @@ def test_upsert_with_hash_id(fresh_db): assert "a5e744d0164540d33b1d7ea616c28f2fa97e754a" == table.last_pk +@pytest.mark.parametrize("hash_id", (None, "custom_id")) +def test_upsert_with_hash_id_columns(fresh_db, hash_id): + table = fresh_db["table"] + table.upsert({"a": 1, "b": 2, "c": 3}, hash_id=hash_id, hash_id_columns=("a", "b")) + assert list(table.rows) == [ + { + hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49", + "a": 1, + "b": 2, + "c": 3, + } + ] + assert table.last_pk == "4acc71e0547112eb432f0a36fb1924c4a738cb49" + table.upsert({"a": 1, "b": 2, "c": 4}, hash_id=hash_id, hash_id_columns=("a", "b")) + assert list(table.rows) == [ + { + hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49", + "a": 1, + "b": 2, + "c": 4, + } + ] + + def test_upsert_compound_primary_key(fresh_db): table = fresh_db["table"] table.upsert_all( diff --git a/tests/test_utils.py b/tests/test_utils.py index a44b2e0..3de5e94 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,6 @@ from sqlite_utils import utils +import csv +import io import pytest @@ -22,6 +24,79 @@ def test_decode_base64_values(input, expected, should_be_is): assert actual == expected -def test_find_spatialite(): - spatialite = utils.find_spatialite() - assert spatialite is None or isinstance(spatialite, str) +@pytest.mark.parametrize( + "size,expected", + ( + (1, [["a"], ["b"], ["c"], ["d"]]), + (2, [["a", "b"], ["c", "d"]]), + (3, [["a", "b", "c"], ["d"]]), + (4, [["a", "b", "c", "d"]]), + ), +) +def test_chunks(size, expected): + input = ["a", "b", "c", "d"] + chunks = list(map(list, utils.chunks(input, size))) + assert chunks == expected + + +def test_hash_record(): + expected = "d383e7c0ba88f5ffcdd09be660de164b3847401a" + assert utils.hash_record({"name": "Cleo", "twitter": "CleoPaws"}) == expected + assert ( + utils.hash_record( + {"name": "Cleo", "twitter": "CleoPaws", "age": 7}, keys=("name", "twitter") + ) + == expected + ) + assert ( + utils.hash_record({"name": "Cleo", "twitter": "CleoPaws", "age": 7}) != expected + ) + + +def test_maximize_csv_field_size_limit(): + # Reset to default in case other tests have changed it + csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT) + long_value = "a" * 131073 + long_csv = "id,text\n1,{}".format(long_value) + fp = io.BytesIO(long_csv.encode("utf-8")) + # Using rows_from_file should error + with pytest.raises(csv.Error): + rows, _ = utils.rows_from_file(fp, utils.Format.CSV) + list(rows) + # But if we call maximize_csv_field_size_limit() first it should be OK: + utils.maximize_csv_field_size_limit() + fp2 = io.BytesIO(long_csv.encode("utf-8")) + rows2, _ = utils.rows_from_file(fp2, utils.Format.CSV) + rows_list2 = list(rows2) + assert len(rows_list2) == 1 + assert rows_list2[0]["id"] == "1" + assert rows_list2[0]["text"] == long_value + + +@pytest.mark.parametrize( + "input,expected", + ( + ({"foo": {"bar": 1}}, {"foo_bar": 1}), + ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}), + ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}), + ), +) +def test_flatten(input, expected): + assert utils.flatten(input) == expected + + +@pytest.mark.parametrize( + "input,expected", + ( + ([], []), + (["id", "name"], ["id", "name"]), + (["id", "id"], ["id", "id_2"]), + (["id", "id", "id"], ["id", "id_2", "id_3"]), + # A renamed duplicate must not clobber a real column called id_2 + (["id", "id", "id_2"], ["id", "id_3", "id_2"]), + (["id_2", "id", "id"], ["id_2", "id", "id_3"]), + (["id", "id", "id_2", "id_2"], ["id", "id_3", "id_2", "id_2_2"]), + ), +) +def test_dedupe_keys(input, expected): + assert utils.dedupe_keys(input) == expected diff --git a/tests/test_wal.py b/tests/test_wal.py index 23ca144..2ddcf54 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -1,5 +1,6 @@ import pytest from sqlite_utils import Database +from sqlite_utils.db import TransactionError @pytest.fixture @@ -21,3 +22,67 @@ def test_enable_disable_wal(db_path_tmpdir): db.disable_wal() assert "delete" == db.journal_mode assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] + + +def test_enable_wal_inside_transaction_raises(db_path_tmpdir): + db, path, tmpdir = db_path_tmpdir + db["test"].insert({"id": 1}, pk="id") + with pytest.raises(TransactionError): + with db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.enable_wal() + # The atomic() block must have rolled back cleanly and the + # journal mode must be unchanged + assert db.journal_mode == "delete" + assert [r["id"] for r in db["test"].rows] == [1] + + +def test_disable_wal_inside_transaction_raises(db_path_tmpdir): + db, path, tmpdir = db_path_tmpdir + db.enable_wal() + db["test"].insert({"id": 1}, pk="id") + with pytest.raises(TransactionError): + with db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.disable_wal() + assert db.journal_mode == "wal" + assert [r["id"] for r in db["test"].rows] == [1] + + +def test_ensure_autocommit_on(db_path_tmpdir): + db, path, tmpdir = db_path_tmpdir + previous_isolation_level = db.conn.isolation_level + assert previous_isolation_level is not None + with db.ensure_autocommit_on(): + # isolation_level of None means driver-level autocommit mode + assert db.conn.isolation_level is None + # Restored afterwards + assert db.conn.isolation_level == previous_isolation_level + + +def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): + # Calling enable_wal() when WAL is already enabled is a no-op, + # so it is fine inside a transaction + db, path, tmpdir = db_path_tmpdir + db.enable_wal() + with db.atomic(): + db["test"].insert({"id": 1}, pk="id") + db.enable_wal() + assert [r["id"] for r in db["test"].rows] == [1] + + +def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): + # Setting isolation_level commits any pending transaction as a side + # effect, silently breaking the caller's rollback guarantee - so + # entering autocommit mode with a transaction open is an error + db, path, tmpdir = db_path_tmpdir + db["test"].insert({"id": 1}, pk="id") + db.begin() + db.execute("insert into test (id) values (2)") + with pytest.raises(TransactionError): + with db.ensure_autocommit_on(): + pass + # The transaction is still open and can still be rolled back + assert db.conn.in_transaction + db.rollback() + assert [r["id"] for r in db["test"].rows] == [1] diff --git a/tests/utils.py b/tests/utils.py deleted file mode 100644 index 1e3d968..0000000 --- a/tests/utils.py +++ /dev/null @@ -1,7 +0,0 @@ -import re - -COLLAPSE_RE = re.compile(r"\s+") - - -def collapse_whitespace(s): - return COLLAPSE_RE.sub(" ", s)