Changelog now links to CLI and Python API in most recent entry

Refs #791, #792, #794, #795

.github/actions/setup-sqlite-version/action.yml

@@ -0,0 +1,39 @@
+name: "Setup SQLite version"
+description: "Build and activate a specific SQLite version from its amalgamation archive"
+inputs:
+  version:
+    description: "The SQLite version to install"
+    required: true
+  cflags:
+    description: "CFLAGS to use when compiling SQLite"
+    required: false
+    default: ""
+  skip-activate:
+    description: "Set to true to skip modifying the library path"
+    required: false
+    default: "false"
+  fallback-urls:
+    description: "Whitespace-separated fallback download URLs to try after sqlite.org"
+    required: false
+    default: ""
+outputs:
+  sqlite-location:
+    description: "Directory containing the compiled SQLite library"
+    value: ${{ steps.build.outputs.sqlite-location }}
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads"
+    - uses: actions/cache@v6
+      with:
+        path: ${{ runner.temp }}/sqlite-versions/downloads
+        key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1
+    - id: build
+      shell: bash
+      run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh"
+      env:
+        SQLITE_VERSION: ${{ inputs.version }}
+        SQLITE_CFLAGS: ${{ inputs.cflags }}
+        SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }}
+        SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }}

.github/actions/setup-sqlite-version/setup-sqlite-version.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}"
+cflags="${SQLITE_CFLAGS:-}"
+skip_activate="${SQLITE_SKIP_ACTIVATE:-false}"
+extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}"
+
+case "$version_spec" in
+  3.46 | 3.46.0)
+    sqlite_version="3.46.0"
+    sqlite_year="2024"
+    amalgamation_id="3460000"
+    builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip"
+    ;;
+  3.23.1)
+    sqlite_version="3.23.1"
+    sqlite_year="2018"
+    amalgamation_id="3230100"
+    builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3230100.zip"
+    ;;
+  *)
+    echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh."
+    exit 1
+    ;;
+esac
+
+case "$(uname -s)" in
+  Linux)
+    library_name="libsqlite3.so.0"
+    library_path_var="LD_LIBRARY_PATH"
+    ;;
+  Darwin)
+    library_name="libsqlite3.dylib"
+    library_path_var="DYLD_LIBRARY_PATH"
+    ;;
+  *)
+    echo "::error::Unsupported platform $(uname -s)"
+    exit 1
+    ;;
+esac
+
+runner_temp="${RUNNER_TEMP:-}"
+if [ -z "$runner_temp" ]; then
+  runner_temp="$(mktemp -d)"
+fi
+
+filename="sqlite-amalgamation-${amalgamation_id}"
+official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip"
+download_dir="${runner_temp}/sqlite-versions/downloads"
+source_root="${runner_temp}/sqlite-versions/source"
+source_dir="${source_root}/${filename}"
+build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}"
+archive_path="${download_dir}/${filename}.zip"
+
+mkdir -p "$download_dir" "$source_root" "$build_dir"
+
+download_archive() {
+  local url
+  local candidate_path="${archive_path}.tmp"
+  local urls=("$official_url")
+
+  for url in $builtin_fallback_urls $extra_fallback_urls; do
+    urls+=("$url")
+  done
+
+  rm -f "$candidate_path"
+  for url in "${urls[@]}"; do
+    echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}"
+    if curl \
+      --fail \
+      --location \
+      --show-error \
+      --retry 5 \
+      --retry-delay 2 \
+      --retry-max-time 180 \
+      --retry-all-errors \
+      --connect-timeout 20 \
+      --max-time 240 \
+      --output "$candidate_path" \
+      "$url"; then
+      mv "$candidate_path" "$archive_path"
+      return 0
+    fi
+
+    echo "::warning::Download failed from ${url}"
+    rm -f "$candidate_path"
+  done
+
+  echo "::error::Could not download SQLite ${sqlite_version} amalgamation"
+  return 1
+}
+
+if [ ! -f "${source_dir}/sqlite3.c" ]; then
+  if [ ! -f "$archive_path" ]; then
+    download_archive
+  fi
+
+  rm -rf "$source_dir"
+  unzip -q "$archive_path" -d "$source_root"
+fi
+
+if [ ! -f "${source_dir}/sqlite3.c" ]; then
+  echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}"
+  exit 1
+fi
+
+read -r -a cflag_args <<< "$cflags"
+
+echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}"
+gcc \
+  -fPIC \
+  -shared \
+  "${cflag_args[@]}" \
+  "${source_dir}/sqlite3.c" \
+  "-I${source_dir}" \
+  -o "${build_dir}/${library_name}"
+
+if [ "$library_name" = "libsqlite3.so.0" ]; then
+  ln -sf "$library_name" "${build_dir}/libsqlite3.so"
+fi
+
+if [ -n "${GITHUB_OUTPUT:-}" ]; then
+  echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT"
+else
+  echo "sqlite-location=${build_dir}"
+fi
+
+case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in
+  true | 1 | yes)
+    echo "Skipping ${library_path_var} activation"
+    ;;
+  *)
+    existing_value="${!library_path_var:-}"
+    if [ -n "${GITHUB_ENV:-}" ]; then
+      if [ -n "$existing_value" ]; then
+        echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV"
+      else
+        echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV"
+      fi
+    fi
+    echo "Added ${build_dir} to ${library_path_var}"
+    ;;
+esac

.github/workflows/documentation-links.yml

@@ -0,0 +1,16 @@
+name: Read the Docs Pull Request Preview
+on:
+  pull_request_target:
+    types:
+      - opened
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: "sqlite-utils"

.github/workflows/publish.yml

@@ -9,24 +9,19 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: actions/cache@v2
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        cache: pip
+        cache-dependency-path: pyproject.toml
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install . --group dev
     - name: Run tests
       run: |
         pytest
@@ -34,25 +29,20 @@ jobs:
     runs-on: ubuntu-latest
     needs: [test]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v7
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
-        python-version: '3.9'
-    - uses: actions/cache@v2
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-publish-pip-
+        python-version: '3.14'
+        cache: pip
+        cache-dependency-path: pyproject.toml
     - name: Install dependencies
       run: |
-        pip install setuptools wheel twine
+        pip install build twine
     - name: Publish
       env:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
       run: |
-        python setup.py sdist bdist_wheel
+        python -m build
         twine upload dist/*

.github/workflows/spellcheck.yml

@@ -6,21 +6,16 @@ jobs:
   spellcheck:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.9
-    - uses: actions/cache@v2
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        python-version: "3.12"
+        cache: pip
+        cache-dependency-path: pyproject.toml
     - name: Install dependencies
       run: |
-        pip install -e '.[docs]'
+        pip install . --group docs
     - name: Check spelling
       run: |
         codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt

.github/workflows/test-coverage.yml

@@ -12,22 +12,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v7
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
-        python-version: 3.9
-    - uses: actions/cache@v2
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        python-version: "3.11"
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Install SpatiaLite
+      run: sudo apt-get install libsqlite3-mod-spatialite
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -e .[test]
+        python -m pip install . --group dev
         python -m pip install pytest-cov
     - name: Run tests
       run: |-

.github/workflows/test-sqlite-support.yml

@@ -0,0 +1,41 @@
+name: Test SQLite versions
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ${{ matrix.platform }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        platform: [ubuntu-latest]
+        python-version: ["3.10"]
+        sqlite-version: [
+          "3.46",
+          "3.23.1", # 2018-04-10, before UPSERT
+        ]
+    steps:
+    - uses: actions/checkout@v7
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Set up SQLite ${{ matrix.sqlite-version }}
+      uses: ./.github/actions/setup-sqlite-version
+      with:
+        version: ${{ matrix.sqlite-version }}
+        cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1"
+    - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)"
+    - name: Install dependencies
+      run: |
+        pip install . --group dev
+        pip freeze
+    - name: Run tests
+      run: |
+        python -m pytest

.github/workflows/test.yml

@@ -1,40 +1,57 @@
 name: Test
 
-on: [push]
+on: [push, pull_request]
+
+env:
+  FORCE_COLOR: 1
 
 jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15-dev"]
         numpy: [0, 1]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest, macos-14]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: actions/cache@v2
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        allow-prereleases: true
+        cache: pip
+        cache-dependency-path: pyproject.toml
     - name: Install dependencies
       run: |
-        pip install -e '.[test,mypy,flake8]'
+        pip install . --group dev
     - name: Optionally install numpy
       if: matrix.numpy == 1
       run: pip install numpy
+    - name: Install SpatiaLite
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install libsqlite3-mod-spatialite
+    - name: Build extension for --load-extension test
+      if: matrix.os == 'ubuntu-latest'
+      run: |-
+        (cd tests && gcc ext.c -fPIC -shared -o ext.so && ls -lah)
     - name: Run tests
       run: |
-        pytest
+        pytest -v
+    - name: Run autocommit tests just on 3.14/Ubuntu
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14'
+      run: pytest --sqlite-autocommit
     - name: run mypy
-      run: mypy sqlite_utils
+      run: mypy sqlite_utils tests
     - name: run flake8
       run: flake8
+    - name: run ty
+      if: matrix.os != 'windows-latest' && matrix.python-version == '3.14'
+      run: |
+        pip install uv
+        uv run ty check sqlite_utils
     - name: Check formatting
       run: black . --check
+    - name: Check if cog needs to be run
+      run: |
+        cog --check --diff README.md docs/*.rst

.gitignore

@@ -1,4 +1,6 @@
 .venv
+dist
+build
 *.db
 __pycache__/
 *.py[cod]
@@ -12,3 +14,10 @@ venv
 .coverage
 .schema
 .vscode
+.hypothesis
+Pipfile
+Pipfile.lock
+uv.lock
+tests/*.dylib
+tests/*.so
+tests/*.dll

.readthedocs.yaml

@@ -0,0 +1,17 @@
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+  jobs:
+    install:
+    - pip install --upgrade pip
+    - pip install . --group docs
+
+formats:
+- pdf
+- epub

Justfile

@@ -0,0 +1,33 @@
+# Run tests and linters
+@default: test lint
+
+# Run pytest with supplied options
+@test *options:
+  uv run pytest {{options}}
+
+@run *options:
+  uv run -- {{options}}
+
+# Run linters: black, flake8, mypy, ty, cog
+@lint:
+  just run black . --check
+  uv run flake8
+  uv run mypy sqlite_utils tests
+  uv run ty check sqlite_utils
+  uv run cog --check README.md docs/*.rst
+  uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
+
+# Rebuild docs with cog
+@cog:
+  uv run --group docs cog -r README.md docs/*.rst
+
+# Serve live docs on localhost:8000
+@docs: cog
+  #!/usr/bin/env bash
+  cd docs
+  uv run --group docs make livehtml
+
+
+# Apply Black
+@black:
+  uv run black .

README.md

@@ -1,24 +1,29 @@
 # sqlite-utils
 
 [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/)
-[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/latest/changelog.html)
+[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/stable/changelog.html)
 [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/)
 [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest)
-[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest)
+[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable)
 [![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE)
+[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/Ass7bCAMDw)
 
 Python CLI utility and library for manipulating SQLite databases.
 
 ## Some feature highlights
 
 - [Pipe JSON](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema
+- [Run in-memory SQL queries](https://sqlite-utils.datasette.io/en/stable/cli.html#querying-data-directly-using-an-in-memory-database), including joins, directly against data in CSV, TSV or JSON files and view the results
 - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance
-- Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as dropping columns
+- Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as changing the type of a column
 - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data
+- [Manage database migrations](https://sqlite-utils.datasette.io/en/stable/migrations.html) using Python migration files and the `sqlite-utils migrate` command
+- [Install plugins](https://sqlite-utils.datasette.io/en/stable/plugins.html) to add custom SQL functions and additional features
 
-Read more on my blog: [
-sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/).
+Upgrading from sqlite-utils 3.x? See the [4.0 upgrade guide](https://sqlite-utils.datasette.io/en/stable/upgrading.html#upgrading-from-3-x-to-4-0).
+
+Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqlite-utils](https://simonwillison.net/tags/sqlite-utils/).
 
 ## Installation
 
@@ -32,12 +37,19 @@ Or if you use [Homebrew](https://brew.sh/) for macOS:
 
 Now you can do things with the CLI utility like this:
 
+    $ sqlite-utils memory dogs.csv "select * from t"
+    [{"id": 1, "age": 4, "name": "Cleo"},
+     {"id": 2, "age": 2, "name": "Pancakes"}]
+
+    $ sqlite-utils insert dogs.db dogs dogs.csv --csv
+    [####################################]  100%
+
     $ sqlite-utils tables dogs.db --counts
     [{"table": "dogs", "count": 2}]
 
-    $ sqlite-utils dogs.db "select * from dogs"
-    [{"id": 1, "age": 4, "name": "Cleo"},
-     {"id": 2, "age": 2, "name": "Pancakes"}]
+    $ sqlite-utils dogs.db "select id, name from dogs"
+    [{"id": 1, "name": "Cleo"},
+     {"id": 2, "name": "Pancakes"}]
 
     $ sqlite-utils dogs.db "select * from dogs" --csv
     id,age,name
@@ -61,7 +73,7 @@ Or for data in a CSV file:
 
 `sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command:
 
-    $ cat dogs.csv | sqlite-utils memory - "select name, age from dogs"
+    $ cat dogs.csv | sqlite-utils memory - "select name, age from stdin"
 
 See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands.
 

docs/Makefile

@@ -20,4 +20,4 @@ help:
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 livehtml:
-	sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0)
+	sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils

docs/_static/js/custom.js

@@ -0,0 +1,23 @@
+jQuery(function ($) {
+  // Show banner linking to /stable/ if this is a /latest/ page
+  if (!/\/latest\//.test(location.pathname)) {
+    return;
+  }
+  var stableUrl = location.pathname.replace("/latest/", "/stable/");
+  // Check it's not a 404
+  fetch(stableUrl, { method: "HEAD" }).then((response) => {
+    if (response.status == 200) {
+      var warning = $(
+        `<div class="admonition warning">
+           <p class="first admonition-title">Note</p>
+           <p class="last">
+             This documentation covers the <strong>development version</strong> of <code>sqlite-utils</code>.</p>
+             <p>See <a href="${stableUrl}">this page</a> for the current stable release.
+           </p>
+        </div>`
+      );
+      warning.find("a").attr("href", stableUrl);
+      $("article[role=main]").prepend(warning);
+    }
+  });
+});

docs/_templates/base.html

@@ -0,0 +1,42 @@
+{%- extends "!base.html" %}
+
+{% block site_meta %}
+{{ super() }}
+<script defer data-domain="sqlite-utils.datasette.io" src="https://plausible.io/js/plausible.js"></script>
+{% endblock %}
+
+{% block scripts %}
+{{ super() }}
+<style type="text/css">
+.highlight-output .highlight {
+  border-left: 9px solid #30c94f;
+}
+</style>
+<script>
+document.addEventListener("DOMContentLoaded", function() {
+  // Show banner linking to /stable/ if this is a /latest/ page
+  if (!/\/latest\//.test(location.pathname)) {
+    return;
+  }
+  var stableUrl = location.pathname.replace("/latest/", "/stable/");
+  // Check it's not a 404
+  fetch(stableUrl, { method: "HEAD" }).then((response) => {
+    if (response.status === 200) {
+      var warning = document.createElement("div");
+      warning.className = "admonition warning";
+      warning.innerHTML = `
+        <p class="first admonition-title">Note</p>
+        <p class="last">
+          This documentation covers the <strong>development version</strong> of Datasette.
+        </p>
+        <p>
+          See <a href="${stableUrl}">this page</a> for the current stable release.
+        </p>
+      `;
+      var mainArticle = document.querySelector("article[role=main]");
+      mainArticle.insertBefore(warning, mainArticle.firstChild);
+    }
+  });
+});
+</script>
+{% endblock %}

docs/changelog.rst

@@ -1,7 +1,593 @@
+.. _changelog:
+
 ===========
  Changelog
 ===========
 
+.. _v4_1_1:
+
+4.1.1 (2026-07-12)
+------------------
+
+- ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`)
+- The :ref:`CLI <cli>` and :ref:`Python API <python_api>` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`)
+.. _v4_1:
+
+4.1 (2026-07-11)
+----------------
+
+- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code <cli_insert_code>` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`)
+- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created <cli_insert_csv_tsv_column_types>`. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`)
+- New ``table.drop_index(name)`` method and ``sqlite-utils drop-index`` command for dropping an index by name. Both accept ``ignore=True``/``--ignore`` to ignore a missing index. (:issue:`626`)
+- ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`)
+- ``sqlite-utils upsert`` can now infer the primary key of an existing table, so ``--pk`` can be omitted when upserting into a table that already has a primary key.
+- ``table.transform()`` and ``table.transform_sql()`` now accept ``strict=True`` or ``strict=False`` to change a table's `SQLite strict mode <https://www.sqlite.org/stricttables.html>`__. Omitting the option preserves the existing mode. (:issue:`787`)
+- The ``sqlite-utils transform`` command now accepts ``--strict`` and ``--no-strict`` to change a table's strict mode. (:issue:`787`)
+
+.. _v4_0:
+
+4.0 (2026-07-07)
+----------------
+
+The 4.0 release includes some minor backwards-incompatible fixes (hence the major version number bump) and introduces three major new features:
+
+- :ref:`Database migrations <migrations>`, providing a structured mechanism for evolving a project's schema over time. (:issue:`752`)
+- :ref:`Nested transaction support <python_api_atomic>` via ``db.atomic()``, plus numerous improvements to how transactions work across the library. (:issue:`755`)
+- Support for :ref:`compound foreign keys <python_api_compound_foreign_keys>`, including creation, transformation and introspection through :ref:`table.foreign_keys <python_api_introspection_foreign_keys>`. (:issue:`594`)
+
+Other notable changes include:
+
+- Upserts now use SQLite's ``INSERT ... ON CONFLICT ... DO UPDATE SET`` syntax, detect existing table primary keys automatically and reject records that are missing required primary key values. (:issue:`652`)
+- ``db.query()`` now executes immediately and rejects statements that do not return rows; use ``db.execute()`` for writes and DDL.
+- CSV and TSV imports now detect column types by default, while inserts into existing tables preserve those tables' column types. (:issue:`679`)
+- Foreign key handling now preserves ``ON DELETE``/``ON UPDATE`` actions during transforms and resolves referenced primary keys more accurately. (:issue:`530`)
+- Column names passed to Python API methods are now matched case-insensitively, mirroring SQLite's own identifier behavior. (:issue:`760`)
+- The command-line tool now emits UTF-8 JSON output by default, with ``--ascii`` available to restore escaped output. (:issue:`625`)
+- ``table.extract()`` and ``extracts=`` no longer create lookup table records for all-``null`` values. (:issue:`186`)
+
+See :ref:`upgrading_3_to_4` for details on backwards-incompatible changes.
+
+The detailed release notes for the features and fixes shipped during the 4.0 pre-release cycle are available in :ref:`4.0a0 <v4_0a0>`, :ref:`4.0a1 <v4_0a1>`, :ref:`4.0rc1 <v4_0rc1>`, :ref:`4.0rc2 <v4_0rc2>`, :ref:`4.0rc3 <v4_0rc3>` and :ref:`4.0rc4 <v4_0rc4>`.
+
+Bug fixes since 4.0rc4
+~~~~~~~~~~~~~~~~~~~~~~
+
+- Fixed 4.0 regressions in ``insert``/``upsert`` against tables that use SQLite's implicit ``rowid`` primary key. Passing ``pk="rowid"``, ``pk="_rowid_"`` or ``pk="oid"`` now works again for rowid tables, and ``last_pk`` is set correctly. (:issue:`781`)
+- Fixed ``insert(..., ignore=True)`` and ``insert_all(..., ignore=True)`` so an ignored insert that conflicts with an existing primary key row now reports that existing row in ``last_rowid`` and ``last_pk`` where possible. This also works for compound primary keys and list-mode inserts. (:issue:`783`)
+
+.. _v4_0rc4:
+
+4.0rc4 (2026-07-06)
+-------------------
+
+- **Breaking change**: ``table.extract()`` - and the ``sqlite-utils extract`` command - no longer extract rows where every extracted column is ``null``. Those rows now keep a ``null`` value in the new foreign key column instead of pointing at an all-``null`` record in the lookup table. When extracting multiple columns, rows are still extracted if at least one of the columns has a value. (:issue:`186`)
+- The ``extracts=`` option to ``table.insert()`` and friends no longer creates a lookup table record for ``None`` values - the column value stays ``null``. Previously every batch of inserted rows containing a ``None`` value would add a duplicate ``null`` record to the lookup table.
+- Fixed a bug where ``table.lookup()`` inserted a duplicate row on every call if any of the lookup values were ``None``. Lookup values are now compared using ``IS`` so that ``None`` values match existing rows correctly.
+- JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`)
+- ``--no-headers`` now omits the header row from ``--fmt`` and ``--table`` output, not just CSV and TSV output. (:issue:`566`)
+- ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`)
+- Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`)
+- Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before.
+- Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened.
+- Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements.
+- Fixed exception masking when a statement destroys the enclosing transaction. An error such as a ``RAISE(ROLLBACK)`` trigger or ``INSERT OR ROLLBACK`` conflict rolls back the whole transaction, destroying every savepoint - the cleanup in ``db.atomic()`` and ``db.query()`` then failed with ``OperationalError: no such savepoint`` (or ``cannot rollback - no transaction is active``), hiding the original ``IntegrityError`` from code that tried to catch it. Cleanup now checks whether a transaction is still open first, so the original exception propagates.
+- ``sqlite-utils migrate --list`` is now read-only even when the migrations file uses the legacy ``sqlite_migrate.Migrations`` class, whose listing methods create the ``_sqlite_migrations`` table as a side effect. The listing now runs inside a transaction that is rolled back.
+- ``sqlite-utils insert ... --pk <missing column>`` and ``sqlite-utils extract <missing column>`` now show a clean ``Error:`` message instead of a raw Python traceback. The ``extract`` command also shows a clean error when pointed at a view.
+- Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows.
+- ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns.
+- ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op.
+- ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything.
+- Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``.
+- Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table.
+- Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order.
+- The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks.
+- ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was.
+- Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order.
+
+.. _v4_0rc3:
+
+4.0rc3 (2026-07-05)
+-------------------
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+- :ref:`table.foreign_keys <python_api_introspection_foreign_keys>` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`)
+- Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library.
+- The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`)
+
+Compound foreign key support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Tables can now be created with :ref:`compound foreign keys <python_api_compound_foreign_keys>`, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema.
+- ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely.
+- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table.
+- ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key.
+
+Other foreign key improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema.
+- ``table.add_foreign_key()`` accepts new ``on_delete=`` and ``on_update=`` parameters for creating foreign keys with actions, e.g. ``table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")``. (:issue:`530`)
+- Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``.
+- Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound.
+
+Case-insensitive column matching
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Column names passed to Python API methods are now matched against the table schema case-insensitively, mirroring how SQLite itself treats identifiers. Previously many methods accepted mixed-case identifiers in the SQL they generated but then failed - or silently did nothing - when performing Python-side comparisons against the schema. (:issue:`760`) Fixes include:
+
+- ``table.insert()`` and ``table.upsert()`` now populate ``table.last_pk`` correctly when the ``pk=`` argument uses different casing to the table schema or the record keys - previously this raised a ``KeyError`` after the row had already been written.
+- Upserts no longer raise or misbehave when the casing of ``pk=`` differs from the casing of the record keys. The primary key columns are correctly excluded from the generated ``DO UPDATE SET`` clause.
+- ``table.transform()`` arguments ``types=``, ``rename=``, ``drop=``, ``pk=``, ``not_null=``, ``defaults=``, ``column_order=`` and ``drop_foreign_keys=`` all resolve column names case-insensitively. Previously options like ``rename={"name": "title"}`` against a column called ``Name`` were silently ignored.
+- ``db.create_table(..., transform=True)`` now recognizes existing columns that differ only by case, instead of attempting to add them again and failing with ``duplicate column name``. The casing used in the existing schema is preserved.
+- ``table.lookup()`` returns the primary key value even if ``pk=`` casing differs from the schema, and recognizes existing unique indexes case-insensitively instead of creating redundant ones.
+- ``table.extract()`` and ``table.convert()`` - including ``multi=True`` and ``output=`` - accept column names in any casing.
+- Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive.
+- ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``.
+
+Everything else
+~~~~~~~~~~~~~~~
+
+- Fixed a bug where ``table.transform()`` could convert ``DEFAULT TRUE``, ``DEFAULT FALSE`` and ``DEFAULT NULL`` column defaults into quoted string defaults when rebuilding a table. Thanks, `Vincent Gao <https://github.com/gaoflow>`__. (`#764 <https://github.com/simonw/sqlite-utils/pull/764>`__)
+
+.. _v4_0rc2:
+
+4.0rc2 (2026-07-04)
+-------------------
+
+Breaking changes:
+
+- Write statements executed with ``db.execute()`` are now committed automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that stayed open until something committed it - writes appeared to work when read on the same connection but were silently rolled back when the connection closed. Code that relied on rolling back uncommitted ``db.execute()`` writes should use the new ``db.begin()`` method to open an explicit transaction first. The transaction model is documented in full at :ref:`python_api_transactions`.
+- ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` are executed and committed immediately without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. A statement rejected this way is rolled back before the error is raised, so it has no effect on the database.
+- Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead.
+- ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place.
+- ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions.
+- The ``View`` class no longer has an ``enable_fts()`` method. It existed only to raise ``NotImplementedError``, since full-text search is not supported for views - calling it now raises ``AttributeError`` instead, and the method no longer appears in the API reference. The ``sqlite-utils enable-fts`` command shows a clean error when pointed at a view.
+- The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection.
+- ``Database()`` now raises a ``sqlite_utils.db.TransactionError`` if passed a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options. ``commit()`` and ``rollback()`` behave differently on those connections, which previously caused every write made by the library to be silently discarded when the connection closed.
+
+Everything else:
+
+- Fixed a bug where ``table.delete_where()``, ``table.optimize()`` and ``table.rebuild_fts()`` did not commit their changes, leaving the connection inside an open transaction. Their work - and any subsequent writes - could then be silently rolled back when the connection was closed. All three now use ``db.atomic()``, consistent with the other write methods.
+- The ``sqlite-utils drop-table`` command now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. Both now exit with an error suggesting the correct command to use.
+- Migrations applied by the new :ref:`migrations system <migrations>` now run inside a transaction, together with the record of the migration having been applied. If a migration raises an exception its changes are rolled back and it stays pending, so it can be safely re-applied after the error is fixed. Migrations that cannot run inside a transaction, such as those executing ``VACUUM``, can opt out using ``@migrations(transactional=False)`` - see :ref:`migrations_transactions`.
+- ``table.upsert()`` and ``table.upsert_all()`` now detect the primary key or compound primary key of an existing table, so the ``pk=`` argument is no longer required when upserting into a table that already has a primary key.
+- ``db.table(table_name).insert({})`` can now be used to insert a row consisting entirely of default values into an existing table, using ``INSERT INTO ... DEFAULT VALUES``. (:issue:`759`)
+- Improvements to the ``sqlite-utils migrate`` command: ``--stop-before`` values that do not match any known migration are now an error instead of being silently ignored, ``--stop-before`` now works correctly with migration files that still use the older ``sqlite_migrate.Migrations`` class, and ``--list`` is now a read-only operation that no longer creates the database file or the migrations tracking table. ``migrations.applied()`` now returns migrations in the order they were applied.
+- New ``db.begin()``, ``db.commit()`` and ``db.rollback()`` methods for taking manual control of transactions, as an alternative to the ``db.atomic()`` context manager.
+- New documentation: :ref:`python_api_transactions` describes how transactions work and when changes are committed, and a new :ref:`upgrading` page details the changes needed to move between major versions.
+
+.. _v4_0rc1:
+
+4.0rc1 (2026-06-21)
+-------------------
+
+- New :ref:`database migrations system <migrations>`, incorporating functionality that was previously provided by the separate `sqlite-migrate <https://github.com/simonw/sqlite-migrate>`__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using the ``sqlite-utils migrate`` command or the :ref:`migrations Python API <migrations_python>`. (:issue:`752`)
+- New ``db.atomic()`` :ref:`context manager providing nested transaction support <python_api_atomic>` using SQLite transactions and savepoints. Internal multi-step operations such as ``table.transform()`` now use this mechanism to avoid unexpectedly committing an existing transaction. (:issue:`755`)
+- ``Database`` objects can now be :ref:`used as context managers <python_api_close>`, automatically closing the connection when the ``with`` block exits. The CLI also now closes database and file handles more reliably, resolving a number of ``ResourceWarning`` warnings. (:issue:`692`)
+- The ``sqlite-utils convert`` command can now accept a direct callable reference such as ``r.parsedate`` or ``json.loads --import json`` as the conversion code, as an alternative to calling it explicitly with ``r.parsedate(value)``. (:issue:`686`)
+- Fixed a bug where CSV or TSV files with only a header row could crash ``sqlite-utils insert`` and ``sqlite-utils memory`` when type detection was enabled. Thanks, `Rami Abdelrazzaq <https://github.com/RamiNoodle733>`__. (:issue:`702`, `#707 <https://github.com/simonw/sqlite-utils/pull/707>`__)
+- Fixed a bug where installed plugins could be loaded while running the test suite, despite the test-mode safeguard that disables plugin loading. Thanks, `Rami Abdelrazzaq <https://github.com/RamiNoodle733>`__. (:issue:`713`, `#719 <https://github.com/simonw/sqlite-utils/pull/719>`__)
+- ``table.detect_fts()`` now recognizes legacy FTS virtual tables that quote the ``content=`` table name using square brackets, allowing ``table.enable_fts(..., replace=True)`` to replace them correctly. (:issue:`694`)
+- Now depends on Click 8.3.1 or later, removing compatibility workarounds for Click's ``Sentinel`` default values. (:issue:`666`)
+- Improved type annotations throughout the package, with ``ty`` now run in CI. (:issue:`697`)
+- Development tooling now uses ``uv`` dependency groups, with separate ``dev`` and ``docs`` groups. (:issue:`691`)
+- The test suite now runs against Python 3.15-dev. (:issue:`738`)
+
+.. _v3_39:
+
+3.39 (2025-11-24)
+-----------------
+
+- Fixed a bug with ``sqlite-utils install`` when the tool had been installed using ``uv``. (:issue:`687`)
+- The ``--functions`` argument now optionally accepts a path to a Python file as an alternative to a string full of code, and can be specified multiple times - see :ref:`cli_query_functions`. (:issue:`659`)
+- ``sqlite-utils`` now requires Python 3.10 or higher.
+
+.. _v4_0a1:
+
+4.0a1 (2025-11-23)
+------------------
+
+- **Breaking change**: The ``db.table(table_name)`` method now only works with tables. To access a SQL view use ``db.view(view_name)`` instead. (:issue:`657`)
+- The ``table.insert_all()`` and ``table.upsert_all()`` methods can now accept an iterator of lists or tuples as an alternative to dictionaries. The first item should be a list/tuple of column names. See :ref:`python_api_insert_lists` for details. (:issue:`672`)
+- **Breaking change**: The default floating point column type has been changed from ``FLOAT`` to ``REAL``, which is the correct SQLite type for floating point values. This affects auto-detected columns when inserting data. (:issue:`645`)
+- Now uses ``pyproject.toml`` in place of ``setup.py`` for packaging. (:issue:`675`)
+- Tables in the Python API now do a much better job of remembering the primary key and other schema details from when they were first created. (:issue:`655`)
+- **Breaking change**: The ``table.convert()`` and ``sqlite-utils convert`` mechanisms no longer skip values that evaluate to ``False``. Previously the ``--skip-false`` option was needed, this has been removed. (:issue:`542`)
+- **Breaking change**: Tables created by this library now wrap table and column names in ``"double-quotes"`` in the schema. Previously they would use ``[square-braces]``. (:issue:`677`)
+- The ``--functions`` CLI argument now accepts a path to a Python file in addition to accepting a string full of Python code. It can also now be specified multiple times. (:issue:`659`)
+- **Breaking change:** Type detection is now the default behavior for the ``insert`` and ``upsert`` CLI commands when importing CSV or TSV data. Previously all columns were treated as ``TEXT`` unless the ``--detect-types`` flag was passed. Use the new ``--no-detect-types`` flag to restore the old behavior. The ``SQLITE_UTILS_DETECT_TYPES`` environment variable has been removed. (:issue:`679`)
+
+.. _v4_0a0:
+
+4.0a0 (2025-05-08)
+------------------
+
+- Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax on all SQLite versions later than 3.23.1. This is a very slight breaking change for apps that depend on the previous ``INSERT OR IGNORE`` followed by ``UPDATE`` behavior. (:issue:`652`)
+- Python library users can opt-in to the previous implementation by passing ``use_old_upsert=True`` to the ``Database()`` constructor, see :ref:`python_api_old_upsert`.
+- Dropped support for Python 3.8, added support for Python 3.13. (:issue:`646`)
+- ``sqlite-utils tui`` is now provided by the `sqlite-utils-tui <https://github.com/simonw/sqlite-utils-tui>`__ plugin. (:issue:`648`)
+- Test suite now also runs against SQLite 3.23.1, the last version (from 2018-04-10) before the new ``INSERT ... ON CONFLICT SET`` syntax was added. (:issue:`654`)
+
+.. _v3_38:
+
+3.38 (2024-11-23)
+-----------------
+
+- Plugins can now reuse the implementation of the ``sqlite-utils memory`` CLI command with the new ``return_db=True`` parameter. (:issue:`643`)
+- ``table.transform()`` now recreates indexes after transforming a table. A new ``sqlite_utils.db.TransformError`` exception is raised if these indexes cannot be recreated due to conflicting changes to the table such as a column rename. Thanks, `Mat Miller <https://github.com/matdmiller>`__. (:issue:`633`)
+- ``table.search()`` now accepts a ``include_rank=True`` parameter, causing the resulting rows to have a ``rank`` column showing the calculated relevance score. Thanks, `liunux4odoo <https://github.com/liunux4odoo>`__. (`#628 <https://github.com/simonw/sqlite-utils/pull/628>`__)
+- Fixed an error that occurred when creating a strict table with at least one floating point column. These ``FLOAT`` columns are now correctly created as ``REAL`` as well, but only for strict tables. (:issue:`644`)
+
+.. _v3_37:
+
+3.37 (2024-07-18)
+-----------------
+
+- The ``create-table`` and ``insert-files`` commands all now accept multiple ``--pk`` options for compound primary keys. (:issue:`620`)
+- Now tested against Python 3.13 pre-release. (`#619 <https://github.com/simonw/sqlite-utils/pull/619>`__)
+- Fixed a crash that can occur in environments with a broken ``numpy`` installation, producing a ``module 'numpy' has no attribute 'int8'``. (:issue:`632`)
+
+.. _v3_36:
+
+3.36 (2023-12-07)
+-----------------
+
+- Support for creating tables in `SQLite STRICT mode <https://www.sqlite.org/stricttables.html>`__. Thanks, `Taj Khattra <https://github.com/tkhattra>`__. (:issue:`344`)
+    - CLI commands ``create-table``, ``insert`` and ``upsert`` all now accept a ``--strict`` option.
+    - Python methods that can create a table - ``table.create()`` and ``insert/upsert/insert_all/upsert_all`` all now accept an optional ``strict=True`` parameter.
+    - The ``transform`` command and ``table.transform()`` method preserve strict mode when transforming a table.
+- The ``sqlite-utils create-table`` command now accepts ``str``, ``int`` and ``bytes`` as aliases for ``text``, ``integer`` and ``blob`` respectively. (:issue:`606`)
+
+.. _v3_35_2:
+
+3.35.2 (2023-11-03)
+-------------------
+
+- The ``--load-extension=spatialite`` option and :ref:`find_spatialite() <python_api_gis_find_spatialite>` utility function now both work correctly on ``arm64`` Linux. Thanks, `Mike Coats <https://github.com/MikeCoats>`__. (:issue:`599`)
+- Fix for bug where ``sqlite-utils insert`` could cause your terminal cursor to disappear. Thanks, `Luke Plant <https://github.com/spookylukey>`__. (:issue:`433`)
+- ``datetime.timedelta`` values are now stored as ``TEXT`` columns. Thanks, `Harald Nezbeda <https://github.com/nezhar>`__. (:issue:`522`)
+- Test suite is now also run against Python 3.12.
+
+.. _v3_35_1:
+
+3.35.1 (2023-09-08)
+-------------------
+
+- Fixed a bug where :ref:`table.transform() <python_api_transform>` would sometimes re-assign the ``rowid`` values for a table rather than keeping them consistent across the operation. (:issue:`592`)
+
+.. _v3_35:
+
+3.35 (2023-08-17)
+-----------------
+
+Adding foreign keys to a table no longer uses ``PRAGMA writable_schema = 1`` to directly manipulate the ``sqlite_master`` table. This was resulting in errors in some Python installations where the SQLite library was compiled in a way that prevented this from working, in particular on macOS. Foreign keys are now added using the :ref:`table transformation <python_api_transform>` mechanism instead. (:issue:`577`)
+
+This new mechanism creates a full copy of the table, so it is likely to be significantly slower for large tables, but will no longer trigger ``table sqlite_master may not be modified`` errors on platforms that do not support ``PRAGMA writable_schema = 1``.
+
+A new plugin, `sqlite-utils-fast-fks <https://github.com/simonw/sqlite-utils-fast-fks>`__, is now available for developers who still want to use that faster but riskier implementation.
+
+Other changes:
+
+- The :ref:`table.transform() method <python_api_transform>` has two new parameters: ``foreign_keys=`` allows you to replace the foreign key constraints defined on a table, and ``add_foreign_keys=`` lets you specify new foreign keys to add. These complement the existing ``drop_foreign_keys=`` parameter. (:issue:`577`)
+- The :ref:`sqlite-utils transform <cli_transform_table>` command has a new ``--add-foreign-key`` option which can be called multiple times to add foreign keys to a table that is being transformed. (:issue:`585`)
+- :ref:`sqlite-utils convert <cli_convert>` now has a ``--pdb`` option for opening a debugger on the first encountered error in your conversion script. (:issue:`581`)
+- Fixed a bug where ``sqlite-utils install -e '.[test]'`` option did not work correctly.
+
+.. _v3_34:
+
+3.34 (2023-07-22)
+-----------------
+
+This release introduces a new :ref:`plugin system <plugins>`. Read more about this in `sqlite-utils now supports plugins <https://simonwillison.net/2023/Jul/24/sqlite-utils-plugins/>`__. (:issue:`567`)
+
+- Documentation describing :ref:`how to build a plugin <plugins_building>`.
+- Plugin hook: :ref:`plugins_hooks_register_commands`, for plugins to add extra commands to ``sqlite-utils``. (:issue:`569`)
+- Plugin hook: :ref:`plugins_hooks_prepare_connection`. Plugins can use this to help prepare the SQLite connection to do things like registering custom SQL functions. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:issue:`574`)
+- ``sqlite_utils.Database(..., execute_plugins=False)`` option for disabling plugin execution. (:issue:`575`)
+- ``sqlite-utils install -e path-to-directory`` option for installing editable code. This option is useful during the development of a plugin. (:issue:`570`)
+- ``table.create(...)`` method now accepts ``replace=True`` to drop and replace an existing table with the same name, or ``ignore=True`` to silently do nothing if a table already exists with the same name. (:issue:`568`)
+- ``sqlite-utils insert ... --stop-after 10`` option for stopping the insert after a specified number of records. Works for the ``upsert`` command as well. (:issue:`561`)
+- The ``--csv`` and ``--tsv`` modes for ``insert`` now accept a ``--empty-null`` option, which causes empty strings in the CSV file to be stored as ``null`` in the database. (:issue:`563`)
+- New ``db.rename_table(table_name, new_name)`` method for renaming tables. (:issue:`565`)
+- ``sqlite-utils rename-table my.db table_name new_name`` command for renaming tables. (:issue:`565`)
+- The ``table.transform(...)`` method now takes an optional ``keep_table=new_table_name`` parameter, which will cause the original table to be renamed to ``new_table_name`` rather than being dropped at the end of the transformation. (:issue:`571`)
+- Documentation now notes that calling ``table.transform()`` without any arguments will reformat the SQL schema stored by SQLite to be more aesthetically pleasing. (:issue:`564`)
+
+.. _v3_33:
+
+3.33 (2023-06-25)
+-----------------
+
+- ``sqlite-utils`` will now use `sqlean.py <https://github.com/nalgeon/sqlean.py>`__ in place of ``sqlite3`` if it is installed in the same virtual environment. This is useful for Python environments with either an outdated version of SQLite or with restrictions on SQLite such as disabled extension loading or restrictions resulting in the ``sqlite3.OperationalError: table sqlite_master may not be modified`` error. (:issue:`559`)
+- New ``with db.ensure_autocommit_off()`` context manager, which ensures that the database is in autocommit mode for the duration of a block of code. This is used by ``db.enable_wal()`` and ``db.disable_wal()`` to ensure they work correctly with ``pysqlite3`` and ``sqlean.py``.
+- New ``db.iterdump()`` method, providing an iterator over SQL strings representing a dump of the database. This uses ``sqlite-dump`` if it is available, otherwise falling back on the ``conn.iterdump()`` method from ``sqlite3``. Both ``pysqlite3`` and ``sqlean.py`` omit support for ``iterdump()`` - this method helps paper over that difference.
+
+.. _v3_32_1:
+
+3.32.1 (2023-05-21)
+-------------------
+
+- Examples in the :ref:`CLI documentation <cli>` can now all be copied and pasted without needing to remove a leading ``$``. (:issue:`551`)
+- Documentation now covers :ref:`installation_completion` for ``bash`` and ``zsh``. (:issue:`552`)
+
+.. _v3_32:
+
+3.32 (2023-05-21)
+-----------------
+
+- New experimental ``sqlite-utils tui`` interface for interactively building command-line invocations, powered by `Trogon <https://github.com/Textualize/trogon>`__. This requires an optional dependency, installed using ``sqlite-utils install trogon``. (:issue:`545`)
+- ``sqlite-utils analyze-tables`` command (:ref:`documentation <cli_analyze_tables>`) now has a ``--common-limit 20`` option for changing the number of common/least-common values shown for each column. (:issue:`544`)
+- ``sqlite-utils analyze-tables --no-most`` and ``--no-least`` options for disabling calculation of most-common and least-common values.
+- If a column contains only ``null`` values, ``analyze-tables`` will no longer attempt to calculate the most common and least common values for that column. (:issue:`547`)
+- Calling ``sqlite-utils analyze-tables`` with non-existent columns in the ``-c/--column`` option now results in an error message. (:issue:`548`)
+- The ``table.analyze_column()`` method (:ref:`documented here <python_api_analyze_column>`) now accepts ``most_common=False`` and ``least_common=False`` options for disabling calculation of those values.
+
+.. _v3_31:
+
+3.31 (2023-05-08)
+-----------------
+
+- Dropped support for Python 3.6. Tests now ensure compatibility with Python 3.11. (:issue:`517`)
+- Automatically locates the SpatiaLite extension on Apple Silicon. Thanks, Chris Amico. (`#536 <https://github.com/simonw/sqlite-utils/pull/536>`__)
+- New ``--raw-lines`` option for the ``sqlite-utils query`` and ``sqlite-utils memory`` commands, which outputs just the raw value of the first column of every row. (:issue:`539`)
+- Fixed a bug where ``table.upsert_all()`` failed if the ``not_null=`` option was passed. (:issue:`538`)
+- Fixed a ``ResourceWarning`` when using ``sqlite-utils insert``. (:issue:`534`)
+- Now shows a more detailed error message when ``sqlite-utils insert`` is called with invalid JSON. (:issue:`532`)
+- ``table.convert(..., skip_false=False)`` and ``sqlite-utils convert --no-skip-false`` options, for avoiding a misfeature where the :ref:`convert()  <python_api_convert>` mechanism skips rows in the database with a falsey value for the specified column. Fixing this by default would be a backwards-incompatible change and is under consideration for a 4.0 release in the future. (:issue:`527`)
+- Tables can now be created with self-referential foreign keys. Thanks, Scott Perry. (`#537 <https://github.com/simonw/sqlite-utils/pull/537>`__)
+- ``sqlite-utils transform`` no longer breaks if a table defines default values for columns. Thanks, Kenny Song. (:issue:`509`)
+- Fixed a bug where repeated calls to ``table.transform()`` did not work correctly. Thanks, Martin Carpenter. (:issue:`525`)
+- Improved error message if ``rows_from_file()`` is passed a non-binary-mode file-like object. (:issue:`520`)
+
+.. _v3_30:
+
+3.30 (2022-10-25)
+-----------------
+
+- Now tested against Python 3.11. (:issue:`502`)
+- New ``table.search_sql(include_rank=True)`` option, which adds a ``rank`` column to the generated SQL. Thanks, Jacob Chapman. (`#480 <https://github.com/simonw/sqlite-utils/pull/480>`__)
+- Progress bars now display for newline-delimited JSON files using the ``--nl`` option. Thanks, Mischa Untaga. (:issue:`485`)
+- New ``db.close()`` method. (:issue:`504`)
+- Conversion functions passed to :ref:`table.convert(...) <python_api_convert>` can now return lists or dictionaries, which will be inserted into the database as JSON strings. (:issue:`495`)
+- ``sqlite-utils install`` and ``sqlite-utils uninstall`` commands for installing packages into the same virtual environment as ``sqlite-utils``, :ref:`described here <cli_install>`. (:issue:`483`)
+- New :ref:`sqlite_utils.utils.flatten() <reference_utils_flatten>` utility function. (:issue:`500`)
+- Documentation on :ref:`using Just <contributing_just>` to run tests, linters and build documentation.
+- Documentation now covers the :ref:`release_process` for this package.
+
+.. _v3_29:
+
+3.29 (2022-08-27)
+-----------------
+
+- The ``sqlite-utils query``, ``memory`` and ``bulk`` commands now all accept a new ``--functions`` option. This can be passed a string of Python code, and any callable objects defined in that code will be made available to SQL queries as custom SQL functions. See :ref:`cli_query_functions` for details. (:issue:`471`)
+- ``db[table].create(...)`` method now accepts a new ``transform=True`` parameter. If the table already exists it will be :ref:`transformed <python_api_transform>` to match the schema configuration options passed to the function. This may result in columns being added or dropped, column types being changed, column order being updated or not null and default values for columns being set. (:issue:`467`)
+- Related to the above, the ``sqlite-utils create-table`` command now accepts a ``--transform`` option.
+- New introspection property: ``table.default_values`` returns a dictionary mapping each column name with a default value to the configured default value. (:issue:`475`)
+- The ``--load-extension`` option can now be provided a path to a compiled SQLite extension module accompanied by the name of an entrypoint, separated by a colon - for example ``--load-extension ./lines0:sqlite3_lines0_noread_init``. This feature is modelled on code first `contributed to Datasette <https://github.com/simonw/datasette/pull/1789>`__ by Alex Garcia. (:issue:`470`)
+- Functions registered using the :ref:`db.register_function() <python_api_register_function>` method can now have a custom name specified using the new ``db.register_function(fn, name=...)`` parameter. (:issue:`458`)
+- :ref:`sqlite-utils rows <cli_rows>` has a new ``--order`` option for specifying the sort order for the returned rows. (:issue:`469`)
+- All of the CLI options that accept Python code blocks can now all be used to define functions that can access modules imported in that same block of code without needing to use the ``global`` keyword. (:issue:`472`)
+- Fixed bug where ``table.extract()`` would not behave correctly for columns containing null values. Thanks, Forest Gregg. (:issue:`423`)
+- New tutorial: `Cleaning data with sqlite-utils and Datasette <https://datasette.io/tutorials/clean-data>`__ shows how to use ``sqlite-utils`` to import and clean an example CSV file.
+- Datasette and ``sqlite-utils`` now have a Discord community. `Join the Discord here <https://discord.gg/Ass7bCAMDw>`__.
+
+.. _v3_28:
+
+3.28 (2022-07-15)
+-----------------
+
+- New :ref:`table.duplicate(new_name) <python_api_duplicate>` method for creating a copy of a table with a matching schema and row contents. Thanks, `David <https://github.com/davidleejy>`__. (:issue:`449`)
+- New ``sqlite-utils duplicate data.db table_name new_name`` CLI command for :ref:`cli_duplicate_table`. (:issue:`454`)
+- ``sqlite_utils.utils.rows_from_file()`` is now a :ref:`documented API <reference_utils_rows_from_file>`. It can be used to read a sequence of dictionaries from a file-like object containing CSV, TSV, JSON or newline-delimited JSON. It can be passed an explicit format or can attempt to detect the format automatically. (:issue:`443`)
+- ``sqlite_utils.utils.TypeTracker`` is now a documented API for detecting the likely column types for a sequence of string rows, see :ref:`python_api_typetracker`. (:issue:`445`)
+- ``sqlite_utils.utils.chunks()`` is now a documented API for :ref:`splitting an iterator into chunks  <reference_utils_chunks>`. (:issue:`451`)
+- ``sqlite-utils enable-fts`` now has a ``--replace`` option for replacing the existing FTS configuration for a table. (:issue:`450`)
+- The ``create-index``, ``add-column`` and ``duplicate`` commands all now take a ``--ignore`` option for ignoring errors should the database not be in the right state for them to operate. (:issue:`450`)
+
+.. _v3_27:
+
+3.27 (2022-06-14)
+-----------------
+
+See also `the annotated release notes <https://simonwillison.net/2022/Jun/19/weeknotes/#sqlite-utils-3-27>`__ for this release.
+
+- Documentation now uses the `Furo <https://github.com/pradyunsg/furo>`__ Sphinx theme. (:issue:`435`)
+- Code examples in documentation now have a "copy to clipboard" button. (:issue:`436`)
+- ``sqlite_utils.utils.utils.rows_from_file()`` is now a documented API, see :ref:`python_api_rows_from_file`. (:issue:`443`)
+- ``rows_from_file()`` has two new parameters to help handle CSV files with rows that contain more values than are listed in that CSV file's headings: ``ignore_extras=True`` and ``extras_key="name-of-key"``. (:issue:`440`)
+- ``sqlite_utils.utils.maximize_csv_field_size_limit()`` helper function for increasing the field size limit for reading CSV files to its maximum, see :ref:`python_api_maximize_csv_field_size_limit`. (:issue:`442`)
+- ``table.search(where=, where_args=)`` parameters for adding additional ``WHERE`` clauses to a search query. The ``where=`` parameter is available on ``table.search_sql(...)`` as well. See :ref:`python_api_fts_search`. (:issue:`441`)
+- Fixed bug where ``table.detect_fts()`` and other search-related functions could fail if two FTS-enabled tables had names that were prefixes of each other. (:issue:`434`)
+
+.. _v3_26_1:
+
+3.26.1 (2022-05-02)
+-------------------
+
+- Now depends on `click-default-group-wheel <https://github.com/simonw/click-default-group-wheel>`__, a pure Python wheel package. This means you can install and use this package with `Pyodide <https://pyodide.org/>`__, which can run Python entirely in your browser using WebAssembly. (`#429 <https://github.com/simonw/sqlite-utils/pull/429>`__)
+
+  Try that out using the `Pyodide REPL <https://pyodide.org/en/stable/console.html>`__:
+
+  .. code-block:: python
+
+      >>> import micropip
+      >>> await micropip.install("sqlite-utils")
+      >>> import sqlite_utils
+      >>> db = sqlite_utils.Database(memory=True)
+      >>> list(db.query("select 3 * 5"))
+      [{'3 * 5': 15}]
+
+.. _v3_26:
+
+3.26 (2022-04-13)
+-----------------
+
+- New ``errors=r.IGNORE/r.SET_NULL`` parameter for the ``r.parsedatetime()`` and ``r.parsedate()`` :ref:`convert recipes <cli_convert_recipes>`. (:issue:`416`)
+- Fixed a bug where ``--multi`` could not be used in combination with ``--dry-run`` for the :ref:`convert <cli_convert>` command. (:issue:`415`)
+- New documentation: :ref:`cli_convert_complex`. (:issue:`420`)
+- More robust detection for whether or not ``deterministic=True`` is supported. (:issue:`425`)
+
+.. _v3_25_1:
+
+3.25.1 (2022-03-11)
+-------------------
+
+- Improved display of type information and parameters in the :ref:`API reference documentation <reference>`. (:issue:`413`)
+
+.. _v3_25:
+
+3.25 (2022-03-01)
+-----------------
+
+- New ``hash_id_columns=`` parameter for creating a primary key that's a hash of the content of specific columns - see :ref:`python_api_hash` for details. (:issue:`343`)
+- New :ref:`db.sqlite_version <python_api_sqlite_version>` property, returning a tuple of integers representing the version of SQLite, for example ``(3, 38, 0)``.
+- Fixed a bug where :ref:`register_function(deterministic=True) <python_api_register_function>` caused errors on versions of SQLite prior to 3.8.3. (:issue:`408`)
+- New documented :ref:`hash_record(record, keys=...) <reference_utils_hash_record>` function.
+
+.. _v3_24:
+
+3.24 (2022-02-15)
+-----------------
+
+- SpatiaLite helpers for the ``sqlite-utils`` command-line tool - thanks, Chris Amico. (:issue:`398`)
+
+  - :ref:`sqlite-utils create-database <cli_create_database>` ``--init-spatialite`` option for initializing SpatiaLite on a newly created database.
+  - :ref:`sqlite-utils add-geometry-column <cli_spatialite>` command for adding geometry columns.
+  - :ref:`sqlite-utils create-spatial-index <cli_spatialite_indexes>` command for adding spatial indexes.
+
+- ``db[table].create(..., if_not_exists=True)`` option for :ref:`creating a table <python_api_explicit_create>` only if it does not already exist. (:issue:`397`)
+- ``Database(memory_name="my_shared_database")`` parameter for creating a :ref:`named in-memory database <python_api_connect>` that can be shared between multiple connections. (:issue:`405`)
+- Documentation now describes :ref:`how to add a primary key to a rowid table <cli_transform_table_add_primary_key_to_rowid>` using ``sqlite-utils transform``. (:issue:`403`)
+
+.. _v3_23:
+
+3.23 (2022-02-03)
+-----------------
+
+This release introduces four new utility methods for working with `SpatiaLite <https://www.gaia-gis.it/fossil/libspatialite/index>`__. Thanks, Chris Amico. (`#385 <https://github.com/simonw/sqlite-utils/pull/385>`__)
+
+- ``sqlite_utils.utils.find_spatialite()`` :ref:`finds the location of the SpatiaLite module <python_api_gis_find_spatialite>` on disk.
+- ``db.init_spatialite()`` :ref:`initializes SpatiaLite <python_api_gis_init_spatialite>` for the given database.
+- ``table.add_geometry_column(...)`` :ref:`adds a geometry column <python_api_gis_add_geometry_column>` to an existing table.
+- ``table.create_spatial_index(...)`` :ref:`creates a spatial index <python_api_gis_create_spatial_index>` for a column.
+- ``sqlite-utils batch`` now accepts a ``--batch-size`` option. (:issue:`392`)
+
+.. _v3_22_1:
+
+3.22.1 (2022-01-25)
+-------------------
+
+- All commands now include example usage in their ``--help`` - see :ref:`cli_reference`. (:issue:`384`)
+- Python library documentation has a new :ref:`python_api_getting_started` section. (:issue:`387`)
+- Documentation now uses `Plausible analytics <https://plausible.io/>`__. (:issue:`389`)
+
+.. _v3_22:
+
+3.22 (2022-01-11)
+-----------------
+
+- New :ref:`cli_reference` documentation page, listing the output of ``--help`` for every one of the CLI commands. (:issue:`383`)
+- ``sqlite-utils rows`` now has ``--limit`` and ``--offset`` options for paginating through data. (:issue:`381`)
+- ``sqlite-utils rows`` now has ``--where`` and ``-p`` options for filtering the table using a ``WHERE`` query, see :ref:`cli_rows`. (:issue:`382`)
+
+.. _v3_21:
+
+3.21 (2022-01-10)
+-----------------
+
+CLI and Python library improvements to help run `ANALYZE <https://www.sqlite.org/lang_analyze.html>`__ after creating indexes or inserting rows, to gain better performance from the SQLite query planner when it runs against indexes.
+
+Three new CLI commands: ``create-database``, ``analyze`` and ``bulk``.
+
+More details and examples can be found in `the annotated release notes <https://simonwillison.net/2022/Jan/11/sqlite-utils/>`__.
+
+- New ``sqlite-utils create-database`` command for creating new empty database files. (:issue:`348`)
+- New Python methods for running ``ANALYZE`` against a database, table or index: ``db.analyze()`` and ``table.analyze()``, see :ref:`python_api_analyze`. (:issue:`366`)
+- New :ref:`sqlite-utils analyze command <cli_analyze>` for running ``ANALYZE`` using the CLI. (:issue:`379`)
+- The ``create-index``, ``insert`` and ``upsert`` commands now have a new ``--analyze`` option for running ``ANALYZE`` after the command has completed. (:issue:`379`)
+- New :ref:`sqlite-utils bulk command <cli_bulk>` which can import records in the same way as ``sqlite-utils insert`` (from JSON, CSV or TSV) and use them to bulk execute a parametrized SQL query. (:issue:`375`)
+- The CLI tool can now also be run using ``python -m sqlite_utils``. (:issue:`368`)
+- Using ``--fmt`` now implies ``--table``, so you don't need to pass both options. (:issue:`374`)
+- The ``--convert`` function applied to rows can now modify the row in place. (:issue:`371`)
+- The :ref:`insert-files command <cli_insert_files>` supports two new columns: ``stem`` and ``suffix``. (:issue:`372`)
+- The ``--nl`` import option now ignores blank lines in the input. (:issue:`376`)
+- Fixed bug where streaming input to the ``insert`` command with ``--batch-size 1`` would appear to only commit after several rows had been ingested, due to unnecessary input buffering. (:issue:`364`)
+
+.. _v3_20:
+
+3.20 (2022-01-05)
+-----------------
+
+- ``sqlite-utils insert ... --lines`` to insert the lines from a file into a table with a single ``line`` column, see :ref:`cli_insert_unstructured`.
+- ``sqlite-utils insert ... --text`` to insert the contents of the file into a table with a single ``text`` column and a single row.
+- ``sqlite-utils insert ... --convert`` allows a Python function to be provided that will be used to convert each row that is being inserted into the database. See :ref:`cli_insert_convert`, including details on special behavior when combined with ``--lines`` and ``--text``. (:issue:`356`)
+- ``sqlite-utils convert`` now accepts a code value of ``-`` to read code from standard input. (:issue:`353`)
+- ``sqlite-utils convert`` also now accepts code that defines a named ``convert(value)`` function, see :ref:`cli_convert`.
+- ``db.supports_strict`` property showing if the database connection supports `SQLite strict tables <https://www.sqlite.org/stricttables.html>`__.
+- ``table.strict`` property (see :ref:`python_api_introspection_strict`) indicating if the table uses strict mode. (:issue:`344`)
+- Fixed bug where ``sqlite-utils upsert ... --detect-types`` ignored the ``--detect-types`` option. (:issue:`362`)
+
+.. _v3_19:
+
+3.19 (2021-11-20)
+-----------------
+
+- The :ref:`table.lookup() method <python_api_lookup_tables>` now accepts keyword arguments that match those on the underlying ``table.insert()`` method: ``foreign_keys=``, ``column_order=``, ``not_null=``, ``defaults=``, ``extracts=``, ``conversions=`` and ``columns=``. You can also now pass ``pk=`` to specify a different column name to use for the primary key. (:issue:`342`)
+
+.. _v3_18:
+
+3.18 (2021-11-14)
+-----------------
+
+- The ``table.lookup()`` method now has an optional second argument which can be used to populate columns only the first time the record is created, see :ref:`python_api_lookup_tables`. (:issue:`339`)
+- ``sqlite-utils memory`` now has a ``--flatten`` option for :ref:`flattening nested JSON objects <cli_inserting_data_flatten>` into separate columns, consistent with ``sqlite-utils insert``. (:issue:`332`)
+- ``table.create_index(..., find_unique_name=True)`` parameter, which finds an available name for the created index even if the default name has already been taken. This means that ``index-foreign-keys`` will work even if one of the indexes it tries to create clashes with an existing index name. (:issue:`335`)
+- Added ``py.typed`` to the module, so `mypy <http://mypy-lang.org/>`__ should now correctly pick up the type annotations. Thanks, Andreas Longo. (:issue:`331`)
+- Now depends on ``python-dateutil`` instead of depending on ``dateutils``. Thanks, Denys Pavlov. (:issue:`324`)
+- ``table.create()`` (see :ref:`python_api_explicit_create`) now handles ``dict``, ``list`` and ``tuple`` types, mapping them to ``TEXT`` columns in SQLite so that they can be stored encoded as JSON. (:issue:`338`)
+- Inserted data with square braces in the column names (for example a CSV file containing a ``item[price]``) column now have the braces converted to underscores: ``item_price_``. Previously such columns would be rejected with an error. (:issue:`329`)
+- Now also tested against Python 3.10. (`#330 <https://github.com/simonw/sqlite-utils/pull/330>`__)
+
+.. _v3_17.1:
+
+3.17.1 (2021-09-22)
+-------------------
+
+- :ref:`sqlite-utils memory <cli_memory>` now works if files passed to it share the same file name. (:issue:`325`)
+- :ref:`sqlite-utils query <cli_query>` now returns ``[]`` in JSON mode if no rows are returned. (:issue:`328`)
+
+.. _v3_17:
+
+3.17 (2021-08-24)
+-----------------
+
+- The :ref:`sqlite-utils memory <cli_memory>` command has a new ``--analyze`` option, which runs the equivalent of the :ref:`analyze-tables <cli_analyze_tables>` command directly against the in-memory database created from the incoming CSV or JSON data. (:issue:`320`)
+- :ref:`sqlite-utils insert-files <cli_insert_files>` now has the ability to insert file contents in to ``TEXT`` columns in addition to the default ``BLOB``. Pass the ``--text`` option or use ``content_text`` as a column specifier. (:issue:`319`)
+
+.. _v3_16:
+
+3.16 (2021-08-18)
+-----------------
+
+- Type signatures added to  more methods, including ``table.resolve_foreign_keys()``, ``db.create_table_sql()``, ``db.create_table()`` and ``table.create()``. (:issue:`314`)
+- New ``db.quote_fts(value)`` method, see :ref:`python_api_quote_fts` - thanks, Mark Neumann. (:issue:`246`)
+- ``table.search()`` now accepts an optional ``quote=True`` parameter. (:issue:`296`)
+- CLI command ``sqlite-utils search`` now accepts a ``--quote`` option. (:issue:`296`)
+- Fixed bug where ``--no-headers`` and ``--tsv`` options to :ref:`sqlite-utils insert <cli_insert_csv_tsv>` could not be used together. (:issue:`295`)
+- Various small improvements to :ref:`reference` documentation.
+
+.. _v3_15.1:
+
+3.15.1 (2021-08-10)
+-------------------
+
+- Python library now includes type annotations on almost all of the methods, plus detailed docstrings describing each one. (:issue:`311`)
+- New :ref:`reference` documentation page, powered by those docstrings.
+- Fixed bug where ``.add_foreign_keys()`` failed to raise an error if called against a ``View``. (:issue:`313`)
+- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existent table. (:issue:`315`)
+
 .. _v3_15:
 
 3.15 (2021-08-09)
@@ -953,3 +1539,19 @@ A few other changes:
 ----------------
 
 - ``enable_fts()``, ``populate_fts()`` and ``search()`` table methods
+
+0.3.1 (2018-07-31)
+------------------
+
+- Documented related projects
+- Added badges to the documentation
+
+0.3 (2018-07-31)
+----------------
+
+- New ``Table`` class representing a table in the SQLite database
+
+0.2 (2018-07-28)
+----------------
+
+- Initial release to PyPI

docs/conf.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from subprocess import Popen, PIPE
+import inspect
+from pathlib import Path
+from subprocess import Popen, PIPE, check_output
+import sys
 
 # This file is execfile()d with the current directory set to its
 # containing dir.
@@ -30,12 +33,69 @@ from subprocess import Popen, PIPE
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ["sphinx.ext.extlinks"]
+extensions = [
+    "sphinx.ext.extlinks",
+    "sphinx.ext.autodoc",
+    "sphinx_copybutton",
+    "sphinx.ext.linkcode",
+]
+autodoc_member_order = "bysource"
+autodoc_typehints = "description"
 
 extlinks = {
-    "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"),
+    "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#%s"),
 }
 
+
+def _linkcode_git_ref():
+    try:
+        return check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip()
+    except Exception:
+        return "main"
+
+
+def linkcode_resolve(domain, info):
+    if domain != "py":
+        return None
+
+    module_name = info.get("module")
+    if not module_name or module_name.split(".")[0] != "sqlite_utils":
+        return None
+
+    module = sys.modules.get(module_name)
+    if module is None:
+        return None
+
+    obj = module
+    for part in info.get("fullname", "").split("."):
+        obj = getattr(obj, part, None)
+        if obj is None:
+            return None
+
+    if isinstance(obj, property):
+        obj = obj.fget
+
+    try:
+        obj = inspect.unwrap(obj)
+        source_file = inspect.getsourcefile(obj)
+        _, line_number = inspect.getsourcelines(obj)
+    except Exception:
+        return None
+
+    if source_file is None:
+        return None
+
+    try:
+        filename = Path(source_file).resolve().relative_to(Path(__file__).parent.parent)
+    except ValueError:
+        return None
+
+    return (
+        "https://github.com/simonw/sqlite-utils/blob/"
+        f"{_linkcode_git_ref()}/{filename}#L{line_number}"
+    )
+
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
@@ -50,7 +110,7 @@ master_doc = "index"
 
 # General information about the project.
 project = "sqlite-utils"
-copyright = "2018-2021, Simon Willison"
+copyright = "2018-2022, Simon Willison"
 author = "Simon Willison"
 
 # The version info for the project you're documenting, acts as replacement for
@@ -59,7 +119,7 @@ author = "Simon Willison"
 #
 # The short X.Y version.
 pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True)
-git_version = pipe.stdout.read().decode("utf8")
+git_version = pipe.stdout.read().decode("utf8") if pipe.stdout else ""
 
 if git_version:
     version = git_version.rsplit("-", 1)[0]
@@ -73,7 +133,7 @@ else:
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -83,6 +143,9 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "sphinx"
 
+# Only syntax highlight of code-block is used:
+highlight_language = "none"
+
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
@@ -92,7 +155,8 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = "sphinx_rtd_theme"
+html_theme = "furo"
+html_title = "sqlite-utils"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -105,18 +169,7 @@ html_theme = "sphinx_rtd_theme"
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# This is required for the alabaster theme
-# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
-html_sidebars = {
-    "**": [
-        "relations.html",  # needs 'show_related': True theme option to display
-        "searchbox.html",
-    ]
-}
-
+html_js_files = ["js/custom.js"]
 
 # -- Options for HTMLHelp output ------------------------------------------
 
@@ -174,7 +227,7 @@ texinfo_documents = [
         "sqlite-utils documentation",
         author,
         "sqlite-utils",
-        "Python utility functions for manipulating SQLite databases",
+        "Python library for manipulating SQLite databases",
         "Miscellaneous",
     )
 ]

docs/contributing.rst

@@ -4,64 +4,134 @@
  Contributing
 ==============
 
-To work on this library locally, first checkout the code. Then create a new virtual environment::
+Development of ``sqlite-utils`` takes place in the `sqlite-utils GitHub repository <https://github.com/simonw/sqlite-utils>`__.
+
+All improvements to the software should start with an issue. Read `How I build a feature <https://simonwillison.net/2022/Jan/12/how-i-build-a-feature/>`__ for a detailed description of the recommended process for building bug fixes or enhancements.
+
+.. _contributing_checkout:
+
+Obtaining the code
+==================
+
+To work on this library locally, first checkout the code::
 
     git clone git@github.com:simonw/sqlite-utils
     cd sqlite-utils
-    python3 -mvenv venv
-    source venv/bin/activate
 
-Or if you are using ``pipenv``::
+Use ``uv run`` to run the development version of the tool::
 
-    pipenv shell
-
-Within the virtual environment running ``sqlite-utils`` should run your locally editable version of the tool. You can use ``which sqlite-utils`` to confirm that you are running the version that lives in your virtual environment.
+    uv run sqlite-utils --help
 
 .. _contributing_tests:
 
 Running the tests
 =================
 
-To install the dependencies and test dependencies::
+Use ``uv run`` to run the tests::
 
-    pip install -e '.[test]'
-
-To run the tests::
-
-    pytest
+    uv run pytest
 
 .. _contributing_docs:
 
 Building the documentation
 ==========================
 
-To build the documentation, first install the documentation dependencies::
+To build the documentation run this command::
 
-    pip install -e '.[docs]'
+    uv run make livehtml --directory docs
 
-Then run ``make livehtml`` from the ``docs/`` directory to start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file::
+This will start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file.
 
-    cd docs
-    make livehtml
+The `cog <https://github.com/nedbat/cog>`__ tool is used to maintain portions of the documentation. You can run it like so::
+
+    uv run cog -r docs/*.rst
 
 .. _contributing_linting:
 
 Linting and formatting
 ======================
 
-``sqlite-utils`` uses `Black <https://black.readthedocs.io/>`__ for code formatting, and `flake8 <https://flake8.pycqa.org/>`__ and `mypy <https://mypy.readthedocs.io/>`__ for linting and type checking.
+``sqlite-utils`` uses `Black <https://black.readthedocs.io/>`__ for code formatting, and `flake8 <https://flake8.pycqa.org/>`__ and `mypy <https://mypy.readthedocs.io/>`__ for linting and type checking::
 
-Black is installed as part of ``pip install -e '.[test]'`` - you can then format your code by running it in the root of the project::
+    uv run black .
 
-    black .
+Linting tools can be run like this::
 
-To install ``mypy`` and ``flake8`` run the following::
-
-    pip install -e '.[flake8,mypy]'
-
-Both commands can then be run in the root of the project like this::
-
-    flake8
-    mypy sqlite_utils
+    uv run flake8
+    uv run mypy sqlite_utils
 
 All three of these tools are run by our CI mechanism against every commit and pull request.
+
+.. _contributing_just:
+
+Using Just
+==========
+
+If you install `Just <https://github.com/casey/just>`__ you can use it to manage your local development environment.
+
+To run all of the tests and linters::
+
+    just
+
+To run tests, or run a specific test module or test by name::
+
+    just test # All tests
+    just test tests/test_cli_memory.py # Just this module
+    just test -k test_memory_no_detect_types # Just this test
+
+To run just the linters::
+
+    just lint
+
+To apply Black to your code::
+
+    just black
+
+To update documentation using Cog::
+
+    just cog
+
+To run the live documentation server (this will run Cog first)::
+
+    just docs
+
+And to list all available commands::
+
+    just -l
+
+.. _release_process:
+
+Release process
+===============
+
+Releases are performed using tags. When a new release is published on GitHub, a `GitHub Actions workflow <https://github.com/simonw/sqlite-utils/blob/main/.github/workflows/publish.yml>`__ will perform the following:
+
+* Run the unit tests against all supported Python versions. If the tests pass...
+* Build a wheel bundle of the underlying Python source code
+* Push that new wheel up to PyPI: https://pypi.org/project/sqlite-utils/
+
+To deploy new releases you will need to have push access to the GitHub repository.
+
+``sqlite-utils`` follows `Semantic Versioning <https://semver.org/>`__::
+
+    major.minor.patch
+
+We increment ``major`` for backwards-incompatible releases.
+
+We increment ``minor`` for new features.
+
+We increment ``patch`` for bugfix releass.
+
+To release a new version, first create a commit that updates the version number in ``pyproject.toml`` and the :ref:`the changelog <changelog>` with highlights of the new version. An example `commit can be seen here <https://github.com/simonw/sqlite-utils/commit/b491f22d817836829965516983a3f4c3c72c05fc>`__::
+
+    # Update changelog
+    git commit -m " Release 3.29
+
+    Refs #423, #458, #467, #469, #470, #471, #472, #475" -a
+    git push
+
+Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here <https://github.com/simonw/sqlite-utils/issues/458#ref-commit-b491f22>`__.
+
+You can generate the list of issue references for a specific release by copying and pasting text from the release notes or GitHub changes-since-last-release view into this `Extract issue numbers from pasted text <https://observablehq.com/@simonw/extract-issue-numbers-from-pasted-text>`__ tool.
+
+To create the tag for the release, create `a new release <https://github.com/simonw/sqlite-utils/releases/new>`__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool <https://euangoddard.github.io/clipboard2markdown/>`__.

docs/index.rst

@@ -2,7 +2,7 @@
  sqlite-utils |version|
 =======================
 
-|PyPI| |Changelog| |CI| |License|
+|PyPI| |Changelog| |CI| |License| |discord|
 
 .. |PyPI| image:: https://img.shields.io/pypi/v/sqlite-utils.svg
    :target: https://pypi.org/project/sqlite-utils/
@@ -12,8 +12,10 @@
    :target: https://github.com/simonw/sqlite-utils/actions
 .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
    :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE
+.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=discord
+   :target: https://discord.gg/Ass7bCAMDw
 
-*Python utility functions for manipulating SQLite databases*
+*CLI tool and Python library for manipulating SQLite databases*
 
 This library and command-line utility helps create SQLite databases from an existing collection of data.
 
@@ -23,6 +25,8 @@ sqlite-utils is not intended to be a full ORM: the focus is utility helpers to m
 
 It is designed as a useful complement to `Datasette <https://datasette.io/>`_.
 
+`Cleaning data with sqlite-utils and Datasette <https://datasette.io/tutorials/clean-data>`_ provides a tutorial introduction (and accompanying ten minute video) about using this tool.
+
 Contents
 --------
 
@@ -32,7 +36,10 @@ Contents
    installation
    cli
    python-api
+   migrations
+   plugins
+   reference
+   cli-reference
+   upgrading
    contributing
    changelog
-
-Take a look at `this script <https://github.com/simonw/russian-ira-facebook-ads-datasette/blob/master/fetch_and_build_russian_ads.py>`_ for an example of this library in action.

docs/installation.rst

@@ -38,3 +38,53 @@ Using pipx
 `pipx <https://pypi.org/project/pipx/>`__ is a tool for installing Python command-line applications in their own isolated environments. You can use ``pipx`` to install the ``sqlite-utils`` command-line tool like this::
 
     pipx install sqlite-utils
+
+.. _installation_sqlite3_alternatives:
+
+Alternatives to sqlite3
+=======================
+
+By default, ``sqlite-utils`` uses the ``sqlite3`` package bundled with the Python standard library.
+
+Depending on your operating system, this may come with some limitations.
+
+On some platforms the ability to load additional extensions (via ``conn.load_extension(...)`` or ``--load-extension=/path/to/extension``) may be disabled.
+
+You may also see the error ``sqlite3.OperationalError: table sqlite_master may not be modified`` when trying to alter an existing table.
+
+You can work around these limitations by installing the `pysqlite3 <https://pypi.org/project/pysqlite3/>`__ package, which provides a drop-in replacement for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions.
+
+To install ``pysqlite3`` run the following:
+
+.. code-block:: bash
+
+    sqlite-utils install pysqlite3
+
+``pysqlite3`` does not provide an implementation of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package:
+
+.. code-block:: bash
+
+    sqlite-utils install sqlite-dump
+
+.. _installation_completion:
+
+Setting up shell completion
+===========================
+
+You can configure shell tab completion for the ``sqlite-utils`` command using these commands.
+
+For ``bash``:
+
+.. code-block:: bash
+
+    eval "$(_SQLITE_UTILS_COMPLETE=bash_source sqlite-utils)"
+
+For ``zsh``:
+
+.. code-block:: zsh
+
+    eval "$(_SQLITE_UTILS_COMPLETE=zsh_source sqlite-utils)"
+
+Add this code to ``~/.zshrc`` or ``~/.bashrc`` to automatically run it when you start a new shell.
+
+See `the Click documentation <https://click.palletsprojects.com/en/8.1.x/shell-completion/>`__ for more details.

docs/migrations.rst

@@ -0,0 +1,194 @@
+.. _migrations:
+
+=====================
+ Database migrations
+=====================
+
+``sqlite-utils`` includes a migration system for applying repeatable changes to SQLite database files.
+
+A migration is a Python function that receives a :class:`sqlite_utils.Database` instance and then executes Python code to modify that database - creating or transforming tables, adding indexes, inserting rows, or any other operation supported by SQLite.
+
+Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database.
+
+This means you can run the migrate operation multiple times and it will only apply migrations that have not previously been recorded.
+
+.. _migrations_define:
+
+Defining migrations
+===================
+
+Ordered migration sets are defined by first creating a :class:`sqlite_utils.Migrations` object.
+
+Individual migrations are Python functions that are then registered with that migration set. Each migration function is passed a single argument that is a :ref:`sqlite_utils.Database <reference_db_database>` instance.
+
+The name passed to ``Migrations("creatures")`` identifies that set of migrations. Use a name that is unique for your project, since multiple migration sets can be applied to the same database.
+
+Here is a simple example of a ``migrations.py`` file which creates a table, then adds an extra column to that table in a second migration:
+
+.. code-block:: python
+
+    from sqlite_utils import Migrations
+
+    migrations = Migrations("creatures")
+
+    @migrations()
+    def create_table(db):
+        db["creatures"].create(
+            {"id": int, "name": str, "species": str},
+            pk="id",
+        )
+
+    @migrations()
+    def add_weight(db):
+        db["creatures"].add_column("weight", float)
+
+.. _migrations_python:
+
+Applying migrations in Python
+=============================
+
+Once you have a ``Migrations(name)`` collection with one or more migrations registered to it, you can execute them in Python code like this:
+
+.. code-block:: python
+
+    from sqlite_utils import Database
+
+    db = Database("creatures.db")
+    migrations.apply(db)
+
+Running ``migrations.apply(db)`` repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped.
+
+Migration functions are applied in the order that they were registered. The function name is used as the migration name unless you pass one explicitly:
+
+.. code-block:: python
+
+    @migrations(name="001_create_table")
+    def create_table(db):
+        db["creatures"].create({"id": int, "name": str}, pk="id")
+
+When you apply a set of migrations you can stop part way through by specifying a ``stop_before=`` migration name:
+
+.. code-block:: python
+
+    migrations.apply(db, stop_before="add_weight")
+
+.. _migrations_transactions:
+
+Migrations and transactions
+===========================
+
+Each migration runs inside a transaction, together with the ``_sqlite_migrations`` record of it having been applied. If a migration function raises an exception, everything it did is rolled back, no record is written and the migration stays pending - so fixing the error and re-applying will run that migration again from a clean state. Migrations that completed earlier in the same ``apply()`` run stay applied.
+
+Some operations cannot run inside a transaction, for example ``VACUUM`` or changing the journal mode with ``db.enable_wal()``. Register migrations like these with ``transactional=False``:
+
+.. code-block:: python
+
+    @migrations(transactional=False)
+    def compact(db):
+        db.execute("VACUUM")
+
+A migration registered with ``transactional=False`` runs without a wrapping transaction, so if it fails part way through any changes it already made will not be rolled back, and re-applying will run the whole function again.
+
+Avoid calling ``db.commit()`` or otherwise managing transactions manually inside a transactional migration - register the migration with ``transactional=False`` if it needs to control its own transactions. Using ``with db.atomic():`` blocks inside a migration is fine: they nest as savepoints within the migration's transaction, so the migration as a whole still commits or rolls back as a single unit. See :ref:`python_api_transactions`.
+
+Applying migrations using the CLI
+=================================
+
+Run migrations using the ``sqlite-utils migrate`` command:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db path/to/migrations.py
+
+The first argument is the database file. The remaining arguments can be paths to migration files or directories containing migration files.
+
+If you omit migration paths, ``sqlite-utils`` searches the current directory and subdirectories for files called ``migrations.py``:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db
+
+You can also pass a directory. Every ``migrations.py`` file in that directory tree will be considered:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db path/to/project/
+
+Running the command repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped.
+
+Listing migrations
+==================
+
+Use ``--list`` to show applied and pending migrations without running them. This is a read-only operation - it will not create the database file or the ``_sqlite_migrations`` table:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db --list
+
+Example output:
+
+.. code-block:: output
+
+    Migrations for: creatures
+
+      Applied:
+        create_table - 2026-06-09 17:23:12.048092+00:00
+        add_weight - 2026-06-09 17:23:12.051249+00:00
+
+      Pending:
+        add_age
+
+Stopping before a migration
+===========================
+
+When applying migrations using the CLI, you can stop before a named migration:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db path/to/migrations.py --stop-before add_weight
+
+This applies any pending migrations before ``add_weight`` and leaves ``add_weight`` and later migrations pending. An unqualified migration name matches in any migration set.
+
+You can also target a specific migration set using ``migration_set:migration_name``. This is useful if a migrations file contains more than one migration set, or if multiple sets use the same migration name:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db path/to/migrations.py \
+      --stop-before creatures:add_weight \
+      --stop-before sales:drop_index
+
+The ``--stop-before`` option can be passed more than once.
+
+If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. Naming a migration that has already been applied is also an error - stopping before it is impossible to honor - and no pending migrations are applied.
+
+Verbose output
+==============
+
+Use ``--verbose`` or ``-v`` to show the schema before and after migrations are applied, plus a unified diff when the schema changes:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db --verbose
+
+Migrating from sqlite-migrate
+=============================
+
+This system uses the same migration table format as the older `sqlite-migrate <https://github.com/simonw/sqlite-migrate>`__ package. To use existing migration files directly with ``sqlite-utils``, update their import from ``sqlite_migrate`` to ``sqlite_utils``:
+
+.. code-block:: python
+
+    from sqlite_utils import Migrations
+
+    migration = Migrations("creatures")
+
+    @migration()
+    def create_table(db):
+        db["creatures"].create({"id": int, "name": str}, pk="id")
+
+Python API
+==========
+
+.. autoclass:: sqlite_utils.migrations.Migrations
+   :members:
+   :undoc-members:
+   :exclude-members: _Migration, _AppliedMigration

docs/plugins.rst

@@ -0,0 +1,159 @@
+.. _plugins:
+
+=========
+ Plugins
+=========
+
+``sqlite-utils`` supports plugins, which can be used to add extra features to the software.
+
+Plugins can add new commands, for example ``sqlite-utils some-command ...``
+
+Plugins can be installed using the ``sqlite-utils install`` command:
+
+.. code-block:: bash
+
+    sqlite-utils install sqlite-utils-name-of-plugin
+
+You can see a JSON list of plugins that have been installed by running this:
+
+.. code-block:: bash
+
+    sqlite-utils plugins
+
+Plugin hooks such as :ref:`plugins_hooks_prepare_connection` affect each instance of the ``Database`` class. You can opt-out of these plugins by creating that class instance like so:
+
+.. code-block:: python
+
+    db = Database(memory=True, execute_plugins=False)
+
+.. _plugins_building:
+
+Building a plugin
+-----------------
+
+Plugins are created in a directory named after the plugin. To create a "hello world" plugin, first create a ``hello-world`` directory:
+
+.. code-block:: bash
+
+    mkdir hello-world
+    cd hello-world
+
+In that folder create two files. The first is a ``pyproject.toml`` file describing the plugin:
+
+.. code-block:: toml
+
+    [project]
+    name = "sqlite-utils-hello-world"
+    version = "0.1"
+
+    [project.entry-points.sqlite_utils]
+    hello_world = "sqlite_utils_hello_world"
+
+The ``[project.entry-points.sqlite_utils]`` section tells ``sqlite-utils`` which module to load when executing the plugin.
+
+Then create ``sqlite_utils_hello_world.py`` with the following content:
+
+.. code-block:: python
+
+    import click
+    import sqlite_utils
+
+    @sqlite_utils.hookimpl
+    def register_commands(cli):
+        @cli.command()
+        def hello_world():
+            "Say hello world"
+            click.echo("Hello world!")
+
+Install the plugin in "editable" mode - so you can make changes to the code and have them picked up instantly by ``sqlite-utils`` - like this:
+
+.. code-block:: bash
+
+    sqlite-utils install -e .
+
+Or pass the path to your plugin directory:
+
+.. code-block:: bash
+
+    sqlite-utils install -e /dev/sqlite-utils-hello-world
+
+Now, running this should execute your new command:
+
+.. code-block:: bash
+
+    sqlite-utils hello-world
+
+Your command will also be listed in the output of ``sqlite-utils --help``.
+
+See the `LLM plugin documentation <https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#distributing-your-plugin>`__ for tips on distributing your plugin.
+
+.. _plugins_hooks:
+
+Plugin hooks
+------------
+
+Plugin hooks allow ``sqlite-utils`` to be customized.
+
+.. _plugins_hooks_register_commands:
+
+register_commands(cli)
+~~~~~~~~~~~~~~~~~~~~~~
+
+This hook can be used to register additional commands with the ``sqlite-utils`` CLI. It is called with the ``cli`` object, which is a ``click.Group`` instance.
+
+Example implementation:
+
+.. code-block:: python
+
+    import click
+    import sqlite_utils
+
+    @sqlite_utils.hookimpl
+    def register_commands(cli):
+        @cli.command()
+        def hello_world():
+            "Say hello world"
+            click.echo("Hello world!")
+
+New commands implemented by plugins can invoke existing commands using the `context.invoke <https://click.palletsprojects.com/en/stable/api/#click.Context.invoke>`__ mechanism.
+
+As a special niche feature, if your plugin needs to import some files and then act against an in-memory database containing those files you can forward to the :ref:`sqlite-utils memory command <cli_memory>` and pass it ``return_db=True``:
+
+.. code-block:: python
+
+    @cli.command()
+    @click.pass_context
+    @click.argument(
+        "paths",
+        type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+        required=False,
+        nargs=-1,
+    )
+    def show_schema_for_files(ctx, paths):
+        from sqlite_utils.cli import memory
+        db = ctx.invoke(memory, paths=paths, return_db=True)
+        # Now do something with that database
+        click.echo(db.schema)
+
+.. _plugins_hooks_prepare_connection:
+
+prepare_connection(conn)
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This hook is called when a new SQLite database connection is created. You can use it to `register custom SQL functions <https://docs.python.org/2/library/sqlite3.html#sqlite3.Connection.create_function>`_, aggregates and collations. For example:
+
+.. code-block:: python
+
+    import sqlite_utils
+
+    @sqlite_utils.hookimpl
+    def prepare_connection(conn):
+        conn.create_function(
+            "hello", 1, lambda name: f"Hello, {name}!"
+        )
+
+This registers a SQL function called ``hello`` which takes a single argument and can be called like this:
+
+.. code-block:: sql
+
+    select hello("world"); -- "Hello, world!"

docs/reference.rst

@@ -0,0 +1,117 @@
+.. _reference:
+
+===============
+ API reference
+===============
+
+.. contents:: :local:
+   :class: this-will-duplicate-information-and-it-is-still-useful-here
+
+.. _reference_db_database:
+
+sqlite_utils.db.Database
+========================
+
+.. autoclass:: sqlite_utils.db.Database
+    :members:
+    :undoc-members:
+    :special-members: __getitem__
+    :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys
+
+.. _reference_db_queryable:
+
+sqlite_utils.db.Queryable
+=========================
+
+:ref:`Table <reference_db_table>` and :ref:`View <reference_db_view>` are  both subclasses of ``Queryable``, providing access to the following methods:
+
+.. autoclass:: sqlite_utils.db.Queryable
+    :members:
+    :undoc-members:
+    :exclude-members: execute_count
+
+.. _reference_db_table:
+
+sqlite_utils.db.Table
+=====================
+
+.. autoclass:: sqlite_utils.db.Table
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :exclude-members: guess_foreign_column, value_or_default, build_insert_queries_and_params, insert_chunk, add_missing_columns
+
+.. _reference_db_view:
+
+sqlite_utils.db.View
+====================
+
+.. autoclass:: sqlite_utils.db.View
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. _reference_db_other:
+
+Other
+=====
+
+.. _reference_db_other_column:
+
+sqlite_utils.db.Column
+----------------------
+
+.. autoclass:: sqlite_utils.db.Column
+
+.. _reference_db_other_column_details:
+
+sqlite_utils.db.ColumnDetails
+-----------------------------
+
+.. autoclass:: sqlite_utils.db.ColumnDetails
+
+.. _reference_db_other_foreign_key:
+
+sqlite_utils.db.ForeignKey
+--------------------------
+
+.. autoclass:: sqlite_utils.db.ForeignKey
+
+sqlite_utils.utils
+==================
+
+.. _reference_utils_hash_record:
+
+sqlite_utils.utils.hash_record
+------------------------------
+
+.. autofunction:: sqlite_utils.utils.hash_record
+
+.. _reference_utils_rows_from_file:
+
+sqlite_utils.utils.rows_from_file
+---------------------------------
+
+.. autofunction:: sqlite_utils.utils.rows_from_file
+
+.. _reference_utils_typetracker:
+
+sqlite_utils.utils.TypeTracker
+------------------------------
+
+.. autoclass:: sqlite_utils.utils.TypeTracker
+   :members: wrap, types
+
+.. _reference_utils_chunks:
+
+sqlite_utils.utils.chunks
+-------------------------
+
+.. autofunction:: sqlite_utils.utils.chunks
+
+.. _reference_utils_flatten:
+
+sqlite_utils.utils.flatten
+--------------------------
+
+.. autofunction:: sqlite_utils.utils.flatten

docs/tutorial.ipynb

@@ -39,10 +39,8 @@
       "Requirement already satisfied: sqlite-fts4 in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (1.0.1)\n",
       "Requirement already satisfied: click in /Users/simon/Library/Python/3.9/lib/python/site-packages (from sqlite_utils) (7.1.2)\n",
       "Requirement already satisfied: tabulate in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (0.8.7)\n",
-      "Requirement already satisfied: dateutils in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from sqlite_utils) (0.6.12)\n",
-      "Requirement already satisfied: python-dateutil in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2.8.1)\n",
-      "Requirement already satisfied: pytz in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2021.1)\n",
-      "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from python-dateutil->dateutils->sqlite_utils) (1.16.0)\n",
+      "Requirement already satisfied: python-dateutil in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-package (from sqlite-utils) (2.8.1)\n",
+      "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-package (from python-dateutil->sqlite-utils) (1.16.0)\n",
       "\u001b[33mWARNING: You are using pip version 21.1.1; however, version 21.2.2 is available.\n",
       "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n",
       "Note: you may need to restart the kernel to use updated packages.\n"
@@ -182,10 +180,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CREATE TABLE [creatures] (\n",
-      "   [name] TEXT,\n",
-      "   [species] TEXT,\n",
-      "   [age] FLOAT\n",
+      "CREATE TABLE \"creatures\" (\n",
+      "   \"name\" TEXT,\n",
+      "   \"species\" TEXT,\n",
+      "   \"age\" FLOAT\n",
       ")\n"
      ]
     }
@@ -536,11 +534,11 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CREATE TABLE [creatures] (\n",
-      "   [id] INTEGER PRIMARY KEY,\n",
-      "   [name] TEXT,\n",
-      "   [species] TEXT,\n",
-      "   [age] FLOAT\n",
+      "CREATE TABLE \"creatures\" (\n",
+      "   \"id\" INTEGER PRIMARY KEY,\n",
+      "   \"name\" TEXT,\n",
+      "   \"species\" TEXT,\n",
+      "   \"age\" FLOAT\n",
       ")\n"
      ]
     }
@@ -931,11 +929,11 @@
      "output_type": "stream",
      "text": [
       "CREATE TABLE \"creatures\" (\n",
-      "   [id] INTEGER PRIMARY KEY,\n",
-      "   [name] TEXT,\n",
-      "   [species_id] INTEGER,\n",
-      "   [age] FLOAT,\n",
-      "   FOREIGN KEY([species_id]) REFERENCES [species]([id])\n",
+      "   \"id\" INTEGER PRIMARY KEY,\n",
+      "   \"name\" TEXT,\n",
+      "   \"species_id\" INTEGER,\n",
+      "   \"age\" FLOAT,\n",
+      "   FOREIGN KEY(\"species_id\") REFERENCES \"species\"(\"id\")\n",
       ")\n",
       "[{'id': 1, 'name': 'Cleo', 'species_id': 1, 'age': 6.0}, {'id': 2, 'name': 'Lila', 'species_id': 2, 'age': 0.8}, {'id': 3, 'name': 'Bants', 'species_id': 2, 'age': 0.8}, {'id': 4, 'name': 'Azi', 'species_id': 2, 'age': 0.8}, {'id': 5, 'name': 'Snowy', 'species_id': 2, 'age': 0.9}, {'id': 6, 'name': 'Blue', 'species_id': 2, 'age': 0.9}]\n"
      ]
@@ -964,9 +962,9 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CREATE TABLE [species] (\n",
-      "   [id] INTEGER PRIMARY KEY,\n",
-      "   [species] TEXT\n",
+      "CREATE TABLE \"species\" (\n",
+      "   \"id\" INTEGER PRIMARY KEY,\n",
+      "   \"species\" TEXT\n",
       ")\n",
       "[{'id': 1, 'species': 'dog'}, {'id': 2, 'species': 'chicken'}]\n"
      ]

docs/upgrading.rst

@@ -0,0 +1,146 @@
+.. _upgrading:
+
+===========
+ Upgrading
+===========
+
+This page describes the changes you may need to make to your own code or scripts when upgrading between major versions of ``sqlite-utils``.
+
+For the full list of changes in every release see the :ref:`changelog`.
+
+.. _upgrading_3_to_4:
+
+Upgrading from 3.x to 4.0
+=========================
+
+Requirements
+------------
+
+- Python 3.10 or higher is required.
+- The ``click`` dependency must be version 8.3.1 or later.
+
+Command-line changes
+--------------------
+
+**Type detection is now the default for CSV and TSV imports.** ``sqlite-utils insert`` and ``sqlite-utils upsert`` now detect column types when importing CSV or TSV data - previously every column was created as ``TEXT`` unless you passed ``--detect-types``. To restore the old behavior pass the new ``--no-detect-types`` flag:
+
+.. code-block:: bash
+
+    sqlite-utils insert data.db rows data.csv --csv --no-detect-types
+
+Two related things have been removed:
+
+- The ``SQLITE_UTILS_DETECT_TYPES`` environment variable.
+- The old ``-d/--detect-types`` flag itself. Since detection is now the default the flag did nothing - remove it from any scripts that used it.
+
+**The convert command no longer skips falsey values.** ``sqlite-utils convert`` previously skipped values that evaluated to ``False`` (empty strings, ``0``) unless you passed ``--no-skip-false``. All values are now converted and the ``--no-skip-false`` flag has been removed.
+
+**drop-table and drop-view check the object type.** ``sqlite-utils drop-table`` now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. If you relied on that (unlikely), use the matching command instead.
+
+**sqlite-utils tui has moved to a plugin.** The optional terminal interface is now provided by the `sqlite-utils-tui <https://github.com/simonw/sqlite-utils-tui>`__ plugin:
+
+.. code-block:: bash
+
+    sqlite-utils install sqlite-utils-tui
+
+Python API changes
+------------------
+
+**db.query() now rejects SQL that does not return rows.** This is likely the most common change you will need to make to existing code. ``db.query()`` used to accept any SQL statement - passing one that returns no rows, such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause or a ``CREATE TABLE``, did nothing at all, silently. Those statements now raise a ``ValueError``, and are rolled back so they have no effect on the database. Transaction control statements (``BEGIN``, ``COMMIT``, ``END``, ``ROLLBACK``, ``SAVEPOINT``, ``RELEASE``) plus ``VACUUM``, ``ATTACH`` and ``DETACH`` are also rejected with a ``ValueError``, without being executed at all. Use ``db.execute()`` for statements that do not return rows:
+
+.. code-block:: python
+
+    # 3.x accepted this but silently did nothing:
+    db.query("update dogs set name = 'Cleopaws'")
+
+    # In 4.0 use execute() for SQL that does not return rows:
+    db.execute("update dogs set name = 'Cleopaws'")
+
+**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily, but errors in your SQL raise at the ``db.query()`` call site rather than on first iteration, and a write with a ``RETURNING`` clause takes effect even if you never iterate over its results.
+
+**db.table() no longer returns views.** ``db.table(name)`` now raises a ``sqlite_utils.db.NoTable`` exception if ``name`` is a SQL view. Use the new ``db.view(name)`` method for views:
+
+.. code-block:: python
+
+    table = db.table("my_table")
+    view = db.view("my_view")
+
+``db["name"]`` still returns either a ``Table`` or a ``View`` depending on what exists in the database.
+
+**Upserts use INSERT ... ON CONFLICT.** Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax rather than the previous ``INSERT OR IGNORE`` followed by ``UPDATE``. If your code depends on the old behavior, pass ``use_old_upsert=True`` to the ``Database()`` constructor - see :ref:`python_api_old_upsert`.
+
+**Upsert records must include their primary keys.** ``table.upsert()`` and ``table.upsert_all()`` now raise ``sqlite_utils.db.PrimaryKeyRequired`` if a record is missing a value for any primary key column (or has ``None`` for one). Previously such records were quietly inserted as new rows. Relatedly, ``pk=`` is now optional when the table already exists with a primary key - it is detected automatically.
+
+**Floating point columns are now REAL.** Auto-detected floating point columns are created with the correct SQLite type ``REAL`` instead of ``FLOAT``. Code that inspects column types should expect ``REAL``.
+
+**Generated schemas use double quotes.** Tables created by this library now wrap table and column names in ``"double-quotes"`` where they previously used ``[square-braces]``. If you compare ``table.schema`` strings against expected values you will need to update them.
+
+**table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values.
+
+**Null values are no longer extracted into lookup tables.** ``table.extract()`` and the ``sqlite-utils extract`` command leave rows alone if every extracted column is ``null`` - the new foreign key column is left as ``null`` instead of pointing at an all-``null`` record in the lookup table. The ``extracts=`` insert option similarly keeps ``None`` values as ``null``. Relatedly, ``table.lookup()`` now compares values using ``IS`` so that looking up a value containing ``None`` returns the existing matching row - previously it inserted a duplicate row on every call.
+
+**ensure_autocommit_off() is now ensure_autocommit_on().** The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``. The old name described the opposite of what the method did: it temporarily puts the connection into driver-level autocommit mode (by setting ``isolation_level = None``), so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. The behavior is unchanged - update any calls to use the new name.
+
+**View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method.
+
+**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained new fields - ``columns``, ``other_columns``, ``is_compound``, ``on_delete`` and ``on_update`` - so that compound (multi-column) foreign keys and foreign key actions can be represented. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead:
+
+.. code-block:: python
+
+    # 3.x - tuple unpacking, no longer works:
+    for table, column, other_table, other_column in db["courses"].foreign_keys:
+        ...
+
+    # 4.0 - access fields by name:
+    for fk in db["courses"].foreign_keys:
+        fk.table, fk.column, fk.other_table, fk.other_column
+
+Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. Like the old namedtuple, ``ForeignKey`` instances are immutable and hashable - they can be collected into sets and used as dictionary keys. Note that equality now includes the ``on_delete`` and ``on_update`` actions: a ``ForeignKey`` with ``ON DELETE CASCADE`` is not equal to one without.
+
+Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` tuples instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are one-item tuples.
+
+Two related behavior changes to ``table.transform()``: compound foreign keys now survive a transform (previously they were split into separate single-column keys), and ``ON DELETE``/``ON UPDATE`` actions such as ``ON DELETE CASCADE`` are now preserved (previously they were silently stripped from the schema).
+
+**Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``.
+
+**Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() <python_api_atomic>` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice:
+
+- Write statements executed with raw ``db.execute()`` calls now commit automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that nothing committed - if your code used ``db.execute()`` for writes and relied on ``db.conn.rollback()`` to undo them, open an explicit transaction with the new ``db.begin()`` method first.
+- Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead.
+- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, instead of silently committing it.
+- Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - a transaction you explicitly opened with ``db.begin()`` and did not commit is rolled back.
+- ``Database()`` rejects connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options, raising ``sqlite_utils.db.TransactionError``. On those connections every write the library made was silently discarded when the connection closed.
+
+Packaging changes
+-----------------
+
+- ``sqlite-utils`` now uses ``pyproject.toml`` in place of ``setup.py``.
+- ``pip`` is now a runtime dependency, used by the ``sqlite-utils install`` and ``uninstall`` commands.
+
+New features to be aware of
+---------------------------
+
+Not breaking changes, but new in 4.0 and worth knowing about when you upgrade:
+
+- A :ref:`database migrations system <migrations>`, incorporating the functionality of the ``sqlite-migrate`` plugin. If you used that plugin, the built-in system reads the same ``_sqlite_migrations`` table - your applied migrations will not run again. Update your migration files to use ``from sqlite_utils import Migrations``.
+- :ref:`db.atomic() <python_api_atomic>` for nested transaction support.
+- ``table.insert_all()`` and ``table.upsert_all()`` accept an iterator of lists or tuples as an alternative to dictionaries - see :ref:`python_api_insert_lists`.
+
+.. _upgrading_2_to_3:
+
+Upgrading from 2.x to 3.0
+=========================
+
+The 3.0 release redesigned search. The breaking changes were minor:
+
+- ``table.search()`` returns a generator of dictionaries, sorted by relevance. It previously returned a list of tuples sorted by ``rowid``.
+- The ``-c`` shortcut for ``--csv`` and the ``-f`` shortcut for ``--fmt`` were removed from the CLI - use the full option names.
+
+.. _upgrading_1_to_2:
+
+Upgrading from 1.x to 2.0
+=========================
+
+The 2.0 release changed the meaning of *upsert*. In 1.x, ``table.upsert()`` and ``table.upsert_all()`` actually performed ``INSERT OR REPLACE`` operations - entirely replacing the existing row. Since 2.0 an upsert updates only the columns you provide, leaving other columns untouched.
+
+If you want the 1.x behavior, use ``table.insert(..., replace=True)`` or ``table.insert_all(..., replace=True)`` instead.

mypy.ini

@@ -0,0 +1,32 @@
+[mypy]
+python_version = 3.10
+warn_return_any = False
+warn_unused_configs = True
+warn_redundant_casts = False
+warn_unused_ignores = False
+check_untyped_defs = True
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+no_implicit_optional = True
+strict_equality = True
+
+[mypy-sqlite_utils.cli]
+ignore_errors = True
+
+[mypy-pysqlite3.*]
+ignore_missing_imports = True
+
+[mypy-sqlite_dump.*]
+ignore_missing_imports = True
+
+[mypy-sqlite_fts4.*]
+ignore_missing_imports = True
+
+[mypy-pandas.*]
+ignore_missing_imports = True
+
+[mypy-numpy.*]
+ignore_missing_imports = True
+
+[mypy-tests.*]
+ignore_errors = True

pyproject.toml

@@ -0,0 +1,85 @@
+[project]
+name = "sqlite-utils"
+version = "4.1.1"
+description = "CLI tool and Python library for manipulating SQLite databases"
+readme = { file = "README.md", content-type = "text/markdown" }
+authors = [
+    { name = "Simon Willison" },
+]
+license = "Apache-2.0"
+requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: End Users/Desktop",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Database",
+]
+
+dependencies = [
+    "click>=8.3.1",
+    "click-default-group>=1.2.3",
+    "pluggy",
+    "python-dateutil",
+    "sqlite-fts4",
+    "tabulate",
+    "pip",
+]
+
+[dependency-groups]
+dev = [
+    "black>=26.3.1",
+    "click>=8.4.2",
+    "cogapp",
+    "hypothesis",
+    "pytest",
+    # mypy
+    "data-science-types",
+    "mypy",
+    "types-click",
+    "types-pluggy",
+    "types-python-dateutil",
+    "types-tabulate",
+    # flake8
+    "flake8",
+    "flake8-pyproject",
+    "ty>=0.0.37",
+    # For stable cog:
+    "tabulate>=0.10.0",
+]
+docs = [
+    "codespell",
+    "furo",
+    "pygments-csv-lexer",
+    "sphinx-autobuild",
+    "sphinx-copybutton",
+]
+
+[project.urls]
+Homepage = "https://github.com/simonw/sqlite-utils"
+Documentation = "https://sqlite-utils.datasette.io/en/stable/"
+Changelog = "https://sqlite-utils.datasette.io/en/stable/changelog.html"
+Issues = "https://github.com/simonw/sqlite-utils/issues"
+CI = "https://github.com/simonw/sqlite-utils/actions"
+
+[project.scripts]
+sqlite-utils = "sqlite_utils.cli:cli"
+
+[build-system]
+# setuptools 77+ is needed for the PEP 639 license = "Apache-2.0" expression
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[tool.flake8]
+max-line-length = 160
+# Black compatibility, E203 whitespace before ':':
+extend-ignore = ["E203"]
+extend-exclude = [".venv", "build", "dist", "docs", "sqlite_utils.egg-info"]
+
+[tool.setuptools.package-data]
+sqlite_utils = ["py.typed"]

setup.cfg

@@ -1,3 +0,0 @@
-[flake8]
-max-line-length = 160
-extend-ignore = E203 # for Black

setup.py

@@ -1,65 +0,0 @@
-from setuptools import setup, find_packages
-import io
-import os
-
-VERSION = "3.15"
-
-
-def get_long_description():
-    with io.open(
-        os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"),
-        encoding="utf8",
-    ) as fp:
-        return fp.read()
-
-
-setup(
-    name="sqlite-utils",
-    description="CLI tool and Python utility functions for manipulating SQLite databases",
-    long_description=get_long_description(),
-    long_description_content_type="text/markdown",
-    author="Simon Willison",
-    version=VERSION,
-    license="Apache License, Version 2.0",
-    packages=find_packages(exclude=["tests", "tests.*"]),
-    install_requires=[
-        "sqlite-fts4",
-        "click",
-        "click-default-group",
-        "tabulate",
-        "dateutils",
-    ],
-    setup_requires=["pytest-runner"],
-    extras_require={
-        "test": ["pytest", "black", "hypothesis"],
-        "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"],
-        "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"],
-        "flake8": ["flake8"],
-    },
-    entry_points="""
-        [console_scripts]
-        sqlite-utils=sqlite_utils.cli:cli
-    """,
-    tests_require=["sqlite-utils[test]"],
-    url="https://github.com/simonw/sqlite-utils",
-    project_urls={
-        "Documentation": "https://sqlite-utils.datasette.io/en/stable/",
-        "Changelog": "https://sqlite-utils.datasette.io/en/stable/changelog.html",
-        "Source code": "https://github.com/simonw/sqlite-utils",
-        "Issues": "https://github.com/simonw/sqlite-utils/issues",
-        "CI": "https://github.com/simonw/sqlite-utils/actions",
-    },
-    python_requires=">=3.6",
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Intended Audience :: Developers",
-        "Intended Audience :: Science/Research",
-        "Intended Audience :: End Users/Desktop",
-        "Topic :: Database",
-        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-    ],
-)

sqlite_utils/__init__.py

@@ -1,4 +1,7 @@
-from .db import Database
 from .utils import suggest_column_types
+from .hookspecs import hookimpl
+from .hookspecs import hookspec
+from .db import Database
+from .migrations import Migrations
 
-__all__ = ["Database", "suggest_column_types"]
+__all__ = ["Database", "Migrations", "suggest_column_types", "hookimpl", "hookspec"]

sqlite_utils/__main__.py

@@ -0,0 +1,4 @@
+from .cli import cli
+
+if __name__ == "__main__":
+    cli()

sqlite_utils/hookspecs.py

@@ -0,0 +1,18 @@
+import sqlite3
+
+import click
+from pluggy import HookimplMarker
+from pluggy import HookspecMarker
+
+hookspec = HookspecMarker("sqlite_utils")
+hookimpl = HookimplMarker("sqlite_utils")
+
+
+@hookspec
+def register_commands(cli: click.Group) -> None:
+    """Register additional CLI commands, e.g. 'sqlite-utils mycommand ...'"""
+
+
+@hookspec
+def prepare_connection(conn: sqlite3.Connection) -> None:
+    """Modify SQLite connection in some way e.g. register custom SQL functions"""

sqlite_utils/migrations.py

@@ -0,0 +1,177 @@
+from collections.abc import Iterable
+from dataclasses import dataclass
+import datetime
+from typing import Callable, cast, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from sqlite_utils.db import Database, Table
+
+
+class Migrations:
+    migrations_table = "_sqlite_migrations"
+
+    @dataclass
+    class _Migration:
+        name: str
+        fn: Callable
+        transactional: bool = True
+
+    @dataclass
+    class _AppliedMigration:
+        name: str
+        # A string timestamp such as "2026-07-04 12:00:00.000000+00:00" -
+        # stored as TEXT in the _sqlite_migrations table
+        applied_at: str
+
+    def __init__(self, name: str):
+        """
+        :param name: The name of the migration set. This should be unique.
+        """
+        self.name = name
+        self._migrations: list[Migrations._Migration] = []
+
+    def __call__(
+        self, *, name: str | None = None, transactional: bool = True
+    ) -> Callable:
+        """
+        :param name: The name to use for this migration - if not provided,
+          the name of the function will be used.
+        :param transactional: If ``True`` (the default) the migration and the
+          record of it having been applied are wrapped in a transaction, which
+          will be rolled back if the migration raises an exception. Pass
+          ``False`` for migrations that cannot run inside a transaction, for
+          example those that execute ``VACUUM``.
+        """
+
+        def inner(func: Callable) -> Callable:
+            migration_name = name or getattr(func, "__name__")
+            if any(m.name == migration_name for m in self._migrations):
+                raise ValueError(
+                    "Migration '{}' is already registered in set '{}'".format(
+                        migration_name, self.name
+                    )
+                )
+            self._migrations.append(
+                self._Migration(migration_name, func, transactional)
+            )
+            return func
+
+        return inner
+
+    def pending(self, db: "Database") -> list["Migrations._Migration"]:
+        """
+        Return a list of pending migrations.
+
+        This is a read-only operation - it does not write to the database.
+        """
+        already_applied = {migration.name for migration in self.applied(db)}
+        return [
+            migration
+            for migration in self._migrations
+            if migration.name not in already_applied
+        ]
+
+    def applied(self, db: "Database") -> list["Migrations._AppliedMigration"]:
+        """
+        Return a list of applied migrations, in the order they were applied.
+
+        This is a read-only operation - it does not write to the database.
+        """
+        table = _table(db, self.migrations_table)
+        if not table.exists():
+            return []
+        return [
+            self._AppliedMigration(name=row["name"], applied_at=row["applied_at"])
+            for row in table.rows_where(
+                "migration_set = ?", [self.name], order_by="rowid"
+            )
+        ]
+
+    def apply(self, db: "Database", *, stop_before: str | Iterable[str] | None = None):
+        """
+        Apply any pending migrations to the database.
+
+        Each migration runs inside a transaction, together with the record of
+        it having been applied - if the migration raises an exception its
+        changes are rolled back, no record is written and the migration stays
+        pending. Migrations registered with ``transactional=False`` run
+        outside of a transaction.
+
+        :raises ValueError: if a ``stop_before`` name matches a migration in
+          this set that has already been applied - stopping before it is
+          impossible to honor, and no pending migrations are applied
+        """
+        if stop_before is None:
+            stop_before_names = set()
+        elif isinstance(stop_before, str):
+            stop_before_names = {stop_before}
+        else:
+            stop_before_names = set(stop_before)
+        # A stop_before naming an already-applied migration cannot be
+        # honored - error rather than applying everything after it. Names
+        # not in this set at all are ignored, because unqualified CLI
+        # values are offered to every migration set
+        already_applied = stop_before_names.intersection(
+            migration.name for migration in self.applied(db)
+        )
+        if already_applied:
+            raise ValueError(
+                "Cannot stop before migration{} {} in set '{}' - already "
+                "been applied".format(
+                    "s" if len(already_applied) > 1 else "",
+                    ", ".join(sorted(already_applied)),
+                    self.name,
+                )
+            )
+        self.ensure_migrations_table(db)
+        for migration in self.pending(db):
+            name = migration.name
+            if name in stop_before_names:
+                return
+            if migration.transactional:
+                with db.atomic():
+                    migration.fn(db)
+                    self._record_applied(db, name)
+            else:
+                migration.fn(db)
+                self._record_applied(db, name)
+
+    def _record_applied(self, db: "Database", name: str):
+        _table(db, self.migrations_table).insert(
+            {
+                "migration_set": self.name,
+                "name": name,
+                "applied_at": str(datetime.datetime.now(datetime.timezone.utc)),
+            }
+        )
+
+    def ensure_migrations_table(self, db: "Database"):
+        """
+        Ensure the _sqlite_migrations table exists and has the correct schema.
+        """
+        table = _table(db, self.migrations_table)
+        if not table.exists():
+            table.create(
+                {
+                    "id": int,
+                    "migration_set": str,
+                    "name": str,
+                    "applied_at": str,
+                },
+                pk="id",
+            )
+            table.create_index(["migration_set", "name"], unique=True)
+        elif table.pks != ["id"]:
+            table.transform(pk="id")
+            unique_indexes = {tuple(index.columns) for index in table.indexes}
+            if ("migration_set", "name") not in unique_indexes:
+                table.create_index(["migration_set", "name"], unique=True)
+
+    def __repr__(self):
+        return "<Migrations '{}': [{}]>".format(
+            self.name, ", ".join(m.name for m in self._migrations)
+        )
+
+
+def _table(db: "Database", name: str) -> "Table":
+    return cast("Table", db[name])

sqlite_utils/plugins.py

@@ -0,0 +1,35 @@
+from typing import Dict, List, Union
+
+import pluggy
+import sys
+from . import hookspecs
+
+pm: pluggy.PluginManager = pluggy.PluginManager("sqlite_utils")
+pm.add_hookspecs(hookspecs)
+_plugins_loaded = False
+
+
+def ensure_plugins_loaded() -> None:
+    global _plugins_loaded
+    if _plugins_loaded or getattr(sys, "_called_from_test", False):
+        return
+    pm.load_setuptools_entrypoints("sqlite_utils")
+    _plugins_loaded = True
+
+
+def get_plugins() -> List[Dict[str, Union[str, List[str]]]]:
+    ensure_plugins_loaded()
+    plugins: List[Dict[str, Union[str, List[str]]]] = []
+    plugin_to_distinfo = dict(pm.list_plugin_distinfo())
+    for plugin in pm.get_plugins():
+        hookcallers = pm.get_hookcallers(plugin) or []
+        plugin_info: Dict[str, Union[str, List[str]]] = {
+            "name": plugin.__name__,
+            "hooks": [h.name for h in hookcallers],
+        }
+        distinfo = plugin_to_distinfo.get(plugin)
+        if distinfo:
+            plugin_info["version"] = distinfo.version
+            plugin_info["name"] = distinfo.project_name
+        plugins.append(plugin_info)
+    return plugins

sqlite_utils/recipes.py

@@ -1,19 +1,76 @@
+from __future__ import annotations
+
+from typing import Callable, Optional
+
 from dateutil import parser
 import json
 
-
-def parsedate(value, dayfirst=False, yearfirst=False):
-    "Parse a date and convert it to ISO date format: yyyy-mm-dd"
-    return (
-        parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat()
-    )
+IGNORE: object = object()
+SET_NULL: object = object()
 
 
-def parsedatetime(value, dayfirst=False, yearfirst=False):
-    "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS"
-    return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat()
+def parsedate(
+    value: str,
+    dayfirst: bool = False,
+    yearfirst: bool = False,
+    errors: Optional[object] = None,
+) -> Optional[str]:
+    """
+    Parse a date and convert it to ISO date format: yyyy-mm-dd
+    \b
+    - dayfirst=True: treat xx as the day in xx/yy/zz
+    - yearfirst=True: treat xx as the year in xx/yy/zz
+    - errors=r.IGNORE to ignore values that cannot be parsed
+    - errors=r.SET_NULL to set values that cannot be parsed to null
+    """
+    if not value:
+        return value
+    try:
+        return (
+            parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst)
+            .date()
+            .isoformat()
+        )
+    except parser.ParserError:
+        if errors is IGNORE:
+            return value
+        elif errors is SET_NULL:
+            return None
+        else:
+            raise
 
 
-def jsonsplit(value, delimiter=",", type=str):
-    'Convert a string like a,b,c into a JSON array ["a", "b", "c"]'
+def parsedatetime(
+    value: str,
+    dayfirst: bool = False,
+    yearfirst: bool = False,
+    errors: Optional[object] = None,
+) -> Optional[str]:
+    """
+    Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS
+    \b
+    - dayfirst=True: treat xx as the day in xx/yy/zz
+    - yearfirst=True: treat xx as the year in xx/yy/zz
+    - errors=r.IGNORE to ignore values that cannot be parsed
+    - errors=r.SET_NULL to set values that cannot be parsed to null
+    """
+    if not value:
+        return value
+    try:
+        return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat()
+    except parser.ParserError:
+        if errors is IGNORE:
+            return value
+        elif errors is SET_NULL:
+            return None
+        else:
+            raise
+
+
+def jsonsplit(
+    value: str, delimiter: str = ",", type: Callable[[str], object] = str
+) -> str:
+    """
+    Convert a string like a,b,c into a JSON array ["a", "b", "c"]
+    """
     return json.dumps([type(s.strip()) for s in value.split(delimiter)])

sqlite_utils/utils.py

@@ -2,44 +2,154 @@ import base64
 import contextlib
 import csv
 import enum
+import hashlib
+import importlib
 import io
+import itertools
 import json
 import os
-from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type
+import sys
+from typing import (
+    Any,
+    BinaryIO,
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    TYPE_CHECKING,
+    TypeVar,
+    Union,
+    cast,
+)
 
 import click
 
-try:
-    import pysqlite3 as sqlite3  # type: ignore
-    import pysqlite3.dbapi2  # type: ignore
+from . import recipes
 
-    OperationalError = pysqlite3.dbapi2.OperationalError
-except ImportError:
-    # https://github.com/python/mypy/issues/1153#issuecomment-253842414
-    import sqlite3  # type: ignore
+if TYPE_CHECKING:
+    import sqlite3  # noqa: F401
+    from sqlite3 import dbapi2  # noqa: F401
+
+    OperationalError = dbapi2.OperationalError
+else:
+    try:
+        sqlite3 = importlib.import_module("pysqlite3")
+        dbapi2 = importlib.import_module("pysqlite3.dbapi2")
+        OperationalError = dbapi2.OperationalError
+    except ImportError:
+        import sqlite3  # noqa: F401
+        from sqlite3 import dbapi2  # noqa: F401
+
+        OperationalError = dbapi2.OperationalError
 
-    OperationalError = sqlite3.OperationalError
 
 SPATIALITE_PATHS = (
     "/usr/lib/x86_64-linux-gnu/mod_spatialite.so",
+    "/usr/lib/aarch64-linux-gnu/mod_spatialite.so",
     "/usr/local/lib/mod_spatialite.dylib",
+    "/usr/local/lib/mod_spatialite.so",
+    "/opt/homebrew/lib/mod_spatialite.dylib",
 )
 
+# Mainly so we can restore it if needed in the tests:
+ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit()
 
-def suggest_column_types(records):
-    all_column_types = {}
+# Type alias for row dictionaries - values can be various SQLite-compatible types
+RowValue = Union[None, int, float, str, bytes, bool, List[str]]
+Row = Dict[str, RowValue]
+
+T = TypeVar("T")
+
+
+class _CloseableIterator(Iterator[Row]):
+    """Iterator wrapper that closes a file when iteration is complete."""
+
+    def __init__(self, iterator: Iterator[Row], closeable: io.IOBase) -> None:
+        self._iterator = iterator
+        self._closeable = closeable
+
+    def __iter__(self) -> "_CloseableIterator":
+        return self
+
+    def __next__(self) -> Row:
+        try:
+            return next(self._iterator)
+        except StopIteration:
+            self._closeable.close()
+            raise
+
+    def close(self) -> None:
+        self._closeable.close()
+
+
+def maximize_csv_field_size_limit() -> None:
+    """
+    Increase the CSV field size limit to the maximum possible.
+    """
+    # https://stackoverflow.com/a/15063941
+    field_size_limit = sys.maxsize
+
+    while True:
+        try:
+            csv.field_size_limit(field_size_limit)
+            break
+        except OverflowError:
+            field_size_limit = int(field_size_limit / 10)
+
+
+def find_spatialite() -> Optional[str]:
+    """
+    The ``find_spatialite()`` function searches for the `SpatiaLite <https://www.gaia-gis.it/fossil/libspatialite/index>`__
+    SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found.
+
+    You can use it in code like this:
+
+    .. code-block:: python
+
+        from sqlite_utils import Database
+        from sqlite_utils.utils import find_spatialite
+
+        db = Database("mydb.db")
+        spatialite = find_spatialite()
+        if spatialite:
+            db.conn.enable_load_extension(True)
+            db.conn.load_extension(spatialite)
+
+        # or use with db.init_spatialite like this
+        db.init_spatialite(find_spatialite())
+
+    """
+    for path in SPATIALITE_PATHS:
+        if os.path.exists(path):
+            return path
+    return None
+
+
+def suggest_column_types(
+    records: Iterable[Dict[str, Any]],
+) -> Dict[str, type]:
+    all_column_types: Dict[str, Set[type]] = {}
     for record in records:
         for key, value in record.items():
             all_column_types.setdefault(key, set()).add(type(value))
     return types_for_column_types(all_column_types)
 
 
-def types_for_column_types(all_column_types):
-    column_types = {}
+def types_for_column_types(
+    all_column_types: Dict[str, Set[type]],
+) -> Dict[str, type]:
+    column_types: Dict[str, type] = {}
     for key, types in all_column_types.items():
         # Ignore null values if at least one other type present:
         if len(types) > 1:
             types.discard(None.__class__)
+        t: type
         if {None.__class__} == types:
             t = str
         elif len(types) == 1:
@@ -61,7 +171,7 @@ def types_for_column_types(all_column_types):
     return column_types
 
 
-def column_affinity(column_type):
+def column_affinity(column_type: str) -> type:
     # Implementation of SQLite affinity rules from
     # https://www.sqlite.org/datatype3.html#determination_of_column_affinity
     assert isinstance(column_type, str)
@@ -80,40 +190,42 @@ def column_affinity(column_type):
     return float
 
 
-def decode_base64_values(doc):
+def decode_base64_values(doc: Dict[str, Any]) -> Dict[str, Any]:
     # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them
     to_fix = [
         k
         for k in doc
         if isinstance(doc[k], dict)
-        and doc[k].get("$base64") is True
-        and "encoded" in doc[k]
+        and cast(dict, doc[k]).get("$base64") is True
+        and "encoded" in cast(dict, doc[k])
     ]
     if not to_fix:
         return doc
-    return dict(doc, **{k: base64.b64decode(doc[k]["encoded"]) for k in to_fix})
-
-
-def find_spatialite():
-    for path in SPATIALITE_PATHS:
-        if os.path.exists(path):
-            return path
-    return None
+    return dict(
+        doc, **{k: base64.b64decode(cast(dict, doc[k])["encoded"]) for k in to_fix}
+    )
 
 
 class UpdateWrapper:
-    def __init__(self, wrapped, update):
+    def __init__(self, wrapped: io.IOBase, update: Callable[[int], None]) -> None:
         self._wrapped = wrapped
         self._update = update
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[bytes]:
         for line in self._wrapped:
             self._update(len(line))
             yield line
 
+    def read(self, size: int = -1) -> bytes:
+        data = self._wrapped.read(size)
+        self._update(len(data))
+        return data
+
 
 @contextlib.contextmanager
-def file_progress(file, silent=False, **kwargs):
+def file_progress(
+    file: io.IOBase, silent: bool = False, **kwargs: object
+) -> Generator[Union[io.IOBase, "UpdateWrapper"], None, None]:
     if silent:
         yield file
         return
@@ -126,8 +238,8 @@ def file_progress(file, silent=False, **kwargs):
     if fileno == 0:  # 0 means stdin
         yield file
     else:
-        file_length = os.path.getsize(file.name)
-        with click.progressbar(length=file_length, **kwargs) as bar:
+        file_length = os.path.getsize(file.name)  # type: ignore
+        with click.progressbar(length=file_length, **kwargs) as bar:  # type: ignore
             yield UpdateWrapper(file, bar.update)
 
 
@@ -146,12 +258,87 @@ class RowsFromFileBadJSON(RowsFromFileError):
     pass
 
 
+class RowError(Exception):
+    pass
+
+
+def _extra_key_strategy(
+    reader: Iterable[Dict[Optional[str], object]],
+    ignore_extras: Optional[bool] = False,
+    extras_key: Optional[str] = None,
+) -> Iterable[Row]:
+    # Logic for handling CSV rows with more values than there are headings
+    for row in reader:
+        # DictReader adds a 'None' key with extra row values
+        if None not in row:
+            yield cast(Row, row)
+        elif ignore_extras:
+            # ignoring row.pop(none) because of this issue:
+            # https://github.com/simonw/sqlite-utils/issues/440#issuecomment-1155358637
+            row.pop(None)
+            yield cast(Row, row)
+        elif not extras_key:
+            extras = row.pop(None)
+            raise RowError(
+                "Row {} contained these extra values: {}".format(row, extras)
+            )
+        else:
+            extras_value = row.pop(None)
+            row_out = cast(Row, row)
+            row_out[extras_key] = cast(RowValue, extras_value)
+            yield row_out
+
+
 def rows_from_file(
     fp: BinaryIO,
     format: Optional[Format] = None,
     dialect: Optional[Type[csv.Dialect]] = None,
     encoding: Optional[str] = None,
-) -> Tuple[Iterable[dict], Format]:
+    ignore_extras: Optional[bool] = False,
+    extras_key: Optional[str] = None,
+) -> Tuple[Iterable[Row], Format]:
+    """
+    Load a sequence of dictionaries from a file-like object containing one of four different formats.
+
+    .. code-block:: python
+
+        from sqlite_utils.utils import rows_from_file
+        import io
+
+        rows, format = rows_from_file(io.StringIO("id,name\\n1,Cleo")))
+        print(list(rows), format)
+        # Outputs [{'id': '1', 'name': 'Cleo'}] Format.CSV
+
+    This defaults to attempting to automatically detect the format of the data, or you can pass in an
+    explicit format using the format= option.
+
+    Returns a tuple of ``(rows_generator, format_used)`` where ``rows_generator`` can be iterated over
+    to return dictionaries, while ``format_used`` is a value from the ``sqlite_utils.utils.Format`` enum:
+
+    .. code-block:: python
+
+        class Format(enum.Enum):
+            CSV = 1
+            TSV = 2
+            JSON = 3
+            NL = 4
+
+    If a CSV or TSV file includes rows with more fields than are declared in the header a
+    ``sqlite_utils.utils.RowError`` exception will be raised when you loop over the generator.
+
+    You can instead ignore the extra data by passing ``ignore_extras=True``.
+
+    Or pass ``extras_key="rest"`` to put those additional values in a list in a key called ``rest``.
+
+    :param fp: a file-like object containing binary data
+    :param format: the format to use - omit this to detect the format
+    :param dialect: the CSV dialect to use - omit this to detect the dialect
+    :param encoding: the character encoding to use when reading CSV/TSV data
+    :param ignore_extras: ignore any extra fields on rows
+    :param extras_key: put any extra fields in a list with this key
+    """
+    if ignore_extras and extras_key:
+        raise ValueError("Cannot use ignore_extras= and extras_key= together")
     if format == Format.JSON:
         decoded = json.load(fp)
         if isinstance(decoded, dict):
@@ -168,18 +355,30 @@ def rows_from_file(
             reader = csv.DictReader(decoded_fp, dialect=dialect)
         else:
             reader = csv.DictReader(decoded_fp)
-        return reader, Format.CSV
+        rows = _extra_key_strategy(reader, ignore_extras, extras_key)
+        return _CloseableIterator(iter(rows), decoded_fp), Format.CSV
     elif format == Format.TSV:
+        rows, _ = rows_from_file(
+            fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding
+        )
         return (
-            rows_from_file(
-                fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding
-            )[0],
+            _extra_key_strategy(
+                cast(Iterable[Dict[Optional[str], object]], rows),
+                ignore_extras,
+                extras_key,
+            ),
             Format.TSV,
         )
     elif format is None:
         # Detect the format, then call this recursively
         buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096)
-        first_bytes = buffered.peek(2048).strip()
+        try:
+            first_bytes = buffered.peek(2048).strip()
+        except AttributeError:
+            # Likely the user passed a TextIO when this needs a BytesIO
+            raise TypeError(
+                "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO"
+            )
         if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"):
             # TODO: Detect newline-JSON
             return rows_from_file(buffered, format=Format.JSON)
@@ -187,18 +386,54 @@ def rows_from_file(
             dialect = csv.Sniffer().sniff(
                 first_bytes.decode(encoding or "utf-8-sig", "ignore")
             )
-            return rows_from_file(
+            rows, _ = rows_from_file(
                 buffered, format=Format.CSV, dialect=dialect, encoding=encoding
             )
+            # Make sure we return the format we detected
+            detected_format = Format.TSV if dialect.delimiter == "\t" else Format.CSV
+            return (
+                _extra_key_strategy(
+                    cast(Iterable[Dict[Optional[str], object]], rows),
+                    ignore_extras,
+                    extras_key,
+                ),
+                detected_format,
+            )
     else:
         raise RowsFromFileError("Bad format")
 
 
 class TypeTracker:
-    def __init__(self):
-        self.trackers = {}
+    """
+    Wrap an iterator of dictionaries and keep track of which SQLite column
+    types are the most likely fit for each of their keys.
 
-    def wrap(self, iterator):
+    Example usage:
+
+    .. code-block:: python
+
+        from sqlite_utils.utils import TypeTracker
+        import sqlite_utils
+
+        db = sqlite_utils.Database(memory=True)
+        tracker = TypeTracker()
+        rows = [{"id": "1", "name": "Cleo", "id": "2", "name": "Cardi"}]
+        db["creatures"].insert_all(tracker.wrap(rows))
+        print(tracker.types)
+        # Outputs {'id': 'integer', 'name': 'text'}
+        db["creatures"].transform(types=tracker.types)
+    """
+
+    def __init__(self) -> None:
+        self.trackers: Dict[str, "ValueTracker"] = {}
+
+    def wrap(self, iterator: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
+        """
+        Use this to loop through an existing iterator, tracking the column types
+        as part of the iteration.
+
+        :param iterator: The iterator to wrap
+        """
         for row in iterator:
             for key, value in row.items():
                 tracker = self.trackers.setdefault(key, ValueTracker())
@@ -206,41 +441,47 @@ class TypeTracker:
             yield row
 
     @property
-    def types(self):
+    def types(self) -> Dict[str, str]:
+        """
+        A dictionary mapping column names to their detected types. This can be passed
+        to the ``db[table_name].transform(types=tracker.types)`` method.
+        """
         return {key: tracker.guessed_type for key, tracker in self.trackers.items()}
 
 
 class ValueTracker:
-    def __init__(self):
+    couldbe: Dict[str, Callable[[object], bool]]
+
+    def __init__(self) -> None:
         self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()}
 
     @classmethod
-    def get_tests(cls):
+    def get_tests(cls) -> List[str]:
         return [
             key.split("test_")[-1]
             for key in cls.__dict__.keys()
             if key.startswith("test_")
         ]
 
-    def test_integer(self, value):
+    def test_integer(self, value: object) -> bool:
         try:
-            int(value)
+            int(cast(Any, value))
             return True
         except (ValueError, TypeError):
             return False
 
-    def test_float(self, value):
+    def test_float(self, value: object) -> bool:
         try:
-            float(value)
+            float(cast(Any, value))
             return True
         except (ValueError, TypeError):
             return False
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return self.guessed_type + ": possibilities = " + repr(self.couldbe)
 
     @property
-    def guessed_type(self):
+    def guessed_type(self) -> str:
         options = set(self.couldbe.keys())
         # Return based on precedence
         for key in self.get_tests():
@@ -248,10 +489,10 @@ class ValueTracker:
                 return key
         return "text"
 
-    def evaluate(self, value):
+    def evaluate(self, value: object) -> None:
         if not value or not self.couldbe:
             return
-        not_these = []
+        not_these: List[str] = []
         for name, test in self.couldbe.items():
             if not test(value):
                 not_these.append(name)
@@ -260,21 +501,162 @@ class ValueTracker:
 
 
 class NullProgressBar:
-    def __init__(self, *args):
+    def __init__(self, *args: Iterable[T]) -> None:
         self.args = args
 
-    def __iter__(self):
-        yield from self.args[0]
+    def __iter__(self) -> Iterator[T]:
+        yield from self.args[0]  # type: ignore
 
-    def update(self, value):
+    def update(self, value: int) -> None:
         pass
 
 
 @contextlib.contextmanager
-def progressbar(*args, **kwargs):
+def progressbar(*args: Iterable[T], **kwargs: Any) -> Generator[Any, None, None]:
     silent = kwargs.pop("silent")
     if silent:
         yield NullProgressBar(*args)
     else:
-        with click.progressbar(*args, **kwargs) as bar:
+        with click.progressbar(*args, **kwargs) as bar:  # type: ignore
             yield bar
+
+
+def _compile_code(
+    code: str, imports: Iterable[str], variable: str = "value"
+) -> Callable[..., Any]:
+    globals_dict: Dict[str, Any] = {"r": recipes, "recipes": recipes}
+    # Handle imports first so they're available for all approaches
+    for import_ in imports:
+        globals_dict[import_.split(".")[0]] = __import__(import_)
+
+    # If user defined a convert() function, return that
+    try:
+        exec(code, globals_dict)
+        return cast(Callable[..., object], globals_dict["convert"])
+    except (AttributeError, SyntaxError, NameError, KeyError, TypeError):
+        pass
+
+    # Check if code is a direct callable reference
+    # e.g. "r.parsedate" instead of "r.parsedate(value)"
+    try:
+        fn = eval(code, globals_dict)
+        if callable(fn):
+            return cast(Callable[..., object], fn)
+    except Exception:
+        pass
+
+    # Try compiling their code as a function instead
+    body_variants = [code]
+    # If single line and no 'return', try adding the return
+    if "\n" not in code and not code.strip().startswith("return "):
+        body_variants.insert(0, "return {}".format(code))
+
+    code_o = None
+    for variant in body_variants:
+        new_code = ["def fn({}):".format(variable)]
+        for line in variant.split("\n"):
+            new_code.append("    {}".format(line))
+        try:
+            code_o = compile("\n".join(new_code), "<string>", "exec")
+            break
+        except SyntaxError:
+            # Try another variant, e.g. for 'return row["column"] = 1'
+            continue
+
+    if code_o is None:
+        raise SyntaxError("Could not compile code")
+
+    exec(code_o, globals_dict)
+    return cast(Callable[..., object], globals_dict["fn"])
+
+
+def chunks(sequence: Iterable[T], size: int) -> Iterable[Iterable[T]]:
+    """
+    Iterate over chunks of the sequence of the given size.
+
+    :param sequence: Any Python iterator
+    :param size: The size of each chunk
+    """
+    iterator = iter(sequence)
+    for item in iterator:
+        yield itertools.chain([item], itertools.islice(iterator, size - 1))
+
+
+def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> str:
+    """
+    ``record`` should be a Python dictionary. Returns a sha1 hash of the
+    keys and values in that record.
+
+    If ``keys=`` is provided, uses just those keys to generate the hash.
+
+    Example usage::
+
+        from sqlite_utils.utils import hash_record
+
+        hashed = hash_record({"name": "Cleo", "twitter": "CleoPaws"})
+        # Or with the keys= option:
+        hashed = hash_record(
+            {"name": "Cleo", "twitter": "CleoPaws", "age": 7},
+            keys=("name", "twitter")
+        )
+
+    :param record: Record to generate a hash for
+    :param keys: Subset of keys to use for that hash
+    """
+    to_hash: Dict[str, Any] = record
+    if keys is not None:
+        to_hash = {key: record[key] for key in keys}
+    return hashlib.sha1(
+        json.dumps(to_hash, separators=(",", ":"), sort_keys=True, default=repr).encode(
+            "utf8"
+        )
+    ).hexdigest()
+
+
+def dedupe_keys(keys: Iterable[str]) -> List[str]:
+    """
+    Rename duplicates in a list of column names so every name is unique,
+    by appending ``_2``, ``_3``... to later occurrences - skipping any
+    suffix that would collide with another column in the list.
+
+    Used when converting SQL query rows to dictionaries, where duplicate
+    column names would otherwise silently overwrite each other.
+
+    :param keys: List of column names, possibly containing duplicates
+    """
+    keys = list(keys)
+    taken = set(keys)
+    if len(taken) == len(keys):
+        # No duplicates - the common case
+        return keys
+    seen: set = set()
+    result = []
+    for key in keys:
+        if key in seen:
+            new_key = key
+            suffix = 2
+            while new_key in seen or new_key in taken:
+                new_key = "{}_{}".format(key, suffix)
+                suffix += 1
+            key = new_key
+        seen.add(key)
+        result.append(key)
+    return result
+
+
+def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]:
+    for key, value in d.items():
+        if isinstance(value, dict):
+            for key2, value2 in _flatten(value):
+                yield key + "_" + key2, value2
+        else:
+            yield key, value
+
+
+def flatten(row: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Turn a nested dict e.g. ``{"a": {"b": 1}}`` into a flat dict: ``{"a_b": 1}``
+
+    :param row: A Python dictionary, optionally with nested dictionaries
+    """
+    return dict(_flatten(row))

tests/conftest.py

@@ -1,6 +1,63 @@
 from sqlite_utils import Database
+from sqlite_utils.utils import sqlite3
 import pytest
 
+CREATE_TABLES = """
+create table Gosh (c1 text, c2 text, c3 text);
+create table Gosh2 (c1 text, c2 text, c3 text);
+"""
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--sqlite-autocommit",
+        action="store_true",
+        default=False,
+        help=(
+            "Run every test against connections created with the Python 3.12+ "
+            "sqlite3.connect(autocommit=True) mode"
+        ),
+    )
+
+
+def pytest_configure(config):
+    import sys
+
+    sys._called_from_test = True  # type: ignore[attr-defined]
+
+    if config.getoption("--sqlite-autocommit"):
+        if sys.version_info < (3, 12):
+            raise pytest.UsageError(
+                "--sqlite-autocommit requires Python 3.12 or higher"
+            )
+        real_connect = sqlite3.connect
+
+        def autocommit_connect(*args, **kwargs):
+            kwargs.setdefault("autocommit", True)
+            return real_connect(*args, **kwargs)
+
+        sqlite3.connect = autocommit_connect
+
+
+@pytest.fixture(autouse=True)
+def close_all_databases():
+    """Automatically close all Database objects created during a test."""
+    databases = []
+    original_init = Database.__init__
+
+    def tracking_init(self, *args, **kwargs):
+        original_init(self, *args, **kwargs)
+        databases.append(self)
+
+    Database.__init__ = tracking_init  # type: ignore[method-assign]
+    yield
+    Database.__init__ = original_init  # type: ignore[method-assign]
+    for db in databases:
+        try:
+            db.close()
+        except Exception:
+            pass
+
 
 @pytest.fixture
 def fresh_db():
@@ -10,12 +67,19 @@ def fresh_db():
 @pytest.fixture
 def existing_db():
     database = Database(memory=True)
-    database.executescript(
-        """
+    database.executescript("""
         CREATE TABLE foo (text TEXT);
         INSERT INTO foo (text) values ("one");
         INSERT INTO foo (text) values ("two");
         INSERT INTO foo (text) values ("three");
-    """
-    )
+    """)
     return database
+
+
+@pytest.fixture
+def db_path(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = sqlite3.connect(path)
+    db.executescript(CREATE_TABLES)
+    db.close()
+    return path

tests/ext.c

@@ -0,0 +1,48 @@
+/*
+** This file implements a SQLite extension with multiple entrypoints.
+**
+** The default entrypoint, sqlite3_ext_init, has a single function "a".
+** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b".
+** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c".
+**
+** Compiling instructions:
+**     https://www.sqlite.org/loadext.html#compiling_a_loadable_extension
+**
+*/
+
+#include "sqlite3ext.h"
+
+SQLITE_EXTENSION_INIT1
+
+// SQL function that returns back the value supplied during sqlite3_create_function()
+static void func(sqlite3_context *context, int argc, sqlite3_value **argv) {
+  sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC);
+}
+
+
+// The default entrypoint, since it matches the "ext.dylib"/"ext.so" name
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0);
+}
+
+// Alternate entrypoint #1
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0);
+}
+
+// Alternate entrypoint #2
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0);
+}

tests/test_analyze.py

@@ -0,0 +1,51 @@
+import pytest
+
+
+@pytest.fixture
+def db(fresh_db):
+    fresh_db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db["one_index"].create_index(["name"])
+    fresh_db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id")
+    fresh_db["two_indexes"].create_index(["name"])
+    fresh_db["two_indexes"].create_index(["species"])
+    return fresh_db
+
+
+def test_analyze_whole_database(db):
+    assert set(db.table_names()) == {"one_index", "two_indexes"}
+    db.analyze()
+    assert set(db.table_names()).issuperset(
+        {"one_index", "two_indexes", "sqlite_stat1"}
+    )
+    assert list(db["sqlite_stat1"].rows) == [
+        {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"},
+        {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"},
+        {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"},
+    ]
+
+
+@pytest.mark.parametrize("method", ("db_method_with_name", "table_method"))
+def test_analyze_one_table(db, method):
+    assert set(db.table_names()).issuperset({"one_index", "two_indexes"})
+    if method == "db_method_with_name":
+        db.analyze("one_index")
+    elif method == "table_method":
+        db["one_index"].analyze()
+
+    assert set(db.table_names()).issuperset(
+        {"one_index", "two_indexes", "sqlite_stat1"}
+    )
+    assert list(db["sqlite_stat1"].rows) == [
+        {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}
+    ]
+
+
+def test_analyze_index_by_name(db):
+    assert set(db.table_names()) == {"one_index", "two_indexes"}
+    db.analyze("idx_two_indexes_species")
+    assert set(db.table_names()).issuperset(
+        {"one_index", "two_indexes", "sqlite_stat1"}
+    )
+    assert list(db["sqlite_stat1"].rows) == [
+        {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"},
+    ]

tests/test_analyze_tables.py

@@ -24,11 +24,35 @@ def db_to_analyze(fresh_db):
     return fresh_db
 
 
+@pytest.fixture
+def big_db_to_analyze_path(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    categories = {
+        "A": 40,
+        "B": 30,
+        "C": 20,
+        "D": 10,
+    }
+    to_insert = []
+    for category, count in categories.items():
+        for _ in range(count):
+            to_insert.append(
+                {
+                    "category": category,
+                    "all_null": None,
+                }
+            )
+    db["stuff"].insert_all(to_insert)
+    return path
+
+
 @pytest.mark.parametrize(
-    "column,expected",
+    "column,extra_kwargs,expected",
     [
         (
             "id",
+            {},
             ColumnDetails(
                 table="stuff",
                 column="id",
@@ -42,6 +66,7 @@ def db_to_analyze(fresh_db):
         ),
         (
             "owner",
+            {},
             ColumnDetails(
                 table="stuff",
                 column="owner",
@@ -55,6 +80,7 @@ def db_to_analyze(fresh_db):
         ),
         (
             "size",
+            {},
             ColumnDetails(
                 table="stuff",
                 column="size",
@@ -66,11 +92,41 @@ def db_to_analyze(fresh_db):
                 least_common=None,
             ),
         ),
+        (
+            "owner",
+            {"most_common": False},
+            ColumnDetails(
+                table="stuff",
+                column="owner",
+                total_rows=8,
+                num_null=0,
+                num_blank=0,
+                num_distinct=4,
+                most_common=None,
+                least_common=[("Anne", 1), ("Terry...", 2)],
+            ),
+        ),
+        (
+            "owner",
+            {"least_common": False},
+            ColumnDetails(
+                table="stuff",
+                column="owner",
+                total_rows=8,
+                num_null=0,
+                num_blank=0,
+                num_distinct=4,
+                most_common=[("Joan", 3), ("Kumar", 2)],
+                least_common=None,
+            ),
+        ),
     ],
 )
-def test_analyze_column(db_to_analyze, column, expected):
+def test_analyze_column(db_to_analyze, column, extra_kwargs, expected):
     assert (
-        db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5)
+        db_to_analyze["stuff"].analyze_column(
+            column, common_limit=2, value_truncate=5, **extra_kwargs
+        )
         == expected
     )
 
@@ -79,16 +135,15 @@ def test_analyze_column(db_to_analyze, column, expected):
 def db_to_analyze_path(db_to_analyze, tmpdir):
     path = str(tmpdir / "test.db")
     db = sqlite3.connect(path)
-    db.executescript("\n".join(db_to_analyze.conn.iterdump()))
+    sql = "\n".join(db_to_analyze.iterdump())
+    db.executescript(sql)
+    db.close()
     return path
 
 
 def test_analyze_table(db_to_analyze_path):
     result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path])
-    assert (
-        result.output.strip()
-        == (
-            """
+    assert result.output.strip() == ("""
 stuff.id: (1/3)
 
   Total rows: 8
@@ -121,9 +176,7 @@ stuff.size: (3/3)
 
   Most common:
     5: 5
-    3: 4"""
-        ).strip()
-    )
+    3: 4""").strip()
 
 
 def test_analyze_table_save(db_to_analyze_path):
@@ -164,3 +217,101 @@ def test_analyze_table_save(db_to_analyze_path):
             "least_common": None,
         },
     ]
+
+
+@pytest.mark.parametrize(
+    "no_most,no_least",
+    (
+        (False, False),
+        (True, False),
+        (False, True),
+        (True, True),
+    ),
+)
+def test_analyze_table_save_no_most_no_least_options(
+    no_most, no_least, big_db_to_analyze_path
+):
+    args = [
+        "analyze-tables",
+        big_db_to_analyze_path,
+        "--save",
+        "--common-limit",
+        "2",
+        "--column",
+        "category",
+    ]
+    if no_most:
+        args.append("--no-most")
+    if no_least:
+        args.append("--no-least")
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 0
+    rows = list(Database(big_db_to_analyze_path)["_analyze_tables_"].rows)
+    expected = {
+        "table": "stuff",
+        "column": "category",
+        "total_rows": 100,
+        "num_null": 0,
+        "num_blank": 0,
+        "num_distinct": 4,
+        "most_common": None,
+        "least_common": None,
+    }
+    if not no_most:
+        expected["most_common"] = '[["A", 40], ["B", 30]]'
+    if not no_least:
+        expected["least_common"] = '[["D", 10], ["C", 20]]'
+
+    assert rows == [expected]
+
+
+def test_analyze_table_column_all_nulls(big_db_to_analyze_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["analyze-tables", big_db_to_analyze_path, "stuff", "--column", "all_null"],
+    )
+    assert result.exit_code == 0
+    assert result.output == (
+        "stuff.all_null: (1/1)\n\n  Total rows: 100\n"
+        "  Null rows: 100\n"
+        "  Blank rows: 0\n"
+        "\n"
+        "  Distinct values: 0\n\n"
+    )
+
+
+@pytest.mark.parametrize(
+    "args,expected_error",
+    (
+        (["-c", "bad_column"], "These columns were not found: bad_column\n"),
+        (["one", "-c", "age"], "These columns were not found: age\n"),
+        (["two", "-c", "age"], None),
+        (
+            ["one", "-c", "age", "--column", "bad"],
+            "These columns were not found: age, bad\n",
+        ),
+    ),
+)
+def test_analyze_table_validate_columns(tmpdir, args, expected_error):
+    path = str(tmpdir / "test_validate_columns.db")
+    db = Database(path)
+    db["one"].insert(
+        {
+            "id": 1,
+            "name": "one",
+        }
+    )
+    db["two"].insert(
+        {
+            "id": 1,
+            "age": 5,
+        }
+    )
+    result = CliRunner().invoke(
+        cli.cli,
+        ["analyze-tables", path] + args,
+        catch_exceptions=False,
+    )
+    assert result.exit_code == (1 if expected_error else 0)
+    if expected_error:
+        assert expected_error in result.output

tests/test_atomic.py

@@ -0,0 +1,382 @@
+import pytest
+
+from sqlite_utils.db import Database, _iter_complete_sql_statements
+from sqlite_utils.utils import sqlite3
+
+
+@pytest.mark.parametrize(
+    "sql,expected",
+    (
+        (
+            "CREATE TABLE t(id); INSERT INTO t VALUES (1)",
+            ["CREATE TABLE t(id);", "INSERT INTO t VALUES (1)"],
+        ),
+        (
+            "INSERT INTO t VALUES ('a;b');",
+            ["INSERT INTO t VALUES ('a;b');"],
+        ),
+        (
+            "-- comment;\nCREATE TABLE t(id);",
+            ["-- comment;\nCREATE TABLE t(id);"],
+        ),
+        (
+            """
+            CREATE TRIGGER t_ai AFTER INSERT ON t
+            BEGIN
+                UPDATE t SET value = 'a;b' WHERE id = new.id;
+                INSERT INTO log VALUES ('x;y');
+            END;
+            """,
+            [
+                "CREATE TRIGGER t_ai AFTER INSERT ON t\n"
+                "            BEGIN\n"
+                "                UPDATE t SET value = 'a;b' WHERE id = new.id;\n"
+                "                INSERT INTO log VALUES ('x;y');\n"
+                "            END;"
+            ],
+        ),
+    ),
+)
+def test_iter_complete_sql_statements(sql, expected):
+    assert list(_iter_complete_sql_statements(sql)) == expected
+
+
+def test_atomic_commits(fresh_db):
+    with fresh_db.atomic():
+        fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+
+    assert list(fresh_db["dogs"].rows) == [{"id": 1, "name": "Cleo"}]
+
+
+def test_atomic_rolls_back(fresh_db):
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+            raise RuntimeError("boom")
+
+    assert not fresh_db["dogs"].exists()
+
+
+def test_nested_atomic_rolls_back_to_savepoint(fresh_db):
+    fresh_db["dogs"].create({"id": int, "name": str}, pk="id")
+
+    with fresh_db.atomic():
+        fresh_db["dogs"].insert({"id": 1, "name": "Cleo"})
+        with pytest.raises(RuntimeError):
+            with fresh_db.atomic():
+                fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"})
+                raise RuntimeError("boom")
+        fresh_db["dogs"].insert({"id": 3, "name": "Marnie"})
+
+    assert list(fresh_db["dogs"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 3, "name": "Marnie"},
+    ]
+
+
+def test_outer_atomic_rolls_back_released_savepoint(fresh_db):
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+            with fresh_db.atomic():
+                fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"})
+            raise RuntimeError("boom")
+
+    assert not fresh_db["dogs"].exists()
+
+
+def test_executescript_does_not_commit_open_atomic_block(fresh_db):
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db.executescript("""
+                CREATE TABLE dogs(id INTEGER PRIMARY KEY, name TEXT);
+                CREATE TRIGGER dogs_ai AFTER INSERT ON dogs
+                BEGIN
+                    UPDATE dogs SET name = upper(new.name) || '; updated' WHERE id = new.id;
+                END;
+                -- This comment has a semicolon;
+                INSERT INTO dogs VALUES (1, 'Cleo; the first');
+            """)
+            raise RuntimeError("boom")
+
+    assert not fresh_db["dogs"].exists()
+
+
+def test_transform_does_not_commit_open_atomic_block(fresh_db):
+    fresh_db["dogs"].insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id")
+
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"})
+            fresh_db["dogs"].transform(rename={"age": "dog_age"})
+            raise RuntimeError("boom")
+
+    assert (
+        fresh_db["dogs"].schema
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n)'
+    )
+    assert list(fresh_db["dogs"].rows) == [
+        {"id": 1, "name": "Cleo", "age": "5"},
+    ]
+
+
+def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db):
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id")
+    fresh_db["books"].insert(
+        {"id": 1, "title": "Book", "author_id": 1},
+        pk="id",
+        foreign_keys={"author_id"},
+    )
+
+    with fresh_db.atomic():
+        fresh_db["authors"].transform(rename={"name": "full_name"})
+        assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+    assert (
+        fresh_db["authors"].schema
+        == 'CREATE TABLE "authors" (\n   "id" INTEGER PRIMARY KEY,\n   "full_name" TEXT\n)'
+    )
+    assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == []
+
+
+def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db):
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id")
+    fresh_db["books"].insert(
+        {"id": 1, "title": "Book", "author_id": 1},
+        pk="id",
+        foreign_keys={"author_id"},
+    )
+
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db["authors"].transform(rename={"name": "full_name"})
+            raise RuntimeError("boom")
+
+    assert (
+        fresh_db["authors"].schema
+        == 'CREATE TABLE "authors" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n)'
+    )
+    assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+    assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == []
+
+
+def test_transform_detects_foreign_key_check_violations(fresh_db):
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 2}, pk="id")
+
+    with pytest.raises(sqlite3.IntegrityError):
+        fresh_db["books"].transform(add_foreign_keys=(("author_id", "authors", "id"),))
+
+    assert fresh_db["books"].foreign_keys == []
+    assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db):
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.execute("begin")
+    with fresh_db.atomic():
+        fresh_db["t"].insert({"id": 2}, pk="id")
+    # Nothing is committed until the user's own transaction commits
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1]
+    # And with a commit instead, the atomic block's writes persist
+    fresh_db.execute("begin")
+    with fresh_db.atomic():
+        fresh_db["t"].insert({"id": 3}, pk="id")
+    fresh_db.commit()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1, 3]
+
+
+def test_begin_commit_rollback(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["t"].insert({"id": 1}, pk="id")
+    db.begin()
+    db["t"].insert({"id": 2}, pk="id")
+    assert db.conn.in_transaction
+    db.rollback()
+    assert not db.conn.in_transaction
+    assert [r["id"] for r in db["t"].rows] == [1]
+    db.begin()
+    db["t"].insert({"id": 3}, pk="id")
+    db.commit()
+    db.close()
+    db2 = Database(path)
+    assert [r["id"] for r in db2["t"].rows] == [1, 3]
+    db2.close()
+
+
+def test_begin_inside_transaction_errors(fresh_db):
+    fresh_db.begin()
+    with pytest.raises(sqlite3.OperationalError):
+        fresh_db.begin()
+    fresh_db.rollback()
+
+
+def test_commit_and_rollback_without_transaction_are_noops(fresh_db):
+    fresh_db.commit()
+    fresh_db.rollback()
+    assert not fresh_db.conn.in_transaction
+
+
+def test_execute_write_commits_immediately(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["t"].insert({"id": 1}, pk="id")
+    db.execute("insert into t (id) values (2)")
+    # No implicit transaction is left open
+    assert not db.conn.in_transaction
+    # A completely separate connection sees the row straight away
+    other = sqlite3.connect(path)
+    assert other.execute("select count(*) from t").fetchone()[0] == 2
+    other.close()
+    db.close()
+
+
+def test_execute_write_respects_explicit_transaction(fresh_db):
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.begin()
+    fresh_db.execute("insert into t (id) values (2)")
+    # Still inside the explicit transaction - not committed
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1]
+
+
+def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db):
+    # A BEGIN hidden behind a leading comment must not be auto-committed
+    # out from under the caller
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.execute("-- start a transaction\nbegin")
+    assert fresh_db.conn.in_transaction
+    fresh_db.execute("insert into t (id) values (2)")
+    fresh_db.rollback()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1]
+
+
+def _sqlite_accepts_bom():
+    try:
+        sqlite3.connect(":memory:").execute("\ufeffselect 1")
+        return True
+    except sqlite3.OperationalError:
+        return False
+
+
+@pytest.mark.parametrize("begin_sql", ["; begin", "\ufeffbegin"])
+def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql):
+    # sqlite3 tolerates empty statements and a UTF-8 BOM before the first
+    # real token, so a BEGIN behind either must not be auto-committed
+    # out from under the caller
+    if begin_sql.startswith("\ufeff") and not _sqlite_accepts_bom():
+        pytest.skip("This SQLite version rejects a leading byte order mark")
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.execute(begin_sql)
+    assert fresh_db.conn.in_transaction
+    fresh_db.execute("insert into t (id) values (2)")
+    fresh_db.rollback()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1]
+
+
+def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir):
+    # A failed write must not leave the driver's implicit transaction open -
+    # that would silently disable auto-commit for every subsequent write
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["t"].insert({"id": 1}, pk="id")
+    with pytest.raises(sqlite3.IntegrityError):
+        db.execute("insert into t (id) values (1)")
+    assert not db.conn.in_transaction
+    # Subsequent writes commit as normal and survive closing the connection
+    db["other"].insert({"id": 2})
+    db.close()
+    db2 = Database(path)
+    assert db2["other"].exists()
+    db2.close()
+
+
+def test_execute_failed_write_preserves_explicit_transaction(fresh_db):
+    # A failed write inside an explicit transaction must not roll back
+    # the caller's earlier work - only the caller decides that
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.begin()
+    fresh_db.execute("insert into t (id) values (2)")
+    with pytest.raises(sqlite3.IntegrityError):
+        fresh_db.execute("insert into t (id) values (1)")
+    assert fresh_db.conn.in_transaction
+    fresh_db.commit()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1, 2]
+
+
+def test_execute_failed_write_inside_atomic_preserves_block(fresh_db):
+    # A caught failure inside an atomic() block must leave the block's
+    # transaction open so its other work still commits
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    with fresh_db.atomic():
+        fresh_db.execute("insert into t (id) values (2)")
+        with pytest.raises(sqlite3.IntegrityError):
+            fresh_db.execute("insert into t (id) values (1)")
+    assert [r["id"] for r in fresh_db["t"].rows] == [1, 2]
+
+
+def test_query_returning_commits_after_iteration(tmpdir):
+    if sqlite3.sqlite_version_info < (3, 35, 0):
+        import pytest as _pytest
+
+        _pytest.skip("RETURNING requires SQLite 3.35.0 or higher")
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["t"].insert({"id": 1}, pk="id")
+    rows = list(db.query("insert into t (id) values (2) returning id"))
+    assert rows == [{"id": 2}]
+    assert not db.conn.in_transaction
+    other = sqlite3.connect(path)
+    assert other.execute("select count(*) from t").fetchone()[0] == 2
+    other.close()
+    db.close()
+
+
+TRIGGER_SQL = """
+create trigger no_bad before insert on t
+when new.v = 'bad'
+begin
+    select raise(rollback, 'trigger says no');
+end
+"""
+
+
+def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db):
+    # RAISE(ROLLBACK) rolls back the whole transaction and destroys every
+    # savepoint - atomic()'s cleanup must not mask the IntegrityError
+    # with "cannot rollback - no transaction is active"
+    fresh_db.execute("create table t (id integer primary key, v text)")
+    fresh_db.execute(TRIGGER_SQL)
+    with pytest.raises(sqlite3.IntegrityError, match="trigger says no"):
+        with fresh_db.atomic():
+            fresh_db.execute("insert into t (v) values ('bad')")
+    assert not fresh_db.conn.in_transaction
+
+
+def test_nested_atomic_preserves_error_from_transaction_destroying_trigger(
+    fresh_db,
+):
+    # The nested savepoint branch previously raised
+    # "no such savepoint" from ROLLBACK TO SAVEPOINT
+    fresh_db.execute("create table t (id integer primary key, v text)")
+    fresh_db.execute(TRIGGER_SQL)
+    with pytest.raises(sqlite3.IntegrityError, match="trigger says no"):
+        with fresh_db.atomic():
+            with fresh_db.atomic():
+                fresh_db.execute("insert into t (v) values ('bad')")
+    assert not fresh_db.conn.in_transaction
+
+
+def test_atomic_preserves_error_from_insert_or_rollback(fresh_db):
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    with pytest.raises(sqlite3.IntegrityError):
+        with fresh_db.atomic():
+            fresh_db.execute("insert or rollback into t (id) values (1)")
+    assert not fresh_db.conn.in_transaction

tests/test_cli_bulk.py

@@ -0,0 +1,123 @@
+from click.testing import CliRunner
+from sqlite_utils import cli, Database
+import pathlib
+import pytest
+import subprocess
+import sys
+import time
+
+
+@pytest.fixture
+def test_db_and_path(tmpdir):
+    db_path = str(pathlib.Path(tmpdir) / "data.db")
+    db = Database(db_path)
+    db["example"].insert_all(
+        [
+            {"id": 1, "name": "One"},
+            {"id": 2, "name": "Two"},
+        ],
+        pk="id",
+    )
+    return db, db_path
+
+
+def test_cli_bulk(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "bulk",
+            db_path,
+            "insert into example (id, name) values (:id, myupper(:name))",
+            "-",
+            "--nl",
+            "--functions",
+            "myupper = lambda s: s.upper()",
+        ],
+        input='{"id": 3, "name": "Three"}\n{"id": 4, "name": "Four"}\n',
+    )
+    assert result.exit_code == 0, result.output
+    assert [
+        {"id": 1, "name": "One"},
+        {"id": 2, "name": "Two"},
+        {"id": 3, "name": "THREE"},
+        {"id": 4, "name": "FOUR"},
+    ] == list(db["example"].rows)
+
+
+def test_cli_bulk_multiple_functions(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "bulk",
+            db_path,
+            "insert into example (id, name) values (:id, myupper(mylower(:name)))",
+            "-",
+            "--nl",
+            "--functions",
+            "myupper = lambda s: s.upper()",
+            "--functions",
+            "mylower = lambda s: s.lower()",
+        ],
+        input='{"id": 3, "name": "ThReE"}\n{"id": 4, "name": "FoUr"}\n',
+    )
+    assert result.exit_code == 0, result.output
+    assert [
+        {"id": 1, "name": "One"},
+        {"id": 2, "name": "Two"},
+        {"id": 3, "name": "THREE"},
+        {"id": 4, "name": "FOUR"},
+    ] == list(db["example"].rows)
+
+
+def test_cli_bulk_batch_size(test_db_and_path):
+    db, db_path = test_db_and_path
+    proc = subprocess.Popen(
+        [
+            sys.executable,
+            "-m",
+            "sqlite_utils",
+            "bulk",
+            db_path,
+            "insert into example (id, name) values (:id, :name)",
+            "-",
+            "--nl",
+            "--batch-size",
+            "2",
+        ],
+        stdin=subprocess.PIPE,
+        stdout=sys.stdout,
+    )
+    # Writing one record should not commit
+    proc.stdin.write(b'{"id": 3, "name": "Three"}\n\n')
+    proc.stdin.flush()
+    time.sleep(1)
+    assert db["example"].count == 2
+
+    # Writing another should trigger a commit:
+    proc.stdin.write(b'{"id": 4, "name": "Four"}\n\n')
+    proc.stdin.flush()
+    time.sleep(1)
+    assert db["example"].count == 4
+
+    proc.stdin.close()
+    proc.wait()
+    assert proc.returncode == 0
+
+
+def test_cli_bulk_error(test_db_and_path):
+    _, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "bulk",
+            db_path,
+            "insert into example (id, name) value (:id, :name)",
+            "-",
+            "--nl",
+        ],
+        input='{"id": 3, "name": "Three"}',
+    )
+    assert result.exit_code == 1
+    assert result.output == 'Error: near "value": syntax error\n'

tests/test_cli_convert.py

@@ -35,39 +35,40 @@ def fresh_db_and_path(tmpdir):
         "return value.replace('October', 'Spooktober')",
         # Return is optional:
         "value.replace('October', 'Spooktober')",
+        # Multiple lines are supported:
+        "v = value.replace('October', 'Spooktober')\nreturn v",
+        # Can also define a convert() function
+        "def convert(value): return value.replace('October', 'Spooktober')",
+        # ... with imports
+        "import re\n\ndef convert(value): return value.replace('October', 'Spooktober')",
     ],
 )
-def test_convert_single_line(test_db_and_path, code):
-    db, db_path = test_db_and_path
-    result = CliRunner().invoke(cli.cli, ["convert", db_path, "example", "dt", code])
-    assert 0 == result.exit_code, result.output
-    assert [
-        {"id": 1, "dt": "5th Spooktober 2019 12:04"},
-        {"id": 2, "dt": "6th Spooktober 2019 00:05:06"},
-        {"id": 3, "dt": ""},
-        {"id": 4, "dt": None},
-    ] == list(db["example"].rows)
-
-
-def test_convert_multiple_lines(test_db_and_path):
-    db, db_path = test_db_and_path
+def test_convert_code(fresh_db_and_path, code):
+    db, db_path = fresh_db_and_path
+    db["t"].insert({"text": "October"})
     result = CliRunner().invoke(
-        cli.cli,
-        [
-            "convert",
-            db_path,
-            "example",
-            "dt",
-            "v = value.replace('October', 'Spooktober')\nreturn v.upper()",
-        ],
+        cli.cli, ["convert", db_path, "t", "text", code], catch_exceptions=False
     )
-    assert 0 == result.exit_code, result.output
-    assert [
-        {"id": 1, "dt": "5TH SPOOKTOBER 2019 12:04"},
-        {"id": 2, "dt": "6TH SPOOKTOBER 2019 00:05:06"},
-        {"id": 3, "dt": ""},
-        {"id": 4, "dt": None},
-    ] == list(db["example"].rows)
+    assert result.exit_code == 0, result.output
+    value = list(db["t"].rows)[0]["text"]
+    assert value == "Spooktober"
+
+
+@pytest.mark.parametrize(
+    "bad_code",
+    (
+        "def foo(value)",
+        "$",
+    ),
+)
+def test_convert_code_errors(fresh_db_and_path, bad_code):
+    db, db_path = fresh_db_and_path
+    db["t"].insert({"text": "October"})
+    result = CliRunner().invoke(
+        cli.cli, ["convert", db_path, "t", "text", bad_code], catch_exceptions=False
+    )
+    assert result.exit_code == 1
+    assert result.output == "Error: Could not compile code\n"
 
 
 def test_convert_import(test_db_and_path):
@@ -79,12 +80,12 @@ def test_convert_import(test_db_and_path):
             db_path,
             "example",
             "dt",
-            "return re.sub('O..', 'OXX', value)",
+            "return re.sub('O..', 'OXX', value) if value else value",
             "--import",
             "re",
         ],
     )
-    assert 0 == result.exit_code, result.output
+    assert result.exit_code == 0, result.output
     assert [
         {"id": 1, "dt": "5th OXXober 2019 12:04"},
         {"id": 2, "dt": "6th OXXober 2019 00:05:06"},
@@ -93,6 +94,27 @@ def test_convert_import(test_db_and_path):
     ] == list(db["example"].rows)
 
 
+def test_convert_import_nested(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["example"].insert({"xml": '<item name="Cleo" />'})
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "xml",
+            'xml.etree.ElementTree.fromstring(value).attrib["name"]',
+            "--import",
+            "xml.etree.ElementTree",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert [
+        {"xml": "Cleo"},
+    ] == list(db["example"].rows)
+
+
 def test_convert_dryrun(test_db_and_path):
     db, db_path = test_db_and_path
     result = CliRunner().invoke(
@@ -157,6 +179,61 @@ def test_convert_dryrun(test_db_and_path):
     assert result.output.strip().split("\n")[-1] == "Would affect 1 row"
 
 
+def test_convert_multi_dryrun(test_db_and_path):
+    db_path = test_db_and_path[1]
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "{'foo': 'bar', 'baz': 1}",
+            "--dry-run",
+            "--multi",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == (
+        "5th October 2019 12:04\n"
+        " --- becomes:\n"
+        '{"foo": "bar", "baz": 1}\n'
+        "\n"
+        "6th October 2019 00:05:06\n"
+        " --- becomes:\n"
+        '{"foo": "bar", "baz": 1}\n'
+        "\n"
+        "\n"
+        " --- becomes:\n"
+        "\n"
+        "\n"
+        "None\n"
+        " --- becomes:\n"
+        "None\n"
+        "\n"
+        "Would affect 4 rows"
+    )
+
+
+def test_convert_multi_dryrun_unicode_not_escaped(test_db_and_path):
+    db_path = test_db_and_path[1]
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "{'text': 'Japanese 日本語'}",
+            "--dry-run",
+            "--multi",
+        ],
+    )
+    assert result.exit_code == 0
+    # Preview should match what jsonify_if_needed() would actually store
+    assert '{"text": "Japanese 日本語"}' in result.output
+
+
 @pytest.mark.parametrize("drop", (True, False))
 def test_convert_output_column(test_db_and_path, drop):
     db, db_path = test_db_and_path
@@ -165,14 +242,14 @@ def test_convert_output_column(test_db_and_path, drop):
         db_path,
         "example",
         "dt",
-        "value.replace('October', 'Spooktober')",
+        "value.replace('October', 'Spooktober') if value else value",
         "--output",
         "newcol",
     ]
     if drop:
         args += ["--drop"]
     result = CliRunner().invoke(cli.cli, args)
-    assert 0 == result.exit_code, result.output
+    assert result.exit_code == 0, result.output
     expected = [
         {
             "id": 1,
@@ -219,7 +296,7 @@ def test_convert_output_column_output_type(test_db_and_path, output_type, expect
         cli.cli,
         args,
     )
-    assert 0 == result.exit_code, result.output
+    assert result.exit_code == 0, result.output
     assert expected == list(db.execute("select id, new_id from example"))
 
 
@@ -313,16 +390,14 @@ def test_convert_multi_complex_column_types(fresh_db_and_path):
         ],
         pk="id",
     )
-    code = textwrap.dedent(
-        """
+    code = textwrap.dedent("""
     if value == 1:
         return {"is_str": "", "is_float": 1.2, "is_int": None}
     elif value == 2:
         return {"is_float": 1, "is_int": 12}
     elif value == 3:
         return {"is_bytes": b"blah"}
-    """
-    )
+    """)
     result = CliRunner().invoke(
         cli.cli,
         [
@@ -348,9 +423,9 @@ def test_convert_multi_complex_column_types(fresh_db_and_path):
         {"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None},
     ]
     assert db["rows"].schema == (
-        "CREATE TABLE [rows] (\n"
-        "   [id] INTEGER PRIMARY KEY\n"
-        ", [is_str] TEXT, [is_float] FLOAT, [is_int] INTEGER, [is_bytes] BLOB)"
+        'CREATE TABLE "rows" (\n'
+        '   "id" INTEGER PRIMARY KEY\n'
+        ', "is_str" TEXT, "is_float" REAL, "is_int" INTEGER, "is_bytes" BLOB)'
     )
 
 
@@ -370,7 +445,7 @@ def test_recipe_jsonsplit(tmpdir, delimiter):
         code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter)
     args = ["convert", db_path, "example", "tags", code]
     result = CliRunner().invoke(cli.cli, args)
-    assert 0 == result.exit_code, result.output
+    assert result.exit_code == 0, result.output
     assert list(db["example"].rows) == [
         {"id": 1, "tags": '["foo", "bar"]'},
         {"id": 2, "tags": '["bar", "baz"]'},
@@ -398,7 +473,7 @@ def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array):
         code = "recipes.jsonsplit(value, type={})".format(type)
     args = ["convert", db_path, "example", "records", code]
     result = CliRunner().invoke(cli.cli, args)
-    assert 0 == result.exit_code, result.output
+    assert result.exit_code == 0, result.output
     assert json.loads(db["example"].get(1)["records"]) == expected_array
 
 
@@ -416,7 +491,7 @@ def test_recipe_jsonsplit_output(fresh_db_and_path, drop):
     if drop:
         args += ["--drop"]
     result = CliRunner().invoke(cli.cli, args)
-    assert 0 == result.exit_code, result.output
+    assert result.exit_code == 0, result.output
     expected = {
         "id": 1,
         "records": "1,2,3",
@@ -477,7 +552,7 @@ def test_convert_where(test_db_and_path):
             "id = :id",
             "-p",
             "id",
-            2,
+            "2",
         ],
     )
     assert result.exit_code == 0, result.output
@@ -506,12 +581,132 @@ def test_convert_where_multi(fresh_db_and_path):
             "id = :id",
             "-p",
             "id",
-            2,
+            "2",
             "--multi",
         ],
     )
-    assert 0 == result.exit_code, result.output
+    assert result.exit_code == 0, result.output
     assert list(db["names"].rows) == [
         {"id": 1, "name": "Cleo", "upper": None},
         {"id": 2, "name": "Bants", "upper": "BANTS"},
     ]
+
+
+def test_convert_code_standard_input(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "names",
+            "name",
+            "-",
+        ],
+        input="value.upper()",
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["names"].rows) == [
+        {"id": 1, "name": "CLEO"},
+    ]
+
+
+def test_convert_hyphen_workaround(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["convert", db_path, "names", "name", '"-"'],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["names"].rows) == [
+        {"id": 1, "name": "-"},
+    ]
+
+
+def test_convert_initialization_pattern(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "names",
+            "name",
+            "-",
+        ],
+        input="import random\nrandom.seed(1)\ndef convert(value):    return random.randint(0, 100)",
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["names"].rows) == [
+        {"id": 1, "name": "17"},
+    ]
+
+
+def test_convert_handles_falsey_values(fresh_db_and_path):
+    # Falsey values like 0 should be converted (issue #527)
+    db, db_path = fresh_db_and_path
+    args = [
+        "convert",
+        db_path,
+        "t",
+        "x",
+        "-",
+    ]
+    db["t"].insert_all([{"x": 0}, {"x": 1}])
+    assert db["t"].get(1)["x"] == 0
+    assert db["t"].get(2)["x"] == 1
+    result = CliRunner().invoke(cli.cli, args, input="value + 1")
+    assert result.exit_code == 0, result.output
+    assert db["t"].get(1)["x"] == 1
+    assert db["t"].get(2)["x"] == 2
+
+
+@pytest.mark.parametrize(
+    "code",
+    [
+        # Direct callable reference (issue #686)
+        "r.parsedate",
+        "recipes.parsedate",
+        # Traditional call syntax still works
+        "r.parsedate(value)",
+        "recipes.parsedate(value)",
+    ],
+)
+def test_convert_callable_reference(test_db_and_path, code):
+    """Test that callable references like r.parsedate work without (value)"""
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli, ["convert", db_path, "example", "dt", code], catch_exceptions=False
+    )
+    assert result.exit_code == 0, result.output
+    rows = list(db["example"].rows)
+    assert rows[0]["dt"] == "2019-10-05"
+    assert rows[1]["dt"] == "2019-10-06"
+    assert rows[2]["dt"] == ""
+    assert rows[3]["dt"] is None
+
+
+def test_convert_callable_reference_with_import(fresh_db_and_path):
+    """Test callable reference from an imported module"""
+    db, db_path = fresh_db_and_path
+    db["example"].insert({"id": 1, "data": '{"name": "test"}'})
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "data",
+            "json.loads",
+            "--import",
+            "json",
+        ],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0, result.output
+    # json.loads returns a dict, which sqlite stores as JSON string
+    row = db["example"].get(1)
+    assert row["data"] == '{"name": "test"}'

tests/test_cli_insert.py

@@ -0,0 +1,890 @@
+from sqlite_utils import cli, Database
+from click.testing import CliRunner
+import json
+import pytest
+import subprocess
+import sys
+import time
+
+
+def test_insert_simple(tmpdir):
+    json_path = str(tmpdir / "dog.json")
+    db_path = str(tmpdir / "dogs.db")
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps({"name": "Cleo", "age": 4}))
+    result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path])
+    assert result.exit_code == 0
+    assert [{"age": 4, "name": "Cleo"}] == list(
+        Database(db_path).query("select * from dogs")
+    )
+    db = Database(db_path)
+    assert ["dogs"] == db.table_names()
+    assert [] == db["dogs"].indexes
+
+
+def test_insert_from_stdin(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", "-"],
+        input=json.dumps({"name": "Cleo", "age": 4}),
+    )
+    assert result.exit_code == 0
+    assert [{"age": 4, "name": "Cleo"}] == list(
+        Database(db_path).query("select * from dogs")
+    )
+
+
+def test_insert_invalid_json_error(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", "-"],
+        input="name,age\nCleo,4",
+    )
+    assert result.exit_code == 1
+    assert result.output == (
+        "Error: Invalid JSON - use --csv for CSV or --tsv for TSV files\n\n"
+        "JSON error: Expecting value: line 1 column 1 (char 0)\n"
+    )
+
+
+def test_insert_json_flatten(tmpdir):
+    db_path = str(tmpdir / "flat.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "items", "-", "--flatten"],
+        input=json.dumps({"nested": {"data": 4}}),
+    )
+    assert result.exit_code == 0
+    assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}]
+
+
+def test_insert_json_flatten_nl(tmpdir):
+    db_path = str(tmpdir / "flat.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "items", "-", "--flatten", "--nl"],
+        input="\n".join(
+            json.dumps(item)
+            for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}]
+        ),
+    )
+    assert result.exit_code == 0
+    assert list(Database(db_path).query("select * from items")) == [
+        {"nested_data": 4, "nested_other": None},
+        {"nested_data": None, "nested_other": 3},
+    ]
+
+
+@pytest.mark.parametrize(
+    "args,expected_pks",
+    (
+        (["--pk", "id"], ["id"]),
+        (["--pk", "id", "--pk", "name"], ["id", "name"]),
+    ),
+)
+def test_insert_with_primary_keys(db_path, tmpdir, args, expected_pks):
+    json_path = str(tmpdir / "dog.json")
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps({"id": 1, "name": "Cleo", "age": 4}))
+    result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path] + args)
+    assert result.exit_code == 0
+    assert [{"id": 1, "age": 4, "name": "Cleo"}] == list(
+        Database(db_path).query("select * from dogs")
+    )
+    db = Database(db_path)
+    assert db["dogs"].pks == expected_pks
+
+
+def test_insert_multiple_with_primary_key(db_path, tmpdir):
+    json_path = str(tmpdir / "dogs.json")
+    dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    assert dogs == list(db.query("select * from dogs order by id"))
+    assert ["id"] == db["dogs"].pks
+
+
+def test_insert_multiple_with_compound_primary_key(db_path, tmpdir):
+    json_path = str(tmpdir / "dogs.json")
+    dogs = [
+        {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3}
+        for i in range(1, 21)
+    ]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--pk", "breed"]
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    assert dogs == list(db.query("select * from dogs order by breed, id"))
+    assert {"breed", "id"} == set(db["dogs"].pks)
+    assert (
+        'CREATE TABLE "dogs" (\n'
+        '   "breed" TEXT,\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT,\n'
+        '   "age" INTEGER,\n'
+        '   PRIMARY KEY ("id", "breed")\n'
+        ")"
+    ) == db["dogs"].schema
+
+
+def test_insert_not_null_default(db_path, tmpdir):
+    json_path = str(tmpdir / "dogs.json")
+    dogs = [
+        {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10}
+        for i in range(1, 21)
+    ]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(dogs))
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", json_path, "--pk", "id"]
+        + ["--not-null", "name", "--not-null", "age"]
+        + ["--default", "score", "5", "--default", "age", "1"],
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    assert (
+        'CREATE TABLE "dogs" (\n'
+        '   "id" INTEGER PRIMARY KEY,\n'
+        '   "name" TEXT NOT NULL,\n'
+        "   \"age\" INTEGER NOT NULL DEFAULT '1',\n"
+        "   \"score\" INTEGER DEFAULT '5'\n)"
+    ) == db["dogs"].schema
+
+
+def test_insert_binary_base64(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "files", "-"],
+        input=r'{"content": {"$base64": true, "encoded": "aGVsbG8="}}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    actual = list(db.query("select content from files"))
+    assert actual == [{"content": b"hello"}]
+
+
+def test_insert_newline_delimited(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl"],
+        input='{"foo": "bar", "n": 1}\n\n{"foo": "baz", "n": 2}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert [
+        {"foo": "bar", "n": 1},
+        {"foo": "baz", "n": 2},
+    ] == list(db.query("select foo, n from from_json_nl"))
+
+
+def test_insert_ignore(db_path, tmpdir):
+    db = Database(db_path)
+    db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+    json_path = str(tmpdir / "dogs.json")
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps([{"id": 1, "name": "Bailey"}]))
+    # Should raise error without --ignore
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
+    )
+    assert result.exit_code != 0, result.output
+    # If we use --ignore it should run OK
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--ignore"]
+    )
+    assert result.exit_code == 0, result.output
+    # ... but it should actually have no effect
+    assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs"))
+
+
+@pytest.mark.parametrize(
+    "content,options",
+    [
+        ("foo\tbar\tbaz\n1\t2\tcat,dog", ["--tsv"]),
+        ('foo,bar,baz\n1,2,"cat,dog"', ["--csv"]),
+        ('foo;bar;baz\n1;2;"cat,dog"', ["--csv", "--delimiter", ";"]),
+        # --delimiter implies --csv:
+        ('foo;bar;baz\n1;2;"cat,dog"', ["--delimiter", ";"]),
+        ("foo,bar,baz\n1,2,|cat,dog|", ["--csv", "--quotechar", "|"]),
+        ("foo,bar,baz\n1,2,|cat,dog|", ["--quotechar", "|"]),
+    ],
+)
+def test_insert_csv_tsv(content, options, db_path, tmpdir):
+    db = Database(db_path)
+    file_path = str(tmpdir / "insert.csv-tsv")
+    with open(file_path, "w") as fp:
+        fp.write(content)
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", file_path] + options + ["--no-detect-types"],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows)
+
+
+@pytest.mark.parametrize("empty_null", (True, False))
+def test_insert_csv_empty_null(db_path, empty_null):
+    options = ["--csv", "--no-detect-types"]
+    if empty_null:
+        options.append("--empty-null")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", "-"] + options,
+        catch_exceptions=False,
+        input="foo,bar,baz\n1,,cat,dog",
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    assert [r for r in db["data"].rows] == [
+        {"foo": "1", "bar": None if empty_null else "", "baz": "cat"}
+    ]
+
+
+@pytest.mark.parametrize(
+    "input,args",
+    (
+        (
+            json.dumps(
+                [{"name": "One"}, {"name": "Two"}, {"name": "Three"}, {"name": "Four"}]
+            ),
+            [],
+        ),
+        ("name\nOne\nTwo\nThree\nFour\n", ["--csv"]),
+    ),
+)
+def test_insert_stop_after(tmpdir, input, args):
+    db_path = str(tmpdir / "data.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "rows", "-", "--stop-after", "2"] + args,
+        input=input,
+    )
+    assert result.exit_code == 0
+    assert [{"name": "One"}, {"name": "Two"}] == list(
+        Database(db_path).query("select * from rows")
+    )
+
+
+@pytest.mark.parametrize(
+    "options",
+    (
+        ["--tsv", "--nl"],
+        ["--tsv", "--csv"],
+        ["--csv", "--nl"],
+        ["--csv", "--nl", "--tsv"],
+    ),
+)
+def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir):
+    file_path = str(tmpdir / "insert.csv-tsv")
+    with open(file_path, "w") as fp:
+        fp.write("foo")
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "data", file_path] + options
+    )
+    assert result.exit_code != 0
+    assert "Error: Use just one of --nl, --csv or --tsv" == result.output.strip()
+
+
+def test_insert_replace(db_path, tmpdir):
+    test_insert_multiple_with_primary_key(db_path, tmpdir)
+    json_path = str(tmpdir / "insert-replace.json")
+    db = Database(db_path)
+    assert db["dogs"].count == 20
+    insert_replace_dogs = [
+        {"id": 1, "name": "Insert replaced 1", "age": 4},
+        {"id": 2, "name": "Insert replaced 2", "age": 4},
+        {"id": 21, "name": "Fresh insert 21", "age": 6},
+    ]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(insert_replace_dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"]
+    )
+    assert result.exit_code == 0, result.output
+    assert db["dogs"].count == 21
+    assert (
+        list(db.query("select * from dogs where id in (1, 2, 21) order by id"))
+        == insert_replace_dogs
+    )
+
+
+def test_insert_truncate(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl", "--batch-size=1"],
+        input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert [
+        {"foo": "bar", "n": 1},
+        {"foo": "baz", "n": 2},
+    ] == list(db.query("select foo, n from from_json_nl"))
+    # Truncate and insert new rows
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "from_json_nl",
+            "-",
+            "--nl",
+            "--truncate",
+            "--batch-size=1",
+        ],
+        input='{"foo": "bam", "n": 3}\n{"foo": "bat", "n": 4}',
+    )
+    assert result.exit_code == 0, result.output
+    assert [
+        {"foo": "bam", "n": 3},
+        {"foo": "bat", "n": 4},
+    ] == list(db.query("select foo, n from from_json_nl"))
+
+
+def test_insert_alter(db_path, tmpdir):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl"],
+        input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}',
+    )
+    assert result.exit_code == 0, result.output
+    # Should get an error with incorrect shaped additional data
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl"],
+        input='{"foo": "bar", "baz": 5}',
+    )
+    assert result.exit_code != 0, result.output
+    # If we run it again with --alter it should work correctly
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl", "--alter"],
+        input='{"foo": "bar", "baz": 5}',
+    )
+    assert result.exit_code == 0, result.output
+    # Soundness check the database itself
+    db = Database(db_path)
+    assert {"foo": str, "n": int, "baz": int} == db["from_json_nl"].columns_dict
+    assert [
+        {"foo": "bar", "n": 1, "baz": None},
+        {"foo": "baz", "n": 2, "baz": None},
+        {"foo": "bar", "baz": 5, "n": None},
+    ] == list(db.query("select foo, n, baz from from_json_nl"))
+
+
+def test_insert_analyze(db_path):
+    db = Database(db_path)
+    db["rows"].insert({"foo": "x", "n": 3})
+    db["rows"].create_index(["n"])
+    assert "sqlite_stat1" not in db.table_names()
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "rows", "-", "--nl", "--analyze"],
+        input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}',
+    )
+    assert result.exit_code == 0, result.output
+    assert "sqlite_stat1" in db.table_names()
+
+
+def test_insert_lines(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_lines", "-", "--lines"],
+        input='First line\nSecond line\n{"foo": "baz"}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert [
+        {"line": "First line"},
+        {"line": "Second line"},
+        {"line": '{"foo": "baz"}'},
+    ] == list(db.query("select line from from_lines"))
+
+
+def test_insert_text(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_text", "-", "--text"],
+        input='First line\nSecond line\n{"foo": "baz"}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert [{"text": 'First line\nSecond line\n{"foo": "baz"}'}] == list(
+        db.query("select text from from_text")
+    )
+
+
+@pytest.mark.parametrize(
+    "options,input",
+    (
+        ([], '[{"id": "1", "name": "Bob"}, {"id": "2", "name": "Cat"}]'),
+        (["--csv", "--no-detect-types"], "id,name\n1,Bob\n2,Cat"),
+        (["--nl"], '{"id": "1", "name": "Bob"}\n{"id": "2", "name": "Cat"}'),
+    ),
+)
+def test_insert_convert_json_csv_jsonnl(db_path, options, input):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "rows", "-", "--convert", '{**row, **{"extra": 1}}']
+        + options,
+        input=input,
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query("select id, name, extra from rows"))
+    assert rows == [
+        {"id": "1", "name": "Bob", "extra": 1},
+        {"id": "2", "name": "Cat", "extra": 1},
+    ]
+
+
+def test_insert_convert_text(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "text",
+            "-",
+            "--text",
+            "--convert",
+            '{"text": text.upper()}',
+        ],
+        input="This is text\nwill be upper now",
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query('select "text" from "text"'))
+    assert rows == [{"text": "THIS IS TEXT\nWILL BE UPPER NOW"}]
+
+
+def test_insert_convert_text_returning_iterator(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "text",
+            "-",
+            "--text",
+            "--convert",
+            '({"word": w} for w in text.split())',
+        ],
+        input="A bunch of words",
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query('select "word" from "text"'))
+    assert rows == [{"word": "A"}, {"word": "bunch"}, {"word": "of"}, {"word": "words"}]
+
+
+def test_insert_convert_lines(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "all",
+            "-",
+            "--lines",
+            "--convert",
+            '{"line": line.upper()}',
+        ],
+        input="This is text\nwill be upper now",
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query('select "line" from "all"'))
+    assert rows == [{"line": "THIS IS TEXT"}, {"line": "WILL BE UPPER NOW"}]
+
+
+def test_insert_convert_row_modifying_in_place(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "rows",
+            "-",
+            "--convert",
+            'row["is_chicken"] = True',
+        ],
+        input='{"name": "Azi"}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query("select name, is_chicken from rows"))
+    assert rows == [{"name": "Azi", "is_chicken": 1}]
+
+
+@pytest.mark.parametrize(
+    "options,expected_error",
+    (
+        (
+            ["--text", "--convert", "1"],
+            "Error: --convert must return dict or iterator\n",
+        ),
+        (["--convert", "1"], "Error: Rows must all be dictionaries, got: 1\n"),
+    ),
+)
+def test_insert_convert_error_messages(db_path, options, expected_error):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "rows",
+            "-",
+        ]
+        + options,
+        input='{"name": "Azi"}',
+    )
+    assert result.exit_code == 1
+    assert result.output == expected_error
+
+
+def test_insert_streaming_batch_size_1(db_path):
+    # https://github.com/simonw/sqlite-utils/issues/364
+    # Streaming with --batch-size 1 should commit on each record
+    # Can't use CliRunner().invoke() here bacuse we need to
+    # run assertions in between writing to process stdin
+    proc = subprocess.Popen(
+        [
+            sys.executable,
+            "-m",
+            "sqlite_utils",
+            "insert",
+            db_path,
+            "rows",
+            "-",
+            "--nl",
+            "--batch-size",
+            "1",
+        ],
+        stdin=subprocess.PIPE,
+        stdout=sys.stdout,
+    )
+    proc.stdin.write(b'{"name": "Azi"}\n')
+    proc.stdin.flush()
+
+    def try_until(expected):
+        tries = 0
+        while True:
+            rows = list(Database(db_path)["rows"].rows)
+            if rows == expected:
+                return
+            tries += 1
+            if tries > 10:
+                assert False, "Expected {}, got {}".format(expected, rows)
+            time.sleep(tries * 0.1)
+
+    try_until([{"name": "Azi"}])
+    proc.stdin.write(b'{"name": "Suna"}\n')
+    proc.stdin.flush()
+    try_until([{"name": "Azi"}, {"name": "Suna"}])
+    proc.stdin.close()
+    proc.wait()
+    assert proc.returncode == 0
+
+
+def test_insert_csv_headers_only(tmpdir):
+    """Test that CSV with only header row (no data) works with --detect-types (issue #702)"""
+    db_path = str(tmpdir / "test.db")
+    csv_path = str(tmpdir / "headers_only.csv")
+    with open(csv_path, "w") as fp:
+        fp.write("id,name,age\n")
+    # Should not crash with --detect-types (which is now the default)
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", csv_path, "--csv"],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    # Table should not exist since there were no data rows
+    db = Database(db_path)
+    assert not db["data"].exists()
+
+
+def test_insert_into_view_errors(tmpdir):
+    db_path = str(tmpdir / "test.db")
+    db = Database(db_path)
+    db["t"].insert({"id": 1})
+    db.create_view("v", "select * from t")
+    db.close()
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "v", "-"], input='{"id": 2}'
+    )
+    assert result.exit_code == 1
+    assert result.output.strip() == "Error: Table v is actually a view"
+
+
+def test_insert_csv_detect_types_leaves_existing_table_alone(db_path):
+    # Type detection is the default for CSV/TSV inserts, but it must only
+    # apply to tables created by this command - transforming a pre-existing
+    # table would rewrite its column types and corrupt data such as
+    # TEXT zip codes with leading zeros
+    db = Database(db_path)
+    db["places"].insert({"name": "Boston", "zip": "01234"})
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "places", "-", "--csv"],
+        catch_exceptions=False,
+        input="name,zip\nSF,94107",
+    )
+    assert result.exit_code == 0, result.output
+    assert db["places"].columns_dict["zip"] is str
+    assert list(db["places"].rows) == [
+        {"name": "Boston", "zip": "01234"},
+        {"name": "SF", "zip": "94107"},
+    ]
+
+
+def test_insert_csv_detect_types_new_table(db_path):
+    # A table created by the insert still gets detected types
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", "-", "--csv"],
+        catch_exceptions=False,
+        input="name,age,weight\nCleo,5,12.5",
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert db["data"].columns_dict == {"name": str, "age": int, "weight": float}
+
+
+@pytest.mark.parametrize(
+    "command,extra_args,input_text,expected_row",
+    (
+        (
+            "insert",
+            [],
+            "zipcode,score\n01234,9.5\n",
+            {"zipcode": "01234", "score": 9.5},
+        ),
+        (
+            "upsert",
+            ["--pk", "id"],
+            "id,zipcode,score\n1,01234,9.5\n",
+            {"id": 1, "zipcode": "01234", "score": 9.5},
+        ),
+    ),
+)
+def test_insert_upsert_csv_type_overrides_detected_types(
+    db_path, command, extra_args, input_text, expected_row
+):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            command,
+            db_path,
+            "places",
+            "-",
+            "--csv",
+        ]
+        + extra_args
+        + [
+            "--type",
+            "zipcode",
+            "text",
+        ],
+        catch_exceptions=False,
+        input=input_text,
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    expected_columns = {"zipcode": str, "score": float}
+    if command == "upsert":
+        expected_columns = {"id": int, **expected_columns}
+    assert db["places"].columns_dict == expected_columns
+    assert list(db["places"].rows) == [expected_row]
+
+
+def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path):
+    db = Database(db_path)
+    db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["upsert", db_path, "places", "-", "--csv", "--pk", "id"],
+        catch_exceptions=False,
+        input="id,name,zip\n2,SF,94107",
+    )
+    assert result.exit_code == 0, result.output
+    assert db["places"].columns_dict["zip"] is str
+    assert db["places"].get(1)["zip"] == "01234"
+
+
+def test_insert_invalid_pk_clean_error(db_path):
+    # An invalid --pk against an existing table should be a clean CLI
+    # error, not a raw InvalidColumns traceback
+    db = Database(db_path)
+    db["t"].insert({"a": 1})
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "t", "-", "--pk", "badcol"],
+        input='{"a": 2}',
+    )
+    assert result.exit_code == 1
+    assert result.exception is None or isinstance(result.exception, SystemExit)
+    assert result.output.startswith("Error: Invalid primary key column")
+
+
+# --code tests, see https://github.com/simonw/sqlite-utils/issues/684
+CODE_ROWS_FUNCTION = """
+def rows():
+    yield {"id": 1, "name": "Cleo"}
+    yield {"id": 2, "name": "Suna"}
+"""
+
+CODE_ROWS_ITERABLE = """
+rows = [
+    {"id": 1, "name": "Cleo"},
+    {"id": 2, "name": "Suna"},
+]
+"""
+
+
+@pytest.mark.parametrize("code", (CODE_ROWS_FUNCTION, CODE_ROWS_ITERABLE))
+def test_insert_code(tmpdir, code):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", code, "--pk", "id"],
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert db["creatures"].pks == ["id"]
+    assert list(db["creatures"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Suna"},
+    ]
+
+
+def test_insert_code_from_file(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    code_path = str(tmpdir / "gen.py")
+    with open(code_path, "w") as fp:
+        fp.write(CODE_ROWS_FUNCTION)
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", code_path],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(Database(db_path)["creatures"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Suna"},
+    ]
+
+
+def test_upsert_code(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    db = Database(db_path)
+    db["creatures"].insert_all(
+        [{"id": 1, "name": "old"}, {"id": 2, "name": "Suna"}], pk="id"
+    )
+    result = CliRunner().invoke(
+        cli.cli,
+        ["upsert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--pk", "id"],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["creatures"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Suna"},
+    ]
+
+
+def test_insert_code_requires_file_or_code(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(cli.cli, ["insert", db_path, "creatures"])
+    assert result.exit_code == 1
+    assert "Provide either a FILE argument or --code" in result.output
+
+
+def test_insert_code_mutually_exclusive_with_file(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "-", "--code", CODE_ROWS_FUNCTION],
+        input="{}",
+    )
+    assert result.exit_code == 1
+    assert "--code cannot be used with a FILE argument" in result.output
+
+
+def test_insert_code_rejects_input_format_options(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--csv"],
+    )
+    assert result.exit_code == 1
+    assert "--code cannot be used with input format options" in result.output
+
+
+def test_insert_code_missing_rows(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", "x = 1"],
+    )
+    assert result.exit_code == 1
+    assert "must define a 'rows' function or iterable" in result.output
+
+
+def test_insert_code_single_dict(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "creatures",
+            "--code",
+            'rows = {"id": 1, "name": "Cleo"}',
+            "--pk",
+            "id",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(Database(db_path)["creatures"].rows) == [{"id": 1, "name": "Cleo"}]
+
+
+def test_insert_code_not_iterable(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", "rows = 5"],
+    )
+    assert result.exit_code == 1
+    assert "must define a 'rows' function or iterable" in result.output
+
+
+def test_insert_code_syntax_error(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", "def rows(:"],
+    )
+    assert result.exit_code == 1
+    assert "Error in --code" in result.output
+
+
+def test_insert_code_file_not_found(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", "missing.py"],
+    )
+    assert result.exit_code == 1
+    assert "File not found: missing.py" in result.output

tests/test_cli_memory.py

@@ -1,5 +1,5 @@
+import click
 import json
-
 import pytest
 from click.testing import CliRunner
 
@@ -24,7 +24,8 @@ def test_memory_csv(tmpdir, sql_from, use_stdin):
             sql_from = "stdin"
     else:
         csv_path = str(tmpdir / "test.csv")
-        open(csv_path, "w").write(content)
+        with open(csv_path, "w") as fp:
+            fp.write(content)
     result = CliRunner().invoke(
         cli.cli,
         ["memory", csv_path, "select * from {}".format(sql_from), "--nl"],
@@ -46,7 +47,8 @@ def test_memory_tsv(tmpdir, use_stdin):
     else:
         input = None
         path = str(tmpdir / "chickens.tsv")
-        open(path, "w").write(data)
+        with open(path, "w") as fp:
+            fp.write(data)
         path = path + ":tsv"
         sql_from = "chickens"
     result = CliRunner().invoke(
@@ -71,7 +73,8 @@ def test_memory_json(tmpdir, use_stdin):
     else:
         input = None
         path = str(tmpdir / "chickens.json")
-        open(path, "w").write(data)
+        with open(path, "w") as fp:
+            fp.write(data)
         path = path + ":json"
         sql_from = "chickens"
     result = CliRunner().invoke(
@@ -96,7 +99,8 @@ def test_memory_json_nl(tmpdir, use_stdin):
     else:
         input = None
         path = str(tmpdir / "chickens.json")
-        open(path, "w").write(data)
+        with open(path, "w") as fp:
+            fp.write(data)
         path = path + ":nl"
         sql_from = "chickens"
     result = CliRunner().invoke(
@@ -152,6 +156,24 @@ def test_memory_csv_encoding(tmpdir, use_stdin):
     }
 
 
+def test_memory_csv_headers_only(tmpdir):
+    csv_path = str(tmpdir / "headers_only.csv")
+    with open(csv_path, "w") as fp:
+        fp.write("id,name,age\n")
+
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", csv_path, "", "--schema"],
+        catch_exceptions=False,
+    )
+
+    assert result.exit_code == 0
+    assert result.output.strip() == (
+        'CREATE VIEW "t1" AS select * from "headers_only";\n'
+        'CREATE VIEW "t" AS select * from "headers_only";'
+    )
+
+
 @pytest.mark.parametrize("extra_args", ([], ["select 1"]))
 def test_memory_dump(extra_args):
     result = CliRunner().invoke(
@@ -160,18 +182,21 @@ def test_memory_dump(extra_args):
         input="id,name\n1,Cleo\n2,Bants",
     )
     assert result.exit_code == 0
-    assert result.output.strip() == (
+    expected = (
         "BEGIN TRANSACTION;\n"
-        'CREATE TABLE "stdin" (\n'
-        "   [id] INTEGER,\n"
-        "   [name] TEXT\n"
+        'CREATE TABLE IF NOT EXISTS "stdin" (\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT\n'
         ");\n"
         "INSERT INTO \"stdin\" VALUES(1,'Cleo');\n"
         "INSERT INTO \"stdin\" VALUES(2,'Bants');\n"
-        "CREATE VIEW t1 AS select * from [stdin];\n"
-        "CREATE VIEW t AS select * from [stdin];\n"
+        'CREATE VIEW "t1" AS select * from "stdin";\n'
+        'CREATE VIEW "t" AS select * from "stdin";\n'
         "COMMIT;"
     )
+    # Using sqlite-dump it won't have IF NOT EXISTS
+    expected_alternative = expected.replace("IF NOT EXISTS ", "")
+    assert result.output.strip() in (expected, expected_alternative)
 
 
 @pytest.mark.parametrize("extra_args", ([], ["select 1"]))
@@ -184,11 +209,11 @@ def test_memory_schema(extra_args):
     assert result.exit_code == 0
     assert result.output.strip() == (
         'CREATE TABLE "stdin" (\n'
-        "   [id] INTEGER,\n"
-        "   [name] TEXT\n"
+        '   "id" INTEGER,\n'
+        '   "name" TEXT\n'
         ");\n"
-        "CREATE VIEW t1 AS select * from [stdin];\n"
-        "CREATE VIEW t AS select * from [stdin];"
+        'CREATE VIEW "t1" AS select * from "stdin";\n'
+        'CREATE VIEW "t" AS select * from "stdin";'
     )
 
 
@@ -220,3 +245,111 @@ def test_memory_no_detect_types(option):
         {"id": "1", "name": "Cleo", "weight": "45.5"},
         {"id": "2", "name": "Bants", "weight": "3.5"},
     ]
+
+
+def test_memory_flatten():
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-", "select * from stdin", "--flatten"],
+        input=json.dumps(
+            {
+                "httpRequest": {
+                    "latency": "0.112114537s",
+                    "requestMethod": "GET",
+                },
+                "insertId": "6111722f000b5b4c4d4071e2",
+            }
+        ),
+    )
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.output.strip()) == [
+        {
+            "httpRequest_latency": "0.112114537s",
+            "httpRequest_requestMethod": "GET",
+            "insertId": "6111722f000b5b4c4d4071e2",
+        }
+    ]
+
+
+def test_memory_analyze():
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-", "--analyze"],
+        input="id,name\n1,Cleo\n2,Bants",
+    )
+    assert result.exit_code == 0
+    assert result.output == (
+        "stdin.id: (1/2)\n\n"
+        "  Total rows: 2\n"
+        "  Null rows: 0\n"
+        "  Blank rows: 0\n\n"
+        "  Distinct values: 2\n\n"
+        "stdin.name: (2/2)\n\n"
+        "  Total rows: 2\n"
+        "  Null rows: 0\n"
+        "  Blank rows: 0\n\n"
+        "  Distinct values: 2\n\n"
+    )
+
+
+def test_memory_two_files_with_same_stem(tmpdir):
+    (tmpdir / "one").mkdir()
+    (tmpdir / "two").mkdir()
+    one = tmpdir / "one" / "data.csv"
+    two = tmpdir / "two" / "data.csv"
+    one.write_text("id,name\n1,Cleo\n2,Bants", encoding="utf-8")
+    two.write_text("id,name\n3,Blue\n4,Lila", encoding="utf-8")
+    result = CliRunner().invoke(cli.cli, ["memory", str(one), str(two), "", "--schema"])
+    assert result.exit_code == 0
+    assert result.output == (
+        'CREATE TABLE "data" (\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT\n'
+        ");\n"
+        'CREATE VIEW "t1" AS select * from "data";\n'
+        'CREATE VIEW "t" AS select * from "data";\n'
+        'CREATE TABLE "data_2" (\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT\n'
+        ");\n"
+        'CREATE VIEW "t2" AS select * from "data_2";\n'
+    )
+
+
+def test_memory_functions():
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "select hello()", "--functions", "hello = lambda: 'Hello'"],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == '[{"hello()": "Hello"}]'
+
+
+def test_memory_functions_multiple():
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "memory",
+            "select triple(2), quadruple(2)",
+            "--functions",
+            "def triple(x):\n    return x * 3",
+            "--functions",
+            "def quadruple(x):\n    return x * 4",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == '[{"triple(2)": 6, "quadruple(2)": 8}]'
+
+
+def test_memory_return_db(tmpdir):
+    # https://github.com/simonw/sqlite-utils/issues/643
+    from sqlite_utils.cli import cli
+
+    path = str(tmpdir / "dogs.csv")
+    with open(path, "w") as f:
+        f.write("id,name\n1,Cleo")
+
+    with click.Context(cli) as ctx:  # type: ignore[attr-defined]
+        db = ctx.invoke(cli.commands["memory"], paths=(path,), return_db=True)
+
+    assert db.table_names() == ["dogs"]

tests/test_cli_migrate.py

@@ -0,0 +1,507 @@
+import pathlib
+
+from click.testing import CliRunner
+import pytest
+import sqlite_utils
+import sqlite_utils.cli
+
+TWO_MIGRATIONS = """
+from sqlite_utils import Migrations
+
+m = Migrations("hello")
+
+@m()
+def foo(db):
+    db["foo"].insert({"hello": "world"})
+
+@m()
+def bar(db):
+    db["bar"].insert({"hello": "world"})
+"""
+
+
+@pytest.fixture
+def two_migrations(tmpdir):
+    path = pathlib.Path(tmpdir)
+    (path / "foo").mkdir()
+    migrations_py = path / "foo" / "migrations.py"
+    migrations_py.write_text(TWO_MIGRATIONS, "utf-8")
+    return path, migrations_py
+
+
+@pytest.fixture
+def two_sets_same_migration_name(tmpdir):
+    path = pathlib.Path(tmpdir)
+    migrations_py = path / "migrations.py"
+    migrations_py.write_text(
+        """
+from sqlite_utils import Migrations
+
+creatures = Migrations("creatures")
+
+@creatures()
+def create_table(db):
+    db["creatures"].insert({"name": "Cleo"})
+
+@creatures()
+def add_weight(db):
+    db["creature_weights"].insert({"weight": 4.2})
+
+sales = Migrations("sales")
+
+@sales()
+def create_table(db):
+    db["sales"].insert({"id": 1})
+
+@sales()
+def add_weight(db):
+    db["sales_weights"].insert({"weight": 10})
+""",
+        "utf-8",
+    )
+    return path, migrations_py
+
+
+@pytest.mark.parametrize("arg", ("TMPDIR", "TMPDIR/foo/migrations.py", "TMPDIR/foo/"))
+def test_basic(two_migrations, arg):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+
+    runner = CliRunner()
+
+    def _list():
+        list_result = runner.invoke(
+            sqlite_utils.cli.cli,
+            ["migrate", db_path, "--list", arg.replace("TMPDIR", str(path))],
+        )
+        assert list_result.exit_code == 0
+        return list_result.output
+
+    assert _list() == (
+        "Migrations for: hello\n\n"
+        "  Applied:\n\n"
+        "  Pending:\n"
+        "    foo\n"
+        "    bar\n\n"
+    )
+
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, arg.replace("TMPDIR", str(path))]
+    )
+    assert result.exit_code == 0, result.output
+
+    list_output = _list()
+    assert "Migrations for: hello\n\n  Applied:\n    " in list_output
+    prior_to_pending = list_output.split(" Pending")[0]
+    assert "  foo" in prior_to_pending
+    assert "  bar" in prior_to_pending
+    assert " Pending:\n    (none)" in list_output
+
+    db = sqlite_utils.Database(db_path)
+    assert db["foo"].exists()
+    assert db["bar"].exists()
+    assert db["_sqlite_migrations"].exists()
+    rows = list(db["_sqlite_migrations"].rows)
+    assert len(rows) == 2
+    assert rows[0]["name"] == "foo"
+    assert rows[1]["name"] == "bar"
+
+
+def test_list_same_migration_names_in_different_sets(capsys):
+    applied = sqlite_utils.Migrations("applied")
+
+    @applied(name="foo")
+    def applied_foo(db):
+        db["applied"].insert({"hello": "world"})
+
+    pending = sqlite_utils.Migrations("pending")
+
+    @pending(name="foo")
+    def pending_foo(db):
+        db["pending"].insert({"hello": "world"})
+
+    db = sqlite_utils.Database(memory=True)
+    applied.apply(db)
+
+    sqlite_utils.cli._display_migration_list(db, [applied, pending])
+
+    output = capsys.readouterr().out
+    assert (
+        "Migrations for: pending\n\n" "  Applied:\n\n" "  Pending:\n" "    foo\n\n"
+    ) in output
+
+
+def test_verbose(tmpdir):
+    path = pathlib.Path(tmpdir)
+    (path / "foo").mkdir()
+    migrations_py = path / "foo" / "migrations.py"
+    migrations_py.write_text(
+        """
+from sqlite_utils import Migrations
+
+m = Migrations("hello")
+
+@m()
+def foo(db):
+    db["dogs"].insert({"id": 1, "name": "Cleo"})
+    """,
+        "utf-8",
+    )
+    db_path = str(path / "test.db")
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py)]
+    )
+    assert result.exit_code == 0
+
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"]
+    )
+    assert result.exit_code == 0
+    expected = """
+Schema before:
+
+  CREATE TABLE "_sqlite_migrations" (
+     "id" INTEGER PRIMARY KEY,
+     "migration_set" TEXT,
+     "name" TEXT,
+     "applied_at" TEXT
+  );
+  CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name"
+      ON "_sqlite_migrations" ("migration_set", "name");
+  CREATE TABLE "dogs" (
+     "id" INTEGER,
+     "name" TEXT
+  );
+
+Schema after:
+
+  (unchanged)
+""".strip()
+    assert expected in result.output
+
+    new_migration = """
+@m()
+def bar(db):
+    db["dogs"].add_column("age", int)
+    db["dogs"].add_column("weight", float)
+    db["dogs"].transform()
+"""
+    migrations_py.write_text(migrations_py.read_text("utf-8") + new_migration)
+
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"]
+    )
+    assert result.exit_code == 0
+    expected_diff = """
+Schema diff:
+
+     ON "_sqlite_migrations" ("migration_set", "name");
+ CREATE TABLE "dogs" (
+    "id" INTEGER,
+-   "name" TEXT
++   "name" TEXT,
++   "age" INTEGER,
++   "weight" REAL
+ );
+""".strip()
+    assert expected_diff in result.output
+
+
+def test_stop_before(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(path / "foo" / "migrations.py"),
+            "--stop-before",
+            "bar",
+        ],
+    )
+    assert result.exit_code == 0
+    db = sqlite_utils.Database(db_path)
+    assert db["foo"].exists()
+    assert not db["bar"].exists()
+
+
+def test_stop_before_multiple_sets_unqualified(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    (path / "foo" / "migrations2.py").write_text(
+        """
+from sqlite_utils import Migrations
+
+m = Migrations("hello2")
+
+@m()
+def foo(db):
+    db["foo"].insert({"hello": "world"})
+    """,
+        "utf-8",
+    )
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(path / "foo" / "migrations.py"),
+            str(path / "foo" / "migrations2.py"),
+            "--stop-before",
+            "foo",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    db = sqlite_utils.Database(db_path)
+    assert db.table_names() == ["_sqlite_migrations"]
+    assert list(db["_sqlite_migrations"].rows) == []
+
+
+def test_stop_before_qualified_only_affects_named_set(two_sets_same_migration_name):
+    path, migrations_py = two_sets_same_migration_name
+    db_path = str(path / "test.db")
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(migrations_py),
+            "--stop-before",
+            "creatures:add_weight",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    db = sqlite_utils.Database(db_path)
+    assert db["creatures"].exists()
+    assert not db["creature_weights"].exists()
+    assert db["sales"].exists()
+    assert db["sales_weights"].exists()
+
+
+def test_stop_before_multiple_qualified(two_sets_same_migration_name):
+    path, migrations_py = two_sets_same_migration_name
+    db_path = str(path / "test.db")
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(migrations_py),
+            "--stop-before",
+            "creatures:add_weight",
+            "--stop-before",
+            "sales:add_weight",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    db = sqlite_utils.Database(db_path)
+    assert db["creatures"].exists()
+    assert not db["creature_weights"].exists()
+    assert db["sales"].exists()
+    assert not db["sales_weights"].exists()
+
+
+LEGACY_MIGRATIONS = """
+import datetime
+
+class _Migration:
+    def __init__(self, name, fn):
+        self.name = name
+        self.fn = fn
+
+class _Applied:
+    def __init__(self, name, applied_at):
+        self.name = name
+        self.applied_at = applied_at
+
+class LegacyMigrations:
+    # Mimics the sqlite-migrate 0.x Migrations class, in particular
+    # apply(db, stop_before=None) taking a single string
+    migrations_table = "_sqlite_migrations"
+
+    def __init__(self, name):
+        self.name = name
+        self._migrations = []
+
+    def __call__(self, fn):
+        self._migrations.append(_Migration(fn.__name__, fn))
+        return fn
+
+    def ensure_migrations_table(self, db):
+        db[self.migrations_table].create(
+            {"migration_set": str, "name": str, "applied_at": str},
+            pk=("migration_set", "name"),
+            if_not_exists=True,
+        )
+
+    def applied(self, db):
+        self.ensure_migrations_table(db)
+        return [
+            _Applied(row["name"], row["applied_at"])
+            for row in db[self.migrations_table].rows_where(
+                "migration_set = ?", [self.name]
+            )
+        ]
+
+    def pending(self, db):
+        applied = {m.name for m in self.applied(db)}
+        return [m for m in self._migrations if m.name not in applied]
+
+    def apply(self, db, stop_before=None):
+        for migration in self.pending(db):
+            if migration.name == stop_before:
+                return
+            migration.fn(db)
+            db[self.migrations_table].insert(
+                {
+                    "migration_set": self.name,
+                    "name": migration.name,
+                    "applied_at": str(
+                        datetime.datetime.now(datetime.timezone.utc)
+                    ),
+                }
+            )
+
+legacy = LegacyMigrations("legacy_set")
+
+@legacy
+def first(db):
+    db["first"].insert({"hello": "world"})
+
+@legacy
+def second(db):
+    db["second"].insert({"hello": "world"})
+"""
+
+
+def test_stop_before_unknown_name_errors(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli,
+        ["migrate", db_path, str(path), "--stop-before", "fooo"],
+    )
+    assert result.exit_code == 1
+    assert "--stop-before did not match any migrations: fooo" in result.output
+    # Nothing should have been applied
+    db = sqlite_utils.Database(db_path)
+    assert "foo" not in db.table_names()
+    assert "bar" not in db.table_names()
+
+
+def test_stop_before_with_legacy_migrations_class(tmpdir):
+    path = pathlib.Path(tmpdir)
+    (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8")
+    db_path = str(path / "test.db")
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli,
+        ["migrate", db_path, str(path), "--stop-before", "second"],
+    )
+    assert result.exit_code == 0, result.output
+    db = sqlite_utils.Database(db_path)
+    assert "first" in db.table_names()
+    assert "second" not in db.table_names()
+
+
+def test_stop_before_multiple_values_for_legacy_set_errors(tmpdir):
+    path = pathlib.Path(tmpdir)
+    (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8")
+    db_path = str(path / "test.db")
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(path),
+            "--stop-before",
+            "legacy_set:first",
+            "--stop-before",
+            "legacy_set:second",
+        ],
+    )
+    assert result.exit_code == 1
+    assert "single --stop-before" in result.output
+
+
+def test_list_does_not_create_database_file(two_migrations):
+    path, _ = two_migrations
+    db_path = path / "test.db"
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", str(db_path), str(path), "--list"]
+    )
+    assert result.exit_code == 0, result.output
+    assert "Pending:\n    foo\n    bar" in result.output
+    # Listing migrations must not create the database file
+    assert not db_path.exists()
+
+
+def test_list_does_not_upgrade_legacy_migrations_table(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    db = sqlite_utils.Database(db_path)
+    db["_sqlite_migrations"].create(
+        {"migration_set": str, "name": str, "applied_at": str},
+        pk=("migration_set", "name"),
+    )
+    db["_sqlite_migrations"].insert(
+        {"migration_set": "hello", "name": "foo", "applied_at": "x"}
+    )
+    db.close()
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"]
+    )
+    assert result.exit_code == 0, result.output
+    assert "foo - x" in result.output
+    # --list must not perform the one-way legacy schema upgrade
+    db2 = sqlite_utils.Database(db_path)
+    assert db2["_sqlite_migrations"].pks == ["migration_set", "name"]
+    db2.close()
+
+
+def test_stop_before_applied_migration_errors(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    migrations_path = str(path / "foo" / "migrations.py")
+    # Apply everything first
+    first = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        ["migrate", db_path, migrations_path, "--stop-before", "bar"],
+    )
+    assert first.exit_code == 0
+    # foo is now applied - stopping before it is an error, and bar
+    # must not be applied as a side effect
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        ["migrate", db_path, migrations_path, "--stop-before", "foo"],
+    )
+    assert result.exit_code != 0
+    assert "already been applied" in result.output
+    db = sqlite_utils.Database(db_path)
+    assert not db["bar"].exists()
+
+
+def test_list_with_legacy_class_is_read_only(tmpdir):
+    # Legacy sqlite-migrate classes create the _sqlite_migrations table
+    # from their pending()/applied() methods - --list must roll that
+    # back so it stays a read-only operation as documented
+    path = pathlib.Path(tmpdir)
+    (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8")
+    db_path = str(path / "test.db")
+    db = sqlite_utils.Database(db_path)
+    db["existing"].insert({"id": 1})
+    db.close()
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"]
+    )
+    assert result.exit_code == 0, result.output
+    assert "first" in result.output
+    db2 = sqlite_utils.Database(db_path)
+    assert "_sqlite_migrations" not in db2.table_names()
+    db2.close()

tests/test_column_casing.py

@@ -0,0 +1,233 @@
+"""
+SQLite treats column names as case-insensitive. These tests exercise the
+places where sqlite-utils performs Python-side lookups of column names
+provided by the caller, which should match the schema case-insensitively.
+
+https://github.com/simonw/sqlite-utils/issues/760
+"""
+
+import pytest
+
+from sqlite_utils import Database
+from sqlite_utils.db import ForeignKey
+
+
+def test_insert_populates_last_pk_case_insensitively(fresh_db):
+    books = fresh_db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.insert({"Id": 1, "Title": "One"}, pk="id")
+    assert books.last_pk == 1
+
+
+def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db):
+    books = fresh_db["books"]
+    books.create({"Author": str, "Position": int, "Title": str})
+    books.insert(
+        {"Author": "Sue", "Position": 1, "Title": "One"}, pk=("author", "position")
+    )
+    assert books.last_pk == ("Sue", 1)
+
+
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_pk_case_differs_from_schema(use_old_upsert):
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    books = db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.insert({"Id": 1, "Title": "One"})
+    books.upsert({"id": 1, "title": "Won"}, pk="id")
+    assert list(books.rows) == [{"Id": 1, "Title": "Won"}]
+    assert books.last_pk == 1
+
+
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_record_key_case_differs_from_pk(use_old_upsert):
+    # all_columns comes from the record keys, pk= from the caller
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    books = db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.upsert({"ID": 1, "Title": "One"}, pk="id")
+    assert list(books.rows) == [{"Id": 1, "Title": "One"}]
+    assert books.last_pk == 1
+
+
+def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db):
+    # pk is inferred from the existing schema as "Id", records use "id"
+    books = fresh_db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.upsert({"id": 1, "title": "One"})
+    assert list(books.rows) == [{"Id": 1, "Title": "One"}]
+    assert books.last_pk == 1
+
+
+def test_upsert_list_mode_pk_case_insensitive(fresh_db):
+    books = fresh_db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.upsert_all([["id", "title"], [1, "One"]], pk="Id")
+    assert list(books.rows) == [{"Id": 1, "Title": "One"}]
+    assert books.last_pk == 1
+
+
+def test_lookup_pk_case_insensitive(fresh_db):
+    fresh_db["species"].create({"ID": int, "Name": str}, pk="ID")
+    fresh_db["species"].insert({"ID": 5, "Name": "Palm"})
+    fresh_db["species"].create_index(["Name"], unique=True)
+    assert fresh_db["species"].lookup({"Name": "Palm"}, pk="id") == 5
+
+
+def test_lookup_does_not_create_redundant_index(fresh_db):
+    fresh_db["species"].create({"id": int, "Name": str}, pk="id")
+    fresh_db["species"].create_index(["Name"], unique=True)
+    fresh_db["species"].lookup({"name": "Palm"})
+    assert len(fresh_db["species"].indexes) == 1
+
+
+def test_create_table_transform_same_columns_different_case(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": int})
+    fresh_db["t"].insert({"Name": "Cleo", "Age": 5})
+    fresh_db.create_table("t", {"name": str, "age": int}, transform=True)
+    # Schema casing is preserved - SQLite considers these the same columns
+    assert fresh_db["t"].columns_dict == {"Name": str, "Age": int}
+    assert list(fresh_db["t"].rows) == [{"Name": "Cleo", "Age": 5}]
+
+
+def test_create_table_transform_case_insensitive_with_changes(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": int})
+    fresh_db.create_table("t", {"name": str, "age": str, "size": int}, transform=True)
+    # age changed type, size added, Name untouched
+    assert fresh_db["t"].columns_dict == {"Name": str, "Age": str, "size": int}
+
+
+def test_transform_types_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": str})
+    fresh_db["t"].transform(types={"age": int})
+    assert fresh_db["t"].columns_dict == {"Name": str, "Age": int}
+
+
+def test_transform_rename_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Name": str})
+    fresh_db["t"].transform(rename={"name": "title"})
+    assert fresh_db["t"].columns_dict == {"title": str}
+
+
+def test_transform_drop_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": int})
+    fresh_db["t"].transform(drop=["name"])
+    assert fresh_db["t"].columns_dict == {"Age": int}
+
+
+def test_transform_not_null_and_defaults_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": int})
+    fresh_db["t"].transform(not_null={"name"}, defaults={"age": 3})
+    columns = {c.name: c for c in fresh_db["t"].columns}
+    assert columns["Name"].notnull
+    assert fresh_db["t"].default_values == {"Age": 3}
+
+
+def test_transform_pk_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Id": int, "Name": str})
+    fresh_db["t"].transform(pk="id")
+    assert fresh_db["t"].pks == ["Id"]
+    assert fresh_db["t"].columns_dict == {"Id": int, "Name": str}
+
+
+def test_transform_drop_foreign_keys_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create(
+        {"id": int, "Parent_ID": int},
+        pk="id",
+        foreign_keys=[("Parent_ID", "parent", "Id")],
+    )
+    fresh_db["child"].transform(drop_foreign_keys=["parent_id"])
+    assert fresh_db["child"].foreign_keys == []
+
+
+def test_add_foreign_key_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id")
+    fresh_db["child"].add_foreign_key("parent_id", "parent", "id")
+    fks = fresh_db["child"].foreign_keys
+    assert len(fks) == 1
+    # The foreign key should use the schema casing of the columns
+    assert fks[0].column == "Parent_ID"
+    assert fks[0].other_column == "Id"
+
+
+def test_add_foreign_keys_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id")
+    fresh_db.add_foreign_keys([("child", "parent_id", "parent", "id")])
+    fks = fresh_db["child"].foreign_keys
+    assert len(fks) == 1
+    assert fks[0].column == "Parent_ID"
+    assert fks[0].other_column == "Id"
+
+
+def test_add_foreign_key_detects_existing_case_insensitively(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create(
+        {"id": int, "Parent_ID": int},
+        pk="id",
+        foreign_keys=[("Parent_ID", "parent", "Id")],
+    )
+    # ignore=True should treat this as already existing, not add a duplicate
+    fresh_db["child"].add_foreign_key("parent_id", "parent", "id", ignore=True)
+    assert len(fresh_db["child"].foreign_keys) == 1
+
+
+def test_add_column_fk_col_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create({"id": int}, pk="id")
+    fresh_db["child"].add_column("parent_id", int, fk="parent", fk_col="id")
+    fks = fresh_db["child"].foreign_keys
+    assert len(fks) == 1
+    assert fks[0].other_column == "Id"
+
+
+def test_extract_case_insensitive(fresh_db):
+    fresh_db["trees"].insert({"id": 1, "Species": "Palm"}, pk="id")
+    fresh_db["trees"].extract("species")
+    assert fresh_db["trees"].columns_dict == {"id": int, "Species_id": int}
+    assert list(fresh_db["Species"].rows) == [{"id": 1, "Species": "Palm"}]
+
+
+def test_convert_multi_case_insensitive(fresh_db):
+    fresh_db["t"].insert({"id": 1, "Name": "Cleo"}, pk="id")
+    fresh_db["t"].convert("name", lambda v: {"upper": v.upper()}, multi=True)
+    assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "upper": "CLEO"}]
+
+
+def test_convert_output_case_insensitive(fresh_db):
+    fresh_db["t"].insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id")
+    fresh_db["t"].convert("name", lambda v: v.upper(), output="upper")
+    assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "Upper": "CLEO"}]
+
+
+def test_create_table_sql_pk_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Id": int, "Name": str}, pk="id")
+    # Should not have created an extra lowercase "id" column
+    assert fresh_db["t"].columns_dict == {"Id": int, "Name": str}
+    assert fresh_db["t"].pks == ["Id"]
+
+
+def test_create_table_not_null_and_defaults_case_insensitive(fresh_db):
+    fresh_db["t"].create(
+        {"Name": str, "Age": int}, not_null={"name"}, defaults={"age": 1}
+    )
+    columns = {c.name: c for c in fresh_db["t"].columns}
+    assert columns["Name"].notnull
+    assert fresh_db["t"].default_values == {"Age": 1}
+
+
+def test_create_table_foreign_keys_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create(
+        {"id": int, "Parent_ID": int},
+        pk="id",
+        foreign_keys=[("parent_id", "parent", "id")],
+    )
+    fks = fresh_db["child"].foreign_keys
+    assert fks == [
+        ForeignKey(
+            table="child", column="Parent_ID", other_table="parent", other_column="Id"
+        )
+    ]

tests/test_constructor.py

@@ -1,4 +1,8 @@
 from sqlite_utils import Database
+from sqlite_utils.db import TransactionError
+from sqlite_utils.utils import sqlite3
+import pytest
+import sys
 
 
 def test_recursive_triggers():
@@ -9,3 +13,101 @@ def test_recursive_triggers():
 def test_recursive_triggers_off():
     db = Database(memory=True, recursive_triggers=False)
     assert not db.execute("PRAGMA recursive_triggers").fetchone()[0]
+
+
+def test_memory_name():
+    db1 = Database(memory_name="shared")
+    db2 = Database(memory_name="shared")
+    db1["dogs"].insert({"name": "Cleo"})
+    assert list(db2["dogs"].rows) == [{"name": "Cleo"}]
+
+
+def test_sqlite_version():
+    db = Database(memory=True)
+    version = db.sqlite_version
+    assert isinstance(version, tuple)
+    as_string = ".".join(map(str, version))
+    actual = next(db.query("select sqlite_version() as v"))["v"]
+    assert actual == as_string
+
+
+def test_database_context_manager(tmpdir):
+    path = str(tmpdir / "test.db")
+    with Database(path) as db:
+        db["t"].insert({"id": 1})
+        # Raw writes commit automatically too
+        db.execute("insert into t (id) values (2)")
+        # An explicitly opened transaction left uncommitted on purpose:
+        db.begin()
+        db.execute("insert into t (id) values (3)")
+    # The connection is closed...
+    with pytest.raises(sqlite3.ProgrammingError):
+        db.execute("select 1")
+    # ... and the open explicit transaction was rolled back, not committed
+    db2 = Database(path)
+    assert [r["id"] for r in db2["t"].rows] == [1, 2]
+    db2.close()
+
+
+@pytest.mark.parametrize("memory", [True, False])
+def test_database_close(tmpdir, memory):
+    if memory:
+        db = Database(memory=True)
+    else:
+        db = Database(str(tmpdir / "test.db"))
+    assert db.execute("select 1 + 1").fetchone()[0] == 2
+    db.close()
+    with pytest.raises(sqlite3.ProgrammingError):
+        db.execute("select 1 + 1")
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 12),
+    reason="sqlite3.connect(autocommit=) requires Python 3.12",
+)
+@pytest.mark.parametrize("autocommit", [True, False])
+def test_autocommit_connections_are_rejected(tmpdir, autocommit):
+    # These connection modes break commit()/rollback() in ways that
+    # silently lose data, so the constructor refuses them
+    conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=autocommit)
+    with pytest.raises(TransactionError):
+        Database(conn)
+    conn.close()
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 12),
+    reason="sqlite3.LEGACY_TRANSACTION_CONTROL requires Python 3.12",
+)
+def test_legacy_transaction_control_connection_is_accepted(tmpdir):
+    conn = sqlite3.connect(
+        str(tmpdir / "test.db"), autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL
+    )
+    db = Database(conn)
+    db["t"].insert({"id": 1}, pk="id")
+    assert [r["id"] for r in db["t"].rows] == [1]
+    db.close()
+
+
+def test_memory_attribute_for_memory_true():
+    db = Database(memory=True)
+    assert db.memory is True
+    assert db.memory_name is None
+
+
+def test_memory_attribute_for_memory_name():
+    db = Database(memory_name="shared_attr")
+    assert db.memory is True
+    assert db.memory_name == "shared_attr"
+
+
+def test_memory_attribute_for_memory_string_path():
+    db = Database(":memory:")
+    assert db.memory is True
+    assert db.memory_name is None
+
+
+def test_memory_attribute_for_file_path(tmpdir):
+    db = Database(str(tmpdir / "file.db"))
+    assert db.memory is False
+    assert db.memory_name is None

tests/test_convert.py

@@ -15,6 +15,14 @@ import pytest
             lambda value: value.upper(),
             {"title": "MIXED CASE", "abstract": "ABSTRACT"},
         ),
+        (
+            "title",
+            lambda value: {"upper": value.upper(), "lower": value.lower()},
+            {
+                "title": '{"upper": "MIXED CASE", "lower": "mixed case"}',
+                "abstract": "Abstract",
+            },
+        ),
     ),
 )
 def test_convert(fresh_db, columns, fn, expected):
@@ -42,6 +50,17 @@ def test_convert_where(fresh_db, where, where_args):
     assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}]
 
 
+def test_convert_handles_falsey_values(fresh_db):
+    # Falsey values like 0 should be converted (issue #527)
+    table = fresh_db["table"]
+    table.insert_all([{"x": 0}, {"x": 1}])
+    assert table.get(1)["x"] == 0
+    assert table.get(2)["x"] == 1
+    table.convert("x", lambda x: x + 1)
+    assert table.get(1)["x"] == 1
+    assert table.get(2)["x"] == 2
+
+
 @pytest.mark.parametrize(
     "drop,expected",
     (
@@ -58,7 +77,7 @@ def test_convert_output(fresh_db, drop, expected):
 
 def test_convert_output_multiple_column_error(fresh_db):
     table = fresh_db["table"]
-    with pytest.raises(AssertionError) as excinfo:
+    with pytest.raises(ValueError) as excinfo:
         table.convert(["title", "other"], lambda v: v, output="out")
         assert "output= can only be used with a single column" in str(excinfo.value)
 
@@ -81,10 +100,24 @@ def test_convert_multi(fresh_db):
     table = fresh_db["table"]
     table.insert({"title": "Mixed Case"})
     table.convert(
-        "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True
+        "title",
+        lambda v: {
+            "upper": v.upper(),
+            "lower": v.lower(),
+            "both": {
+                "upper": v.upper(),
+                "lower": v.lower(),
+            },
+        },
+        multi=True,
     )
     assert list(table.rows) == [
-        {"title": "Mixed Case", "upper": "MIXED CASE", "lower": "mixed case"}
+        {
+            "title": "Mixed Case",
+            "upper": "MIXED CASE",
+            "lower": "mixed case",
+            "both": '{"upper": "MIXED CASE", "lower": "mixed case"}',
+        }
     ]
 
 
@@ -115,3 +148,12 @@ def test_convert_multi_exception(fresh_db):
     table.insert({"title": "Mixed Case"})
     with pytest.raises(BadMultiValues):
         table.convert("title", lambda v: v.upper(), multi=True)
+
+
+def test_convert_repeated(fresh_db):
+    table = fresh_db["table"]
+    col = "num"
+    table.insert({col: 1})
+    table.convert(col, lambda x: x * 2)
+    table.convert(col, lambda _x: 0)
+    assert table.get(1) == {col: 0}

tests/test_create_view.py

@@ -15,7 +15,7 @@ def test_create_view_error(fresh_db):
 
 
 def test_create_view_only_arrow_one_param(fresh_db):
-    with pytest.raises(AssertionError):
+    with pytest.raises(ValueError):
         fresh_db.create_view("bar", "select 1 + 2", ignore=True, replace=True)
 
 

tests/test_default_value.py

@@ -0,0 +1,59 @@
+import pytest
+
+EXAMPLES = [
+    ("TEXT DEFAULT 'foo'", "'foo'", "'foo'"),
+    ("TEXT DEFAULT 'foo)'", "'foo)'", "'foo)'"),
+    ("INTEGER DEFAULT '1'", "'1'", "'1'"),
+    ("INTEGER DEFAULT 1", "1", "'1'"),
+    ("INTEGER DEFAULT (1)", "1", "'1'"),
+    # Expressions
+    (
+        "TEXT DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))",
+        "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')",
+        "(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))",
+    ),
+    # Special values
+    ("TEXT DEFAULT CURRENT_TIME", "CURRENT_TIME", "CURRENT_TIME"),
+    ("TEXT DEFAULT CURRENT_DATE", "CURRENT_DATE", "CURRENT_DATE"),
+    ("TEXT DEFAULT CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"),
+    ("TEXT DEFAULT current_timestamp", "current_timestamp", "current_timestamp"),
+    ("TEXT DEFAULT (CURRENT_TIMESTAMP)", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"),
+    # Strings
+    ("TEXT DEFAULT 'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'"),
+    ('TEXT DEFAULT "CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"'),
+    # Boolean and null keyword literals must stay unquoted
+    ("INTEGER DEFAULT TRUE", "TRUE", "TRUE"),
+    ("INTEGER DEFAULT FALSE", "FALSE", "FALSE"),
+    ("INTEGER DEFAULT true", "true", "true"),
+    ("TEXT DEFAULT NULL", "NULL", "NULL"),
+]
+
+
+@pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES)
+def test_quote_default_value(fresh_db, column_def, initial_value, expected_value):
+    fresh_db.execute("create table foo (col {})".format(column_def))
+    assert initial_value == fresh_db["foo"].columns[0].default_value
+    assert expected_value == fresh_db.quote_default_value(
+        fresh_db["foo"].columns[0].default_value
+    )
+
+
+def test_insert_empty_record_uses_default_values(fresh_db):
+    fresh_db.execute("""
+        CREATE TABLE has_defaults (
+            id INTEGER PRIMARY KEY,
+            name TEXT,
+            timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
+            is_active INTEGER NOT NULL DEFAULT 1
+        )
+        """)
+
+    table = fresh_db["has_defaults"]
+    table.insert({})
+
+    rows = list(table.rows)
+    assert len(rows) == 1
+    assert rows[0]["id"] == 1
+    assert rows[0]["name"] is None
+    assert rows[0]["timestamp"] is not None
+    assert rows[0]["is_active"] == 1

tests/test_delete.py

@@ -1,3 +1,6 @@
+import sqlite_utils
+
+
 def test_delete_rowid_table(fresh_db):
     table = fresh_db["table"]
     table.insert({"foo": 1}).last_pk
@@ -18,15 +21,44 @@ def test_delete_where(fresh_db):
     table = fresh_db["table"]
     for i in range(1, 11):
         table.insert({"id": i}, pk="id")
-    assert 10 == table.count
+    assert table.count == 10
     table.delete_where("id > ?", [5])
-    assert 5 == table.count
+    assert table.count == 5
 
 
 def test_delete_where_all(fresh_db):
     table = fresh_db["table"]
     for i in range(1, 11):
         table.insert({"id": i}, pk="id")
-    assert 10 == table.count
+    assert table.count == 10
     table.delete_where()
-    assert 0 == table.count
+    assert table.count == 0
+
+
+def test_delete_where_commits(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = sqlite_utils.Database(path)
+    db["table"].insert_all([{"id": i} for i in range(5)], pk="id")
+    db["table"].delete_where("id > ?", [2])
+    # The connection must not be left inside an open transaction,
+    # otherwise subsequent atomic() blocks never commit either
+    assert not db.conn.in_transaction
+    db["table"].insert({"id": 100})
+    db.close()
+    db2 = sqlite_utils.Database(path)
+    assert [r["id"] for r in db2["table"].rows] == [0, 1, 2, 100]
+    db2.close()
+
+
+def test_delete_where_analyze(fresh_db):
+    table = fresh_db["table"]
+    table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id")
+    table.create_index(["i"], analyze=True)
+    assert "sqlite_stat1" in fresh_db.table_names()
+    assert list(fresh_db["sqlite_stat1"].rows) == [
+        {"tbl": "table", "idx": "idx_table_i", "stat": "10 1"}
+    ]
+    table.delete_where("id > ?", [5], analyze=True)
+    assert list(fresh_db["sqlite_stat1"].rows) == [
+        {"tbl": "table", "idx": "idx_table_i", "stat": "6 1"}
+    ]

tests/test_docs.py

@@ -5,13 +5,15 @@ import pytest
 import re
 
 docs_path = Path(__file__).parent.parent / "docs"
-commands_re = re.compile(r"(?:\$ |    )sqlite-utils (\S+) ")
+commands_re = re.compile(r"(?:\$ |    )sqlite-utils (\S+)")
 recipes_re = re.compile(r"r\.(\w+)\(")
 
 
 @pytest.fixture(scope="session")
 def documented_commands():
-    rst = (docs_path / "cli.rst").read_text()
+    rst = ""
+    for doc in ("cli.rst", "plugins.rst"):
+        rst += (docs_path / doc).read_text()
     return {
         command
         for command in commands_re.findall(rst)
@@ -39,16 +41,22 @@ def test_convert_help():
     result = CliRunner().invoke(cli.cli, ["convert", "--help"])
     assert result.exit_code == 0
     for expected in (
-        "r.jsonsplit(value, ",
-        "r.parsedate(value, ",
-        "r.parsedatetime(value, ",
+        "r.jsonsplit(value:",
+        "r.parsedate(value:",
+        "r.parsedatetime(value:",
     ):
         assert expected in result.output
 
 
 @pytest.mark.parametrize(
     "recipe",
-    [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")],
+    [
+        n
+        for n in dir(recipes)
+        if not n.startswith("_")
+        and n not in ("json", "parser", "Callable", "Optional")
+        and callable(getattr(recipes, n))
+    ],
 )
 def test_recipes_are_documented(documented_recipes, recipe):
     assert recipe in documented_recipes

tests/test_duplicate.py

@@ -0,0 +1,41 @@
+from sqlite_utils.db import NoTable
+import datetime
+import pytest
+
+
+def test_duplicate(fresh_db):
+    # Create table using native Sqlite statement:
+    fresh_db.execute("""CREATE TABLE "table1" (
+    "text_col" TEXT,
+    "real_col" REAL,
+    "int_col" INTEGER,
+    "bool_col" INTEGER,
+    "datetime_col" TEXT)""")
+    # Insert one row of mock data:
+    dt = datetime.datetime.now()
+    data = {
+        "text_col": "Cleo",
+        "real_col": 3.14,
+        "int_col": -255,
+        "bool_col": True,
+        "datetime_col": str(dt),
+    }
+    table1 = fresh_db["table1"]
+    row_id = table1.insert(data).last_rowid
+    # Duplicate table:
+    table2 = table1.duplicate("table2")
+    # Ensure data integrity:
+    assert data == table2.get(row_id)
+    # Ensure schema integrity:
+    assert [
+        {"name": "text_col", "type": "TEXT"},
+        {"name": "real_col", "type": "REAL"},
+        {"name": "int_col", "type": "INT"},
+        {"name": "bool_col", "type": "INT"},
+        {"name": "datetime_col", "type": "TEXT"},
+    ] == [{"name": col.name, "type": col.type} for col in table2.columns]
+
+
+def test_duplicate_fails_if_table_does_not_exist(fresh_db):
+    with pytest.raises(NoTable):
+        fresh_db["not_a_table"].duplicate("duplicated")

tests/test_enable_counts.py

@@ -15,25 +15,25 @@ def test_enable_counts_specific_table(fresh_db):
     foo.enable_counts()
     assert foo.triggers_dict == {
         "foo_counts_insert": (
-            "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\n"
+            'CREATE TRIGGER "foo_counts_insert" AFTER INSERT ON "foo"\n'
             "BEGIN\n"
-            "    INSERT OR REPLACE INTO [_counts]\n"
+            '    INSERT OR REPLACE INTO "_counts"\n'
             "    VALUES (\n        'foo',\n"
             "        COALESCE(\n"
-            "            (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n"
+            '            (SELECT count FROM "_counts" WHERE "table" = \'foo\'),\n'
             "        0\n"
             "        ) + 1\n"
             "    );\n"
             "END"
         ),
         "foo_counts_delete": (
-            "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\n"
+            'CREATE TRIGGER "foo_counts_delete" AFTER DELETE ON "foo"\n'
             "BEGIN\n"
-            "    INSERT OR REPLACE INTO [_counts]\n"
+            '    INSERT OR REPLACE INTO "_counts"\n'
             "    VALUES (\n"
             "        'foo',\n"
             "        COALESCE(\n"
-            "            (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n"
+            '            (SELECT count FROM "_counts" WHERE "table" = \'foo\'),\n'
             "        0\n"
             "        ) - 1\n"
             "    );\n"
@@ -129,19 +129,19 @@ def test_uses_counts_after_enable_counts(counts_db_path):
     db = Database(counts_db_path)
     logged = []
     with db.tracer(lambda sql, parameters: logged.append((sql, parameters))):
-        assert db["foo"].count == 1
+        assert db.table("foo").count == 1
         assert logged == [
             ("select name from sqlite_master where type = 'view'", None),
-            ("select count(*) from [foo]", []),
+            ('select count(*) from "foo"', []),
         ]
         logged.clear()
         assert not db.use_counts_table
         db.enable_counts()
         assert db.use_counts_table
-        assert db["foo"].count == 1
+        assert db.table("foo").count == 1
     assert logged == [
         (
-            "CREATE TABLE IF NOT EXISTS [_counts](\n   [table] TEXT PRIMARY KEY,\n   count INTEGER DEFAULT 0\n);",
+            'CREATE TABLE IF NOT EXISTS "_counts"(\n   "table" TEXT PRIMARY KEY,\n   count INTEGER DEFAULT 0\n);',
             None,
         ),
         ("select name from sqlite_master where type = 'table'", None),
@@ -157,7 +157,7 @@ def test_uses_counts_after_enable_counts(counts_db_path):
         ("SELECT quote(:value)", {"value": "baz"}),
         ("select sql from sqlite_master where name = ?", ("_counts",)),
         ("select name from sqlite_master where type = 'view'", None),
-        ("select [table], count from _counts where [table] in (?)", ["foo"]),
+        ('select "table", count from _counts where "table" in (?)', ["foo"]),
     ]
 
 

tests/test_extract.py

@@ -24,19 +24,16 @@ def test_extract_single_column(fresh_db, table, fk_column):
     fresh_db["tree"].extract("species", table=table, fk_column=fk_column)
     assert fresh_db["tree"].schema == (
         'CREATE TABLE "tree" (\n'
-        "   [id] INTEGER PRIMARY KEY,\n"
-        "   [name] TEXT,\n"
-        "   [{}] INTEGER,\n".format(expected_fk)
-        + "   [end] INTEGER,\n"
-        + "   FOREIGN KEY([{}]) REFERENCES [{}]([id])\n".format(
-            expected_fk, expected_table
-        )
+        '   "id" INTEGER PRIMARY KEY,\n'
+        '   "name" TEXT,\n'
+        '   "{}" INTEGER REFERENCES "{}"("id"),\n'.format(expected_fk, expected_table)
+        + '   "end" INTEGER\n'
         + ")"
     )
     assert fresh_db[expected_table].schema == (
-        "CREATE TABLE [{}] (\n".format(expected_table)
-        + "   [id] INTEGER PRIMARY KEY,\n"
-        "   [species] TEXT\n"
+        'CREATE TABLE "{}" (\n'.format(expected_table)
+        + '   "id" INTEGER PRIMARY KEY,\n'
+        '   "species" TEXT\n'
         ")"
     )
     assert list(fresh_db[expected_table].rows) == [
@@ -74,17 +71,16 @@ def test_extract_multiple_columns_with_rename(fresh_db):
     )
     assert fresh_db["tree"].schema == (
         'CREATE TABLE "tree" (\n'
-        "   [id] INTEGER PRIMARY KEY,\n"
-        "   [name] TEXT,\n"
-        "   [common_name_latin_name_id] INTEGER,\n"
-        "   FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n"
+        '   "id" INTEGER PRIMARY KEY,\n'
+        '   "name" TEXT,\n'
+        '   "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n'
         ")"
     )
     assert fresh_db["common_name_latin_name"].schema == (
-        "CREATE TABLE [common_name_latin_name] (\n"
-        "   [id] INTEGER PRIMARY KEY,\n"
-        "   [name] TEXT,\n"
-        "   [latin_name] TEXT\n"
+        'CREATE TABLE "common_name_latin_name" (\n'
+        '   "id" INTEGER PRIMARY KEY,\n'
+        '   "name" TEXT,\n'
+        '   "latin_name" TEXT\n'
         ")"
     )
     assert list(fresh_db["common_name_latin_name"].rows) == [
@@ -126,14 +122,11 @@ def test_extract_rowid_table(fresh_db):
     fresh_db["tree"].extract(["common_name", "latin_name"])
     assert fresh_db["tree"].schema == (
         'CREATE TABLE "tree" (\n'
-        "   [name] TEXT,\n"
-        "   [common_name_latin_name_id] INTEGER,\n"
-        "   FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n"
+        '   "name" TEXT,\n'
+        '   "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n'
         ")"
     )
-    assert (
-        fresh_db.execute(
-            """
+    assert fresh_db.execute("""
         select
             tree.name,
             common_name_latin_name.common_name,
@@ -141,10 +134,7 @@ def test_extract_rowid_table(fresh_db):
         from tree
             join common_name_latin_name
             on tree.common_name_latin_name_id = common_name_latin_name.id
-    """
-        ).fetchall()
-        == [("Tree 1", "Palm", "Arecaceae")]
-    )
+    """).fetchall() == [("Tree 1", "Palm", "Arecaceae")]
 
 
 def test_reuse_lookup_table(fresh_db):
@@ -157,17 +147,15 @@ def test_reuse_lookup_table(fresh_db):
     fresh_db["individuals"].extract("species", rename={"species": "name"})
     assert fresh_db["sightings"].schema == (
         'CREATE TABLE "sightings" (\n'
-        "   [id] INTEGER PRIMARY KEY,\n"
-        "   [species_id] INTEGER,\n"
-        "   FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
+        '   "id" INTEGER PRIMARY KEY,\n'
+        '   "species_id" INTEGER REFERENCES "species"("id")\n'
         ")"
     )
     assert fresh_db["individuals"].schema == (
         'CREATE TABLE "individuals" (\n'
-        "   [id] INTEGER PRIMARY KEY,\n"
-        "   [name] TEXT,\n"
-        "   [species_id] INTEGER,\n"
-        "   FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
+        '   "id" INTEGER PRIMARY KEY,\n'
+        '   "name" TEXT,\n'
+        '   "species_id" INTEGER REFERENCES "species"("id")\n'
         ")"
     )
     assert list(fresh_db["species"].rows) == [
@@ -186,3 +174,131 @@ def test_extract_error_on_incompatible_existing_lookup_table(fresh_db):
     fresh_db["species2"].insert({"id": 1, "common_name": 3.5})
     with pytest.raises(InvalidColumns):
         fresh_db["tree"].extract("common_name", table="species2")
+
+
+def test_extract_works_with_null_values(fresh_db):
+    fresh_db["listens"].insert_all(
+        [
+            {"id": 1, "track_title": "foo", "album_title": "bar"},
+            {"id": 2, "track_title": "baz", "album_title": None},
+        ],
+        pk="id",
+    )
+    fresh_db["listens"].extract(
+        columns=["album_title"], table="albums", fk_column="album_id"
+    )
+    assert list(fresh_db["listens"].rows) == [
+        {"id": 1, "track_title": "foo", "album_id": 1},
+        {"id": 2, "track_title": "baz", "album_id": None},
+    ]
+    assert list(fresh_db["albums"].rows) == [
+        {"id": 1, "album_title": "bar"},
+    ]
+
+
+def test_extract_null_values_single_column(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/186
+    fresh_db["species"].insert({"id": 1, "species": "Wolf"}, pk="id")
+    fresh_db["individuals"].insert_all(
+        [
+            {"id": 10, "name": "Terriana", "species": "Fox"},
+            {"id": 11, "name": "Spenidorm", "species": None},
+            {"id": 12, "name": "Grantheim", "species": "Wolf"},
+            {"id": 13, "name": "Turnutopia", "species": None},
+            {"id": 14, "name": "Wargal", "species": "Wolf"},
+        ],
+        pk="id",
+    )
+    fresh_db["individuals"].extract("species")
+    # No null row should have been added to species
+    assert list(fresh_db["species"].rows) == [
+        {"id": 1, "species": "Wolf"},
+        {"id": 2, "species": "Fox"},
+    ]
+    assert list(fresh_db["individuals"].rows) == [
+        {"id": 10, "name": "Terriana", "species_id": 2},
+        {"id": 11, "name": "Spenidorm", "species_id": None},
+        {"id": 12, "name": "Grantheim", "species_id": 1},
+        {"id": 13, "name": "Turnutopia", "species_id": None},
+        {"id": 14, "name": "Wargal", "species_id": 1},
+    ]
+
+
+def test_extract_null_values_multiple_columns(fresh_db):
+    # A row should be extracted if at least one column is not null -
+    # only rows where ALL extracted columns are null are left alone
+    fresh_db["circulation"].insert_all(
+        [
+            {"id": 1, "title": "title one", "creator": "creator one", "year": 2018},
+            {"id": 2, "title": "title two", "creator": None, "year": 2019},
+            {"id": 3, "title": None, "creator": None, "year": 2020},
+            {"id": 4, "title": None, "creator": None, "year": 2021},
+        ],
+        pk="id",
+    )
+    fresh_db["circulation"].extract(
+        ["title", "creator"], table="books", fk_column="book_id"
+    )
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "title one", "creator": "creator one"},
+        {"id": 2, "title": "title two", "creator": None},
+    ]
+    assert list(fresh_db["circulation"].rows) == [
+        {"id": 1, "book_id": 1, "year": 2018},
+        {"id": 2, "book_id": 2, "year": 2019},
+        {"id": 3, "book_id": None, "year": 2020},
+        {"id": 4, "book_id": None, "year": 2021},
+    ]
+
+
+def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db):
+    # Even if the lookup table already contains an all-null row, rows where
+    # every extracted column is null should keep a null foreign key
+    fresh_db["species"].insert({"id": 1, "species": None}, pk="id")
+    fresh_db["individuals"].insert_all(
+        [
+            {"id": 10, "name": "Terriana", "species": "Fox"},
+            {"id": 11, "name": "Spenidorm", "species": None},
+        ],
+        pk="id",
+    )
+    fresh_db["individuals"].extract("species")
+    assert list(fresh_db["species"].rows) == [
+        {"id": 1, "species": None},
+        {"id": 2, "species": "Fox"},
+    ]
+    assert list(fresh_db["individuals"].rows) == [
+        {"id": 10, "name": "Terriana", "species_id": 2},
+        {"id": 11, "name": "Spenidorm", "species_id": None},
+    ]
+
+
+def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db):
+    # Unique indexes treat NULLs as distinct, so INSERT OR IGNORE alone
+    # cannot dedupe NULL-containing rows against the existing lookup
+    # table - extracting a second table into the same lookup previously
+    # inserted duplicate rows that nothing pointed to
+    fresh_db["t1"].insert_all(
+        [
+            {"id": 1, "species": None, "common": "X"},
+            {"id": 2, "species": "Oak", "common": "Oak"},
+        ],
+        pk="id",
+    )
+    fresh_db["t2"].insert_all([{"id": 1, "species": None, "common": "X"}], pk="id")
+    fresh_db["t1"].extract(["species", "common"], table="lk")
+    fresh_db["t2"].extract(["species", "common"], table="lk")
+    assert fresh_db["lk"].count == 2
+    # Both tables point at the same lookup row
+    t1_fk = fresh_db.execute("select lk_id from t1 where id = 1").fetchone()[0]
+    t2_fk = fresh_db.execute("select lk_id from t2 where id = 1").fetchone()[0]
+    assert t1_fk == t2_fk
+
+
+def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db):
+    # Non-NULL rows were already deduped by the unique index - keep it so
+    fresh_db["t1"].insert_all([{"id": 1, "species": "Oak"}], pk="id")
+    fresh_db["t2"].insert_all([{"id": 1, "species": "Oak"}], pk="id")
+    fresh_db["t1"].extract(["species"], table="lk")
+    fresh_db["t2"].extract(["species"], table="lk")
+    assert fresh_db["lk"].count == 1

tests/test_extracts.py

@@ -25,18 +25,18 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory):
             {"id": 2, "species_id": "Oak"},
             {"id": 3, "species_id": "Palm"},
         ],
-        **insert_kwargs
+        **insert_kwargs,
     )
     # Should now have two tables: Trees and Species
     assert {expected_table, "Trees"} == set(fresh_db.table_names())
     assert (
-        "CREATE TABLE [{}] (\n   [id] INTEGER PRIMARY KEY,\n   [value] TEXT\n)".format(
+        'CREATE TABLE "{}" (\n   "id" INTEGER PRIMARY KEY,\n   "value" TEXT\n)'.format(
             expected_table
         )
         == fresh_db[expected_table].schema
     )
     assert (
-        "CREATE TABLE [Trees] (\n   [id] INTEGER,\n   [species_id] INTEGER REFERENCES [{}]([id])\n)".format(
+        'CREATE TABLE "Trees" (\n   "id" INTEGER,\n   "species_id" INTEGER REFERENCES "{}"("id")\n)'.format(
             expected_table
         )
         == fresh_db["Trees"].schema
@@ -67,3 +67,51 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory):
         {"id": 2, "species_id": 1},
         {"id": 3, "species_id": 2},
     ] == list(fresh_db["Trees"].rows)
+
+
+def test_extracts_null_values(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/186
+    # Null values should stay null, not be extracted into the lookup table
+    fresh_db["Trees"].insert_all(
+        [
+            {"id": 1, "species_id": "Oak"},
+            {"id": 2, "species_id": None},
+            {"id": 3, "species_id": "Palm"},
+            {"id": 4, "species_id": None},
+        ],
+        extracts={"species_id": "Species"},
+    )
+    assert list(fresh_db["Species"].rows) == [
+        {"id": 1, "value": "Oak"},
+        {"id": 2, "value": "Palm"},
+    ]
+    assert list(fresh_db["Trees"].rows) == [
+        {"id": 1, "species_id": 1},
+        {"id": 2, "species_id": None},
+        {"id": 3, "species_id": 2},
+        {"id": 4, "species_id": None},
+    ]
+
+
+def test_extracts_null_values_list_mode(fresh_db):
+    # Same as test_extracts_null_values but for list-based records
+    fresh_db["Trees"].insert_all(
+        [
+            ["id", "species_id"],
+            [1, "Oak"],
+            [2, None],
+            [3, "Palm"],
+            [4, None],
+        ],
+        extracts={"species_id": "Species"},
+    )
+    assert list(fresh_db["Species"].rows) == [
+        {"id": 1, "value": "Oak"},
+        {"id": 2, "value": "Palm"},
+    ]
+    assert list(fresh_db["Trees"].rows) == [
+        {"id": 1, "species_id": 1},
+        {"id": 2, "species_id": None},
+        {"id": 3, "species_id": 2},
+        {"id": 4, "species_id": None},
+    ]

tests/test_foreign_keys.py

@@ -0,0 +1,691 @@
+"""Tests for compound (multi-column) foreign keys - issue #594."""
+
+import pytest
+from sqlite_utils import Database
+from sqlite_utils.db import AlterError, ForeignKey
+from sqlite_utils.utils import sqlite3
+
+COMPOUND_SCHEMA = """
+CREATE TABLE departments (
+    campus_name TEXT NOT NULL,
+    dept_code TEXT NOT NULL,
+    dept_name TEXT,
+    PRIMARY KEY (campus_name, dept_code)
+);
+CREATE TABLE courses (
+    course_code TEXT PRIMARY KEY,
+    course_name TEXT,
+    campus_name TEXT NOT NULL,
+    dept_code TEXT NOT NULL,
+    FOREIGN KEY (campus_name, dept_code)
+    REFERENCES departments(campus_name, dept_code)
+);
+"""
+
+
+@pytest.fixture
+def compound_db():
+    db = Database(memory=True)
+    db.executescript(COMPOUND_SCHEMA)
+    return db
+
+
+def test_compound_foreign_key(compound_db):
+    fks = compound_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.table == "courses"
+    assert fk.other_table == "departments"
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_columns == ("campus_name", "dept_code")
+    # Scalar column/other_column can't sensibly hold a compound key
+    assert fk.column is None
+    assert fk.other_column is None
+
+
+def test_single_foreign_key_gets_columns_fields(fresh_db):
+    fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
+    fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1})
+    fresh_db["books"].add_foreign_key("author_id", "authors", "id")
+    fk = fresh_db["books"].foreign_keys[0]
+    assert fk.is_compound is False
+    assert fk.column == "author_id"
+    assert fk.other_column == "id"
+    assert fk.columns == ("author_id",)
+    assert fk.other_columns == ("id",)
+
+
+def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db):
+    # Clean break in 4.0: ForeignKey is a dataclass, not a namedtuple, so the
+    # old tuple unpacking and indexing patterns now fail hard.
+    fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
+    fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1})
+    fresh_db["books"].add_foreign_key("author_id", "authors", "id")
+    fk = fresh_db["books"].foreign_keys[0]
+    with pytest.raises(TypeError):
+        table, column, other_table, other_column = fk
+    with pytest.raises(TypeError):
+        fk[0]
+
+
+def test_foreign_keys_are_sortable(fresh_db):
+    fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
+    fresh_db["categories"].insert({"id": 1, "name": "Wildlife"}, pk="id")
+    fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1, "category_id": 1})
+    fresh_db.add_foreign_keys(
+        [
+            ("books", "author_id", "authors", "id"),
+            ("books", "category_id", "categories", "id"),
+        ]
+    )
+    fks = sorted(fresh_db["books"].foreign_keys)
+    assert fks[0].column == "author_id"
+    assert fks[1].column == "category_id"
+
+
+def test_mixed_compound_and_single_foreign_keys_are_sortable():
+    # compound FKs have column=None, which must not break sorting
+    # against single-column FKs (None < str raises TypeError)
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE departments (
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        PRIMARY KEY (campus_name, dept_code)
+    );
+    CREATE TABLE accreditations (id INTEGER PRIMARY KEY);
+    CREATE TABLE courses (
+        course_code TEXT PRIMARY KEY,
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        accreditation_id INTEGER REFERENCES accreditations(id),
+        FOREIGN KEY (campus_name, dept_code)
+        REFERENCES departments(campus_name, dept_code)
+    );
+    """)
+    fks = db["courses"].foreign_keys
+    assert len(fks) == 2
+    assert {fk.is_compound for fk in fks} == {True, False}
+    fks_sorted = sorted(fks)
+    assert fks_sorted[0].other_table == "accreditations"
+    assert fks_sorted[1].other_table == "departments"
+
+
+@pytest.fixture
+def departments_db():
+    db = Database(memory=True)
+    db.create_table(
+        "departments",
+        {"campus_name": str, "dept_code": str, "dept_name": str},
+        pk=("campus_name", "dept_code"),
+    )
+    return db
+
+
+EXPECTED_COURSES_SCHEMA = (
+    'CREATE TABLE "courses" (\n'
+    '   "course_code" TEXT PRIMARY KEY,\n'
+    '   "campus_name" TEXT,\n'
+    '   "dept_code" TEXT,\n'
+    '   FOREIGN KEY ("campus_name", "dept_code") '
+    'REFERENCES "departments"("campus_name", "dept_code")\n'
+    ")"
+)
+
+
+@pytest.mark.parametrize(
+    "foreign_keys",
+    (
+        [
+            ForeignKey(
+                table="courses",
+                column=None,
+                other_table="departments",
+                other_column=None,
+                columns=("campus_name", "dept_code"),
+                other_columns=("campus_name", "dept_code"),
+                is_compound=True,
+            )
+        ],
+        [(("campus_name", "dept_code"), "departments", ("campus_name", "dept_code"))],
+        # Two-item form guesses the other table's primary key:
+        [(("campus_name", "dept_code"), "departments")],
+        # Lists work too, though tuples are the documented form:
+        [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])],
+    ),
+)
+def test_create_table_with_compound_foreign_key(departments_db, foreign_keys):
+    departments_db.create_table(
+        "courses",
+        {"course_code": str, "campus_name": str, "dept_code": str},
+        pk="course_code",
+        foreign_keys=foreign_keys,
+    )
+    assert departments_db["courses"].schema == EXPECTED_COURSES_SCHEMA
+    fks = departments_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_table == "departments"
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_create_table_compound_foreign_key_enforced(departments_db):
+    departments_db.execute("PRAGMA foreign_keys = ON")
+    departments_db.create_table(
+        "courses",
+        {"course_code": str, "campus_name": str, "dept_code": str},
+        pk="course_code",
+        foreign_keys=[(("campus_name", "dept_code"), "departments")],
+    )
+    departments_db["departments"].insert(
+        {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"}
+    )
+    departments_db["courses"].insert(
+        {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"}
+    )
+    with pytest.raises(sqlite3.IntegrityError):
+        departments_db.execute(
+            "insert into courses (course_code, campus_name, dept_code) "
+            "values ('X1', 'Nowhere', 'NOPE')"
+        )
+
+
+def test_create_table_compound_foreign_key_missing_other_column(departments_db):
+    with pytest.raises(AlterError):
+        departments_db.create_table(
+            "courses",
+            {"course_code": str, "campus_name": str, "dept_code": str},
+            pk="course_code",
+            foreign_keys=[
+                (("campus_name", "dept_code"), "departments", ("campus_name", "nope"))
+            ],
+        )
+
+
+def test_transform_preserves_compound_foreign_key(compound_db):
+    compound_db["courses"].transform(rename={"course_name": "title"})
+    fks = compound_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_table == "departments"
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_transform_rename_member_column_updates_compound_foreign_key(compound_db):
+    compound_db["courses"].transform(rename={"campus_name": "campus"})
+    fks = compound_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus", "dept_code")
+    # Referenced columns in the other table are unchanged
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_transform_drop_member_column_drops_compound_foreign_key(compound_db):
+    # Matches single-column behavior: dropping the column silently
+    # drops the foreign key that used it
+    compound_db["courses"].transform(drop={"dept_code"})
+    assert compound_db["courses"].foreign_keys == []
+    assert "FOREIGN KEY" not in compound_db["courses"].schema
+
+
+@pytest.mark.parametrize(
+    "drop_foreign_keys",
+    (
+        # A bare column name matches any foreign key it participates in:
+        ["campus_name"],
+        # A tuple must match the full compound key:
+        [("campus_name", "dept_code")],
+    ),
+)
+def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys):
+    compound_db["courses"].transform(drop_foreign_keys=drop_foreign_keys)
+    assert compound_db["courses"].foreign_keys == []
+    # The columns themselves survive
+    assert {"campus_name", "dept_code"} <= set(
+        compound_db["courses"].columns_dict.keys()
+    )
+
+
+@pytest.fixture
+def courses_db(departments_db):
+    departments_db.create_table(
+        "courses",
+        {"course_code": str, "campus_name": str, "dept_code": str},
+        pk="course_code",
+    )
+    return departments_db
+
+
+def test_add_compound_foreign_key(courses_db):
+    t = courses_db["courses"].add_foreign_key(
+        ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")
+    )
+    # Returns self
+    assert t.name == "courses"
+    fks = courses_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_table == "departments"
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_add_compound_foreign_key_guesses_other_columns(courses_db):
+    # Lists work here too, though tuples are the documented form
+    courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments")
+    fk = courses_db["courses"].foreign_keys[0]
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_add_compound_foreign_key_error_if_already_exists(courses_db):
+    courses_db["courses"].add_foreign_key(("campus_name", "dept_code"), "departments")
+    with pytest.raises(AlterError) as ex:
+        courses_db["courses"].add_foreign_key(
+            ("campus_name", "dept_code"), "departments"
+        )
+    assert "already exists" in ex.value.args[0]
+    # ignore=True should not raise
+    courses_db["courses"].add_foreign_key(
+        ("campus_name", "dept_code"), "departments", ignore=True
+    )
+
+
+def test_add_compound_foreign_key_error_if_column_missing(courses_db):
+    with pytest.raises(AlterError):
+        courses_db["courses"].add_foreign_key(("campus_name", "nope"), "departments")
+
+
+def test_db_add_foreign_keys_compound(courses_db):
+    courses_db.add_foreign_keys(
+        [
+            (
+                "courses",
+                ("campus_name", "dept_code"),
+                "departments",
+                ("campus_name", "dept_code"),
+            )
+        ]
+    )
+    fk = courses_db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus_name", "dept_code")
+
+
+def test_index_foreign_keys_compound_creates_composite_index(compound_db):
+    compound_db.index_foreign_keys()
+    index_columns = [i.columns for i in compound_db["courses"].indexes]
+    assert ["campus_name", "dept_code"] in index_columns
+    # No separate single-column indexes for the members
+    assert ["campus_name"] not in index_columns
+    assert ["dept_code"] not in index_columns
+
+
+def test_foreign_key_captures_on_delete_and_on_update():
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE authors (id INTEGER PRIMARY KEY);
+    CREATE TABLE books (
+        id INTEGER PRIMARY KEY,
+        author_id INTEGER REFERENCES authors(id)
+            ON DELETE CASCADE ON UPDATE RESTRICT
+    );
+    """)
+    fk = db["books"].foreign_keys[0]
+    assert fk.on_delete == "CASCADE"
+    assert fk.on_update == "RESTRICT"
+
+
+def test_foreign_key_on_delete_defaults_to_no_action(fresh_db):
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
+    fresh_db["books"].add_foreign_key("author_id", "authors", "id")
+    fk = fresh_db["books"].foreign_keys[0]
+    assert fk.on_delete == "NO ACTION"
+    assert fk.on_update == "NO ACTION"
+
+
+def test_create_table_foreign_key_with_on_delete(fresh_db):
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db.create_table(
+        "books",
+        {"id": int, "author_id": int},
+        pk="id",
+        foreign_keys=[
+            ForeignKey(
+                table="books",
+                column="author_id",
+                other_table="authors",
+                other_column="id",
+                on_delete="CASCADE",
+            )
+        ],
+    )
+    assert "ON DELETE CASCADE" in fresh_db["books"].schema
+    assert fresh_db["books"].foreign_keys[0].on_delete == "CASCADE"
+
+
+def test_transform_preserves_on_delete_cascade():
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE authors (id INTEGER PRIMARY KEY);
+    CREATE TABLE books (
+        id INTEGER PRIMARY KEY,
+        title TEXT,
+        author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE
+    );
+    """)
+    db["books"].transform(rename={"title": "book_title"})
+    fk = db["books"].foreign_keys[0]
+    assert fk.on_delete == "CASCADE"
+    assert fk.on_update == "NO ACTION"
+    assert "ON DELETE CASCADE" in db["books"].schema
+
+
+def test_transform_preserves_compound_foreign_key_on_delete():
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE departments (
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        PRIMARY KEY (campus_name, dept_code)
+    );
+    CREATE TABLE courses (
+        course_code TEXT PRIMARY KEY,
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        FOREIGN KEY (campus_name, dept_code)
+        REFERENCES departments(campus_name, dept_code) ON DELETE CASCADE
+    );
+    """)
+    db["courses"].transform(rename={"course_code": "code"})
+    fk = db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.on_delete == "CASCADE"
+    assert "ON DELETE CASCADE" in db["courses"].schema
+
+
+def test_implicit_primary_key_reference_is_resolved():
+    # REFERENCES authors (no column) has "to" of None in the pragma -
+    # it should be resolved to the primary key of the other table
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE authors (author_id INTEGER PRIMARY KEY);
+    CREATE TABLE books (
+        id INTEGER PRIMARY KEY,
+        author_id INTEGER REFERENCES authors
+    );
+    """)
+    fk = db["books"].foreign_keys[0]
+    assert fk.is_compound is False
+    assert fk.other_column == "author_id"
+    assert fk.other_columns == ("author_id",)
+
+
+def test_implicit_compound_primary_key_reference_is_resolved():
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE departments (
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        PRIMARY KEY (campus_name, dept_code)
+    );
+    CREATE TABLE courses (
+        course_code TEXT PRIMARY KEY,
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        FOREIGN KEY (campus_name, dept_code) REFERENCES departments
+    );
+    """)
+    fk = db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_foreign_key_normalizes_list_columns_to_tuples():
+    # Compound columns passed as lists are normalized to tuples, so they
+    # compare equal to introspected ForeignKeys
+    fk = ForeignKey(
+        table="courses",
+        column=None,
+        other_table="departments",
+        other_column=None,
+        columns=["campus_name", "dept_code"],
+        other_columns=["campus_name", "dept_code"],
+        is_compound=True,
+    )
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_add_foreign_keys_preserves_actions(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/594 review finding:
+    # ForeignKey objects passed to db.add_foreign_keys() were flattened
+    # to plain tuples, losing on_delete/on_update
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
+    fresh_db.add_foreign_keys(
+        [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")]
+    )
+    fk = fresh_db["books"].foreign_keys[0]
+    assert fk.on_delete == "CASCADE"
+    assert "ON DELETE CASCADE" in fresh_db["books"].schema
+
+
+def test_add_foreign_keys_preserves_actions_compound(courses_db):
+    courses_db.add_foreign_keys(
+        [
+            ForeignKey(
+                table="courses",
+                column=None,
+                other_table="departments",
+                other_column=None,
+                columns=("campus_name", "dept_code"),
+                other_columns=("campus_name", "dept_code"),
+                is_compound=True,
+                on_delete="CASCADE",
+            )
+        ]
+    )
+    fk = courses_db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.on_delete == "CASCADE"
+    assert "ON DELETE CASCADE" in courses_db["courses"].schema
+
+
+def test_add_foreign_key_on_delete_on_update(fresh_db):
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
+    fresh_db["books"].add_foreign_key(
+        "author_id", "authors", "id", on_delete="CASCADE", on_update="RESTRICT"
+    )
+    fk = fresh_db["books"].foreign_keys[0]
+    assert fk.on_delete == "CASCADE"
+    assert fk.on_update == "RESTRICT"
+    assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db["books"].schema
+    # The cascade should actually fire
+    fresh_db.execute("PRAGMA foreign_keys = ON")
+    fresh_db.execute("delete from authors where id = 1")
+    assert fresh_db["books"].count == 0
+
+
+def test_add_compound_foreign_key_on_delete(courses_db):
+    courses_db["courses"].add_foreign_key(
+        ("campus_name", "dept_code"), "departments", on_delete="SET NULL"
+    )
+    fk = courses_db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.on_delete == "SET NULL"
+    assert "ON DELETE SET NULL" in courses_db["courses"].schema
+
+
+def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db):
+    # The other table's PRIMARY KEY declares its columns in a different
+    # order to the table's column order. SQLite resolves the implicit
+    # "REFERENCES other" using PRIMARY KEY declaration order, so the
+    # introspected other_columns must too
+    fresh_db.execute("create table other (b text, a text, primary key (a, b))")
+    fresh_db.execute(
+        "create table child (x text, y text, foreign key (x, y) references other)"
+    )
+    fk = fresh_db["child"].foreign_keys[0]
+    assert fk.other_columns == ("a", "b")
+
+
+def test_transform_implicit_compound_foreign_key_stays_valid(fresh_db):
+    # transform() rewrites the implicit FK with explicit columns - they
+    # must be in PRIMARY KEY declaration order or valid data fails the
+    # foreign key check with an IntegrityError
+    fresh_db.execute("create table other (b text, a text, primary key (a, b))")
+    fresh_db.execute(
+        "create table child (x text, y text, foreign key (x, y) references other)"
+    )
+    fresh_db.execute("PRAGMA foreign_keys = ON")
+    fresh_db["other"].insert({"a": "A", "b": "B"})
+    fresh_db["child"].insert({"x": "A", "y": "B"})
+    fresh_db["child"].transform(types={"x": str})
+    assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b")
+    # The constraint still points the right way around
+    fresh_db["child"].insert({"x": "A", "y": "B"})
+    with pytest.raises(sqlite3.IntegrityError):
+        fresh_db["child"].insert({"x": "B", "y": "A"})
+
+
+def test_create_compound_foreign_key_guesses_pk_declaration_order(fresh_db):
+    fresh_db.execute("create table other (b text, a text, primary key (a, b))")
+    fresh_db["other"].insert({"a": "A", "b": "B"})
+    fresh_db["child"].create(
+        {"id": int, "x": str, "y": str},
+        pk="id",
+        foreign_keys=[(("x", "y"), "other")],
+    )
+    assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b")
+    fresh_db.execute("PRAGMA foreign_keys = ON")
+    fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"})
+    with pytest.raises(sqlite3.IntegrityError):
+        fresh_db["child"].insert({"id": 2, "x": "B", "y": "A"})
+
+
+def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db):
+    fresh_db.execute("create table other (b text, a text, primary key (a, b))")
+    fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id")
+    fresh_db["child"].add_foreign_key(("x", "y"), "other")
+    assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b")
+
+
+def test_foreign_keys_are_hashable(fresh_db):
+    # set() over foreign_keys worked with the 3.x namedtuple and must
+    # keep working with the dataclass
+    fresh_db["p"].insert({"id": 1}, pk="id")
+    fresh_db["c"].insert(
+        {"id": 1, "pid": 1}, pk="id", foreign_keys=[("pid", "p", "id")]
+    )
+    fks = set(fresh_db["c"].foreign_keys)
+    assert len(fks) == 1
+    assert ForeignKey("c", "pid", "p", "id") in fks
+    # Usable as dict keys too
+    assert {fk: True for fk in fks}
+
+
+def test_foreign_key_is_immutable():
+    import dataclasses
+
+    fk = ForeignKey("c", "pid", "p", "id")
+    with pytest.raises(dataclasses.FrozenInstanceError):
+        fk.table = "other"
+
+
+def test_foreign_key_equality_and_hash_include_actions():
+    # Two foreign keys differing only in ON DELETE behavior are different
+    # constraints - they compare unequal and hash separately
+    plain = ForeignKey("c", "pid", "p", "id")
+    cascade = ForeignKey("c", "pid", "p", "id", on_delete="CASCADE")
+    assert plain != cascade
+    assert len({plain, cascade}) == 2
+    assert plain == ForeignKey("c", "pid", "p", "id")
+
+
+def test_create_table_mixed_foreign_keys_list(fresh_db):
+    # 3.x accepted a mix of ForeignKey objects, tuples and bare column
+    # strings in foreign_keys= (ForeignKey was a namedtuple, so it passed
+    # the tuple check) - keep accepting the mix
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["publishers"].insert({"id": 1}, pk="id")
+    fresh_db["books"].create(
+        {"id": int, "author_id": int, "publisher_id": int},
+        pk="id",
+        foreign_keys=[
+            ForeignKey("books", "author_id", "authors", "id"),
+            ("publisher_id", "publishers", "id"),
+        ],
+    )
+    fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys}
+    assert fks == {"author_id": "authors", "publisher_id": "publishers"}
+
+
+def test_create_table_mixed_foreign_keys_with_string(fresh_db):
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["publishers"].insert({"id": 1}, pk="id")
+    fresh_db["books"].create(
+        {"id": int, "author_id": int, "publisher_id": int},
+        pk="id",
+        foreign_keys=[
+            "author_id",  # bare column, table and column guessed
+            ("publisher_id", "publishers", "id"),
+        ],
+    )
+    fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys}
+    assert fks == {"author_id": "authors", "publisher_id": "publishers"}
+
+
+def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db):
+    # Requesting an existing foreign key with different ON DELETE/ON UPDATE
+    # actions was silently skipped, dropping the requested change
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert(
+        {"id": 1, "author_id": 1},
+        pk="id",
+        foreign_keys=[("author_id", "authors", "id")],
+    )
+    with pytest.raises(AlterError) as ex:
+        fresh_db.add_foreign_keys(
+            [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")]
+        )
+    assert "ON DELETE" in str(ex.value)
+    assert fresh_db["books"].foreign_keys[0].on_delete == "NO ACTION"
+
+
+def test_add_foreign_keys_identical_existing_is_noop(fresh_db):
+    # An exact match, including actions, is silently skipped so repeated
+    # calls stay idempotent
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
+    fresh_db["books"].add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")
+    fresh_db.add_foreign_keys(
+        [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")]
+    )
+    fks = fresh_db["books"].foreign_keys
+    assert len(fks) == 1
+    assert fks[0].on_delete == "CASCADE"
+
+
+def test_add_foreign_keys_compound_column_count_mismatch_errors(fresh_db):
+    # Previously the extra other-column was silently discarded, creating
+    # a single-column foreign key to just ("id")
+    fresh_db["departments"].insert(
+        {"campus": "north", "code": "cs"}, pk=("campus", "code")
+    )
+    fresh_db["courses"].insert({"id": 1, "campus": "north"}, pk="id")
+    with pytest.raises(ValueError) as ex:
+        fresh_db.add_foreign_keys(
+            [("courses", ("campus",), "departments", ("campus", "code"))]
+        )
+    assert "same number of columns" in str(ex.value)
+    assert fresh_db["courses"].foreign_keys == []

tests/test_fts.py

@@ -1,6 +1,7 @@
 import pytest
 from sqlite_utils import Database
 from sqlite_utils.utils import sqlite3
+from unittest.mock import ANY
 
 search_records = [
     {
@@ -82,6 +83,20 @@ def test_enable_fts_escape_table_names(fresh_db):
     assert [] == list(table.search("bar"))
 
 
+def test_search_duplicate_columns_are_deduped(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/624
+    table = fresh_db["t"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"], fts_version="FTS4")
+    rows = list(table.search("tanuki", columns=["text", "text"]))
+    assert rows == [
+        {
+            "text": "tanuki are running tricksters",
+            "text_2": "tanuki are running tricksters",
+        }
+    ]
+
+
 def test_search_limit_offset(fresh_db):
     table = fresh_db["t"]
     table.insert_all(search_records)
@@ -94,6 +109,64 @@ def test_search_limit_offset(fresh_db):
     )
 
 
+@pytest.mark.parametrize("fts_version", ("FTS4", "FTS5"))
+def test_search_where(fresh_db, fts_version):
+    table = fresh_db["t"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"], fts_version=fts_version)
+    results = list(
+        table.search("are", where="country = :country", where_args={"country": "Japan"})
+    )
+    assert results == [
+        {
+            "rowid": 1,
+            "text": "tanuki are running tricksters",
+            "country": "Japan",
+            "not_searchable": "foo",
+        }
+    ]
+
+
+def test_search_where_args_disallows_query(fresh_db):
+    table = fresh_db["t"]
+    with pytest.raises(ValueError) as ex:
+        list(
+            table.search(
+                "x", where="author = :query", where_args={"query": "not allowed"}
+            )
+        )
+    assert (
+        ex.value.args[0]
+        == "'query' is a reserved key and cannot be passed to where_args for .search()"
+    )
+
+
+def test_search_include_rank(fresh_db):
+    table = fresh_db["t"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"], fts_version="FTS5")
+    results = list(table.search("are", include_rank=True))
+    assert results == [
+        {
+            "rowid": 1,
+            "text": "tanuki are running tricksters",
+            "country": "Japan",
+            "not_searchable": "foo",
+            "rank": ANY,
+        },
+        {
+            "rowid": 2,
+            "text": "racoons are biting trash pandas",
+            "country": "USA",
+            "not_searchable": "bar",
+            "rank": ANY,
+        },
+    ]
+    assert isinstance(results[0]["rank"], float)
+    assert isinstance(results[1]["rank"], float)
+    assert results[0]["rank"] < results[1]["rank"]
+
+
 def test_enable_fts_table_names_containing_spaces(fresh_db):
     table = fresh_db["test"]
     table.insert({"column with spaces": "in its name"})
@@ -148,32 +221,32 @@ def test_populate_fts_escape_table_names(fresh_db):
     ] == list(table.search("usa"))
 
 
-def test_fts_tokenize(fresh_db):
-    for fts_version in ("4", "5"):
-        table_name = "searchable_{}".format(fts_version)
-        table = fresh_db[table_name]
-        table.insert_all(search_records)
-        # Test without porter stemming
-        table.enable_fts(
-            ["text", "country"],
-            fts_version="FTS{}".format(fts_version),
-        )
-        assert [] == list(table.search("bite"))
-        # Test WITH stemming
-        table.disable_fts()
-        table.enable_fts(
-            ["text", "country"],
-            fts_version="FTS{}".format(fts_version),
-            tokenize="porter",
-        )
-        rows = list(table.search("bite", order_by="rowid"))
-        assert len(rows) == 1
-        assert {
-            "rowid": 2,
-            "text": "racoons are biting trash pandas",
-            "country": "USA",
-            "not_searchable": "bar",
-        }.items() <= rows[0].items()
+@pytest.mark.parametrize("fts_version", ("4", "5"))
+def test_fts_tokenize(fresh_db, fts_version):
+    table_name = "searchable_{}".format(fts_version)
+    table = fresh_db[table_name]
+    table.insert_all(search_records)
+    # Test without porter stemming
+    table.enable_fts(
+        ["text", "country"],
+        fts_version="FTS{}".format(fts_version),
+    )
+    assert [] == list(table.search("bite"))
+    # Test WITH stemming
+    table.disable_fts()
+    table.enable_fts(
+        ["text", "country"],
+        fts_version="FTS{}".format(fts_version),
+        tokenize="porter",
+    )
+    rows = list(table.search("bite", order_by="rowid"))
+    assert len(rows) == 1
+    assert {
+        "rowid": 2,
+        "text": "racoons are biting trash pandas",
+        "country": "USA",
+        "not_searchable": "bar",
+    }.items() <= rows[0].items()
 
 
 def test_optimize_fts(fresh_db):
@@ -254,13 +327,12 @@ def test_disable_fts(fresh_db, create_triggers):
     assert ["searchable"] == fresh_db.table_names()
 
 
-@pytest.mark.parametrize("table_to_fix", ["searchable", "searchable_fts"])
-def test_rebuild_fts(fresh_db, table_to_fix):
+def test_rebuild_fts(fresh_db):
     table = fresh_db["searchable"]
     table.insert(search_records[0])
     table.enable_fts(["text", "country"])
     # Run a search
-    rows = list(table.search("tanuki"))
+    rows = list(table.search("are"))
     assert len(rows) == 1
     assert {
         "rowid": 1,
@@ -268,21 +340,32 @@ def test_rebuild_fts(fresh_db, table_to_fix):
         "country": "Japan",
         "not_searchable": "foo",
     }.items() <= rows[0].items()
-    # Delete from searchable_fts_data
-    fresh_db["searchable_fts_data"].delete_where()
-    # This should have broken the index
-    with pytest.raises(sqlite3.DatabaseError):
-        list(table.search("tanuki"))
+    # Insert another record
+    table.insert(search_records[1])
+    # This should NOT show up in searches
+    assert len(list(table.search("are"))) == 1
     # Running rebuild_fts() should fix it
-    fresh_db[table_to_fix].rebuild_fts()
-    rows2 = list(table.search("tanuki"))
-    assert len(rows2) == 1
-    assert {
-        "rowid": 1,
-        "text": "tanuki are running tricksters",
-        "country": "Japan",
-        "not_searchable": "foo",
-    }.items() <= rows2[0].items()
+    table.rebuild_fts()
+    rows2 = list(table.search("are"))
+    assert len(rows2) == 2
+
+
+@pytest.mark.parametrize("method", ["optimize", "rebuild_fts"])
+def test_optimize_and_rebuild_fts_commit(tmpdir, method):
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    table = db["searchable"]
+    table.insert(search_records[0])
+    table.enable_fts(["text", "country"])
+    getattr(table, method)()
+    # The connection must not be left inside an open transaction,
+    # otherwise this and all subsequent writes are lost on close
+    assert not db.conn.in_transaction
+    table.insert(search_records[1])
+    db.close()
+    db2 = Database(path)
+    assert db2["searchable"].count == 2
+    db2.close()
 
 
 @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"])
@@ -369,12 +452,35 @@ def test_enable_fts_replace_does_nothing_if_args_the_same():
     assert all(q[0].startswith("select ") for q in queries)
 
 
-def test_enable_fts_error_message_on_views():
+def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table():
+    db = Database(memory=True)
+    db["books"].insert(
+        {
+            "id": 1,
+            "title": "Habits of Australian Marsupials",
+            "author": "Marlee Hawkins",
+        },
+        pk="id",
+    )
+    db.executescript("""
+        CREATE VIRTUAL TABLE [books_fts] USING FTS5 (
+            [title],
+            content=[books]
+        );
+    """)
+
+    db["books"].enable_fts(["title", "author"], replace=True)
+
+    assert db["books_fts"].columns_dict.keys() == {"title", "author"}
+    assert 'content="books"' in db["books_fts"].schema
+
+
+def test_view_has_no_enable_fts():
     db = Database(memory=True)
     db.create_view("hello", "select 1 + 1")
-    with pytest.raises(NotImplementedError) as e:
-        db["hello"].enable_fts()
-        assert e.value.args[0] == "enable_fts() is supported on tables but not on views"
+    # Views deliberately do not have an enable_fts() method
+    with pytest.raises(AttributeError):
+        db["hello"].enable_fts()  # type: ignore[union-attr]
 
 
 @pytest.mark.parametrize(
@@ -384,85 +490,107 @@ def test_enable_fts_error_message_on_views():
             {},
             "FTS5",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
                 "        *\n"
-                "    from [books]\n"
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].*\n"
+                '    "original".*\n'
                 "from\n"
-                "    [original]\n"
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
-                "    [books_fts].rank"
+                '    "books_fts".rank'
             ),
         ),
         (
             {"columns": ["title"], "order_by": "rowid", "limit": 10},
             "FTS5",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
-                "        [title]\n"
-                "    from [books]\n"
+                '        "title"\n'
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].[title]\n"
+                '    "original"."title"\n'
                 "from\n"
-                "    [original]\n"
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
                 "    rowid\n"
                 "limit 10"
             ),
         ),
+        (
+            {"where": "author = :author"},
+            "FTS5",
+            (
+                'with "original" as (\n'
+                "    select\n"
+                "        rowid,\n"
+                "        *\n"
+                '    from "books"\n'
+                "    where author = :author\n"
+                ")\n"
+                "select\n"
+                '    "original".*\n'
+                "from\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
+                "where\n"
+                '    "books_fts" match :query\n'
+                "order by\n"
+                '    "books_fts".rank'
+            ),
+        ),
         (
             {"columns": ["title"]},
             "FTS4",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
-                "        [title]\n"
-                "    from [books]\n"
+                '        "title"\n'
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].[title]\n"
+                '    "original"."title"\n'
                 "from\n"
-                "    [original]\n"
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
-                "    rank_bm25(matchinfo([books_fts], 'pcnalx'))"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))"
             ),
         ),
         (
             {"offset": 1, "limit": 1},
             "FTS4",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
                 "        *\n"
-                "    from [books]\n"
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].*\n"
+                '    "original".*\n'
                 "from\n"
-                "    [original]\n"
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
-                "    rank_bm25(matchinfo([books_fts], 'pcnalx'))\n"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))\n"
                 "limit 1 offset 1"
             ),
         ),
@@ -470,24 +598,90 @@ def test_enable_fts_error_message_on_views():
             {"limit": 2},
             "FTS4",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
                 "        *\n"
-                "    from [books]\n"
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].*\n"
+                '    "original".*\n'
                 "from\n"
-                "    [original]\n"
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
-                "    rank_bm25(matchinfo([books_fts], 'pcnalx'))\n"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))\n"
                 "limit 2"
             ),
         ),
+        (
+            {"where": "author = :author"},
+            "FTS4",
+            (
+                'with "original" as (\n'
+                "    select\n"
+                "        rowid,\n"
+                "        *\n"
+                '    from "books"\n'
+                "    where author = :author\n"
+                ")\n"
+                "select\n"
+                '    "original".*\n'
+                "from\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
+                "where\n"
+                '    "books_fts" match :query\n'
+                "order by\n"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))"
+            ),
+        ),
+        (
+            {"include_rank": True},
+            "FTS5",
+            (
+                'with "original" as (\n'
+                "    select\n"
+                "        rowid,\n"
+                "        *\n"
+                '    from "books"\n'
+                ")\n"
+                "select\n"
+                '    "original".*,\n'
+                '    "books_fts".rank rank\n'
+                "from\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
+                "where\n"
+                '    "books_fts" match :query\n'
+                "order by\n"
+                '    "books_fts".rank'
+            ),
+        ),
+        (
+            {"include_rank": True},
+            "FTS4",
+            (
+                'with "original" as (\n'
+                "    select\n"
+                "        rowid,\n"
+                "        *\n"
+                '    from "books"\n'
+                ")\n"
+                "select\n"
+                '    "original".*,\n'
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx')) rank\n"
+                "from\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
+                "where\n"
+                '    "books_fts" match :query\n'
+                "order by\n"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))"
+            ),
+        ),
     ],
 )
 def test_search_sql(kwargs, fts, expected):
@@ -501,3 +695,54 @@ def test_search_sql(kwargs, fts, expected):
     db["books"].enable_fts(["title", "author"], fts_version=fts)
     sql = db["books"].search_sql(**kwargs)
     assert sql == expected
+
+
+@pytest.mark.parametrize(
+    "input,expected",
+    (
+        ("dog", '"dog"'),
+        ("cat,", '"cat,"'),
+        ("cat's", '"cat\'s"'),
+        ("dog.", '"dog."'),
+        ("cat dog", '"cat" "dog"'),
+        # If a phrase is already double quoted, leave it so
+        ('"cat dog"', '"cat dog"'),
+        ('"cat dog" fish', '"cat dog" "fish"'),
+        # Sensibly handle unbalanced double quotes
+        ('cat"', '"cat"'),
+        ('"cat dog" "fish', '"cat dog" "fish"'),
+    ),
+)
+def test_quote_fts_query(fresh_db, input, expected):
+    table = fresh_db["searchable"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"])
+    quoted = fresh_db.quote_fts(input)
+    assert quoted == expected
+    # Executing query does not crash.
+    list(table.search(quoted))
+
+
+def test_search_quote(fresh_db):
+    table = fresh_db["searchable"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"])
+    query = "cat's"
+    with pytest.raises(sqlite3.OperationalError):
+        list(table.search(query))
+    # No exception with quote=True
+    list(table.search(query, quote=True))
+
+
+def test_enable_fts_cli_on_view_errors(tmpdir):
+    db_path = str(tmpdir / "test.db")
+    db = Database(db_path)
+    db["t"].insert({"text": "hello"})
+    db.create_view("v", "select * from t")
+    db.close()
+    from click.testing import CliRunner
+    from sqlite_utils import cli as cli_module
+
+    result = CliRunner().invoke(cli_module.cli, ["enable-fts", db_path, "v", "text"])
+    assert result.exit_code == 1
+    assert result.output.strip() == "Error: Table v is actually a view"

tests/test_gis.py

@@ -0,0 +1,236 @@
+import json
+import pytest
+
+from click.testing import CliRunner
+from sqlite_utils.cli import cli
+from sqlite_utils.db import Database
+from sqlite_utils.utils import find_spatialite, sqlite3
+
+pytestmark = [
+    pytest.mark.skipif(
+        not find_spatialite(), reason="Could not find SpatiaLite extension"
+    ),
+    pytest.mark.skipif(
+        not hasattr(sqlite3.Connection, "enable_load_extension"),
+        reason="sqlite3.Connection missing enable_load_extension",
+    ),
+]
+
+
+# python API tests
+def test_find_spatialite():
+    spatialite = find_spatialite()
+    assert spatialite is None or isinstance(spatialite, str)
+
+
+def test_init_spatialite():
+    db = Database(memory=True)
+    spatialite = find_spatialite()
+    db.init_spatialite(spatialite)
+    assert "spatial_ref_sys" in db.table_names()
+
+
+def test_add_geometry_column():
+    db = Database(memory=True)
+    spatialite = find_spatialite()
+    db.init_spatialite(spatialite)
+
+    # create a table first
+    table = db.create_table("locations", {"id": str, "properties": str})
+    table.add_geometry_column(
+        column_name="geometry",
+        geometry_type="Point",
+        srid=4326,
+        coord_dimension="XY",
+    )
+
+    assert db["geometry_columns"].get(["locations", "geometry"]) == {
+        "f_table_name": "locations",
+        "f_geometry_column": "geometry",
+        "geometry_type": 1,  # point
+        "coord_dimension": 2,
+        "srid": 4326,
+        "spatial_index_enabled": 0,
+    }
+
+
+def test_create_spatial_index():
+    db = Database(memory=True)
+    spatialite = find_spatialite()
+    assert db.init_spatialite(spatialite)
+
+    # create a table, add a geometry column with default values
+    table = db.create_table("locations", {"id": str, "properties": str})
+    assert table.add_geometry_column("geometry", "Point")
+
+    # index it
+    assert table.create_spatial_index("geometry")
+
+    assert "idx_locations_geometry" in db.table_names()
+
+
+def test_double_create_spatial_index():
+    db = Database(memory=True)
+    spatialite = find_spatialite()
+    db.init_spatialite(spatialite)
+
+    # create a table, add a geometry column with default values
+    table = db.create_table("locations", {"id": str, "properties": str})
+    table.add_geometry_column("geometry", "Point")
+
+    # index it, return True
+    assert table.create_spatial_index("geometry")
+
+    assert "idx_locations_geometry" in db.table_names()
+
+    # call it again, return False
+    assert not table.create_spatial_index("geometry")
+
+
+# cli tests
+@pytest.mark.parametrize("use_spatialite_shortcut", [True, False])
+def test_query_load_extension(use_spatialite_shortcut):
+    # Without --load-extension:
+    result = CliRunner().invoke(cli, [":memory:", "select spatialite_version()"])
+    assert result.exit_code == 1
+    assert "no such function: spatialite_version" in result.output
+    # With --load-extension:
+    if use_spatialite_shortcut:
+        load_extension = "spatialite"
+    else:
+        load_extension = find_spatialite()
+    result = CliRunner().invoke(
+        cli,
+        [
+            ":memory:",
+            "select spatialite_version()",
+            "--load-extension={}".format(load_extension),
+        ],
+    )
+    assert result.exit_code == 0, result.stdout
+    assert ["spatialite_version()"] == list(json.loads(result.output)[0].keys())
+
+
+def test_cli_create_spatialite(tmpdir):
+    # sqlite-utils create test.db --init-spatialite
+    db_path = tmpdir / "created.db"
+    result = CliRunner().invoke(
+        cli, ["create-database", str(db_path), "--init-spatialite"]
+    )
+
+    assert result.exit_code == 0
+    assert db_path.exists()
+    assert db_path.read_binary()[:16] == b"SQLite format 3\x00"
+
+    db = Database(str(db_path))
+    assert "spatial_ref_sys" in db.table_names()
+
+
+def test_cli_add_geometry_column(tmpdir):
+    # create a rowid table with one column
+    db_path = tmpdir / "spatial.db"
+    db = Database(str(db_path))
+    db.init_spatialite()
+
+    table = db["locations"].create({"name": str})
+
+    result = CliRunner().invoke(
+        cli,
+        [
+            "add-geometry-column",
+            str(db_path),
+            table.name,
+            "geometry",
+            "--type",
+            "POINT",
+        ],
+    )
+
+    assert result.exit_code == 0
+
+    assert db["geometry_columns"].get(["locations", "geometry"]) == {
+        "f_table_name": "locations",
+        "f_geometry_column": "geometry",
+        "geometry_type": 1,  # point
+        "coord_dimension": 2,
+        "srid": 4326,
+        "spatial_index_enabled": 0,
+    }
+
+
+def test_cli_add_geometry_column_options(tmpdir):
+    # create a rowid table with one column
+    db_path = tmpdir / "spatial.db"
+    db = Database(str(db_path))
+    db.init_spatialite()
+    table = db["locations"].create({"name": str})
+
+    result = CliRunner().invoke(
+        cli,
+        [
+            "add-geometry-column",
+            str(db_path),
+            table.name,
+            "geometry",
+            "-t",
+            "POLYGON",
+            "--srid",
+            "3857",  # https://epsg.io/3857
+            "--not-null",
+        ],
+    )
+
+    assert result.exit_code == 0
+
+    assert db["geometry_columns"].get(["locations", "geometry"]) == {
+        "f_table_name": "locations",
+        "f_geometry_column": "geometry",
+        "geometry_type": 3,  # polygon
+        "coord_dimension": 2,
+        "srid": 3857,
+        "spatial_index_enabled": 0,
+    }
+
+    column = table.columns[1]
+    assert column.notnull
+
+
+def test_cli_add_geometry_column_invalid_type(tmpdir):
+    # create a rowid table with one column
+    db_path = tmpdir / "spatial.db"
+    db = Database(str(db_path))
+    db.init_spatialite()
+
+    table = db["locations"].create({"name": str})
+
+    result = CliRunner().invoke(
+        cli,
+        [
+            "add-geometry-column",
+            str(db_path),
+            table.name,
+            "geometry",
+            "--type",
+            "NOT-A-TYPE",
+        ],
+    )
+
+    assert 2 == result.exit_code
+
+
+def test_cli_create_spatial_index(tmpdir):
+    # create a rowid table with one column
+    db_path = tmpdir / "spatial.db"
+    db = Database(str(db_path))
+    db.init_spatialite()
+
+    table = db["locations"].create({"name": str})
+    table.add_geometry_column("geometry", "POINT")
+
+    result = CliRunner().invoke(
+        cli, ["create-spatial-index", str(db_path), table.name, "geometry"]
+    )
+
+    assert result.exit_code == 0
+
+    assert "idx_locations_geometry" in db.table_names()

tests/test_insert_files.py

@@ -3,10 +3,18 @@ from click.testing import CliRunner
 import os
 import pathlib
 import pytest
+import sys
 
 
 @pytest.mark.parametrize("silent", (False, True))
-def test_insert_files(silent):
+@pytest.mark.parametrize(
+    "pk_args,expected_pks",
+    (
+        (["--pk", "path"], ["path"]),
+        (["--pk", "path", "--pk", "name"], ["path", "name"]),
+    ),
+)
+def test_insert_files(silent, pk_args, expected_pks):
     runner = CliRunner()
     with runner.isolated_filesystem():
         tmpdir = pathlib.Path(".")
@@ -14,7 +22,7 @@ def test_insert_files(silent):
         (tmpdir / "one.txt").write_text("This is file one", "utf-8")
         (tmpdir / "two.txt").write_text("Two is shorter", "utf-8")
         (tmpdir / "nested").mkdir()
-        (tmpdir / "nested" / "three.txt").write_text("Three is nested", "utf-8")
+        (tmpdir / "nested" / "three.zz.txt").write_text("Three is nested", "utf-8")
         coltypes = (
             "name",
             "path",
@@ -23,6 +31,7 @@ def test_insert_files(silent):
             "md5",
             "mode",
             "content",
+            "content_text",
             "mtime",
             "ctime",
             "mtime_int",
@@ -30,6 +39,8 @@ def test_insert_files(silent):
             "mtime_iso",
             "ctime_iso",
             "size",
+            "suffix",
+            "stem",
         )
         cols = []
         for coltype in coltypes:
@@ -38,7 +49,7 @@ def test_insert_files(silent):
             cli.cli,
             ["insert-files", db_path, "files", str(tmpdir)]
             + cols
-            + ["--pk", "path"]
+            + pk_args
             + (["--silent"] if silent else []),
             catch_exceptions=False,
         )
@@ -48,31 +59,40 @@ def test_insert_files(silent):
         one, two, three = (
             rows_by_path["one.txt"],
             rows_by_path["two.txt"],
-            rows_by_path[os.path.join("nested", "three.txt")],
+            rows_by_path[os.path.join("nested", "three.zz.txt")],
         )
         assert {
             "content": b"This is file one",
+            "content_text": "This is file one",
             "md5": "556dfb57fce9ca301f914e2273adf354",
             "name": "one.txt",
             "path": "one.txt",
             "sha256": "e34138f26b5f7368f298b4e736fea0aad87ddec69fbd04dc183b20f4d844bad5",
             "size": 16,
+            "stem": "one",
+            "suffix": ".txt",
         }.items() <= one.items()
         assert {
             "content": b"Two is shorter",
+            "content_text": "Two is shorter",
             "md5": "f86f067b083af1911043eb215e74ac70",
             "name": "two.txt",
             "path": "two.txt",
             "sha256": "9368988ed16d4a2da0af9db9b686d385b942cb3ffd4e013f43aed2ec041183d9",
             "size": 14,
+            "stem": "two",
+            "suffix": ".txt",
         }.items() <= two.items()
         assert {
             "content": b"Three is nested",
+            "content_text": "Three is nested",
             "md5": "12580f341781f5a5b589164d3cd39523",
-            "name": "three.txt",
-            "path": os.path.join("nested", "three.txt"),
+            "name": "three.zz.txt",
+            "path": os.path.join("nested", "three.zz.txt"),
             "sha256": "6dd45aaaaa6b9f96af19363a92c8fca5d34791d3c35c44eb19468a6a862cc8cd",
             "size": 15,
+            "stem": "three.zz",
+            "suffix": ".txt",
         }.items() <= three.items()
         # Assert the other int/str/float columns exist and are of the right types
         expected_types = {
@@ -84,24 +104,68 @@ def test_insert_files(silent):
             "mtime_iso": str,
             "mode": int,
             "fullpath": str,
+            "content": bytes,
+            "content_text": str,
+            "stem": str,
+            "suffix": str,
         }
         for colname, expected_type in expected_types.items():
             for row in (one, two, three):
                 assert isinstance(row[colname], expected_type)
+        assert set(db["files"].pks) == set(expected_pks)
 
 
-def test_insert_files_stdin():
+@pytest.mark.parametrize(
+    "use_text,encoding,input,expected",
+    (
+        (False, None, "hello world", b"hello world"),
+        (True, None, "hello world", "hello world"),
+        (False, None, b"S\xe3o Paulo", b"S\xe3o Paulo"),
+        (True, "latin-1", b"S\xe3o Paulo", "S\xe3o Paulo"),
+    ),
+)
+def test_insert_files_stdin(use_text, encoding, input, expected):
     runner = CliRunner()
     with runner.isolated_filesystem():
         tmpdir = pathlib.Path(".")
         db_path = str(tmpdir / "files.db")
+        args = ["insert-files", db_path, "files", "-", "--name", "stdin-name"]
+        if use_text:
+            args += ["--text"]
+        if encoding is not None:
+            args += ["--encoding", encoding]
         result = runner.invoke(
             cli.cli,
-            ["insert-files", db_path, "files", "-", "--name", "stdin-name"],
+            args,
             catch_exceptions=False,
-            input="hello world",
+            input=input,
         )
         assert result.exit_code == 0, result.stdout
         db = Database(db_path)
         row = list(db["files"].rows)[0]
-        assert {"path": "stdin-name", "content": b"hello world", "size": 11} == row
+        key = "content"
+        if use_text:
+            key = "content_text"
+        assert {"path": "stdin-name", key: expected}.items() <= row.items()
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Windows has a different way of handling default encodings",
+)
+def test_insert_files_bad_text_encoding_error():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        tmpdir = pathlib.Path(".")
+        latin = tmpdir / "latin.txt"
+        latin.write_bytes(b"S\xe3o Paulo")
+        db_path = str(tmpdir / "files.db")
+        result = runner.invoke(
+            cli.cli,
+            ["insert-files", db_path, "files", str(latin), "--text"],
+            catch_exceptions=False,
+        )
+        assert result.exit_code == 1, result.output
+        assert result.output.strip().startswith(
+            "Error: Could not read file '{}' as text".format(str(latin.resolve()))
+        )

tests/test_introspect.py

@@ -2,6 +2,14 @@ from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn
 import pytest
 
 
+def _check_supports_strict():
+    """Check if SQLite supports strict tables without leaking the database."""
+    db = Database(memory=True)
+    result = db.supports_strict
+    db.close()
+    return result
+
+
 def test_table_names(existing_db):
     assert ["foo"] == existing_db.table_names()
 
@@ -36,19 +44,36 @@ def test_detect_fts(existing_db):
     assert existing_db["foo"].detect_fts() is None
 
 
+@pytest.mark.parametrize("reverse_order", (True, False))
+def test_detect_fts_similar_tables(fresh_db, reverse_order):
+    # https://github.com/simonw/sqlite-utils/issues/434
+    table1, table2 = ("demo", "demo2")
+    if reverse_order:
+        table1, table2 = table2, table1
+
+    fresh_db[table1].insert({"title": "Hello"}).enable_fts(
+        ["title"], fts_version="FTS4"
+    )
+    fresh_db[table2].insert({"title": "Hello"}).enable_fts(
+        ["title"], fts_version="FTS4"
+    )
+    assert fresh_db[table1].detect_fts() == "{}_fts".format(table1)
+    assert fresh_db[table2].detect_fts() == "{}_fts".format(table2)
+
+
 def test_tables(existing_db):
-    assert 1 == len(existing_db.tables)
-    assert "foo" == existing_db.tables[0].name
+    assert len(existing_db.tables) == 1
+    assert existing_db.tables[0].name == "foo"
 
 
 def test_views(fresh_db):
     fresh_db.create_view("foo_view", "select 1")
-    assert 1 == len(fresh_db.views)
+    assert len(fresh_db.views) == 1
     view = fresh_db.views[0]
     assert isinstance(view, View)
-    assert "foo_view" == view.name
-    assert "<View foo_view (1)>" == repr(view)
-    assert {"1": str} == view.columns_dict
+    assert view.name == "foo_view"
+    assert repr(view) == "<View foo_view (1)>"
+    assert view.columns_dict == {"1": str}
 
 
 def test_count(existing_db):
@@ -84,13 +109,11 @@ def test_table_repr(fresh_db):
 
 
 def test_indexes(fresh_db):
-    fresh_db.executescript(
-        """
+    fresh_db.executescript("""
         create table Gosh (c1 text, c2 text, c3 text);
         create index Gosh_c1 on Gosh(c1);
         create index Gosh_c2c3 on Gosh(c2, c3);
-    """
-    )
+    """)
     assert [
         Index(
             seq=0,
@@ -105,13 +128,11 @@ def test_indexes(fresh_db):
 
 
 def test_xindexes(fresh_db):
-    fresh_db.executescript(
-        """
+    fresh_db.executescript("""
         create table Gosh (c1 text, c2 text, c3 text);
         create index Gosh_c1 on Gosh(c1);
         create index Gosh_c2c3 on Gosh(c2, c3 desc);
-    """
-    )
+    """)
     assert fresh_db["Gosh"].xindexes == [
         XIndex(
             name="Gosh_c2c3",
@@ -183,19 +204,19 @@ def test_triggers_and_triggers_dict(fresh_db):
     }
     expected_triggers = {
         "authors_ai": (
-            "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n"
-            "  INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\n"
+            'CREATE TRIGGER "authors_ai" AFTER INSERT ON "authors" BEGIN\n'
+            '  INSERT INTO "authors_fts" (rowid, "name", "famous_works") VALUES (new.rowid, new."name", new."famous_works");\n'
             "END"
         ),
         "authors_ad": (
-            "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n"
-            "  INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n"
+            'CREATE TRIGGER "authors_ad" AFTER DELETE ON "authors" BEGIN\n'
+            '  INSERT INTO "authors_fts" ("authors_fts", rowid, "name", "famous_works") VALUES(\'delete\', old.rowid, old."name", old."famous_works");\n'
             "END"
         ),
         "authors_au": (
-            "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n"
-            "  INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n"
-            "  INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND"
+            'CREATE TRIGGER "authors_au" AFTER UPDATE ON "authors" BEGIN\n'
+            '  INSERT INTO "authors_fts" ("authors_fts", rowid, "name", "famous_works") VALUES(\'delete\', old.rowid, old."name", old."famous_works");\n'
+            '  INSERT INTO "authors_fts" (rowid, "name", "famous_works") VALUES (new.rowid, new."name", new."famous_works");\nEND'
         ),
     }
     assert authors.triggers_dict == expected_triggers
@@ -252,15 +273,66 @@ def test_has_counts_triggers(fresh_db):
         ),
     ],
 )
-def test_virtual_table_using(sql, expected_name, expected_using):
-    db = Database(memory=True)
-    db.execute(sql)
-    assert db[expected_name].virtual_table_using == expected_using
+def test_virtual_table_using(fresh_db, sql, expected_name, expected_using):
+    fresh_db.execute(sql)
+    assert fresh_db[expected_name].virtual_table_using == expected_using
 
 
-def test_use_rowid():
-    db = Database(memory=True)
-    db["rowid_table"].insert({"name": "Cleo"})
-    db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id")
-    assert db["rowid_table"].use_rowid
-    assert not db["regular_table"].use_rowid
+def test_use_rowid(fresh_db):
+    fresh_db["rowid_table"].insert({"name": "Cleo"})
+    fresh_db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id")
+    assert fresh_db["rowid_table"].use_rowid
+    assert not fresh_db["regular_table"].use_rowid
+
+
+@pytest.mark.skipif(
+    not _check_supports_strict(),
+    reason="Needs SQLite version that supports strict",
+)
+@pytest.mark.parametrize(
+    "create_table,expected_strict",
+    (
+        ("create table t (id integer) strict", True),
+        ("create table t (id integer) STRICT", True),
+        ("create table t (id integer primary key) StriCt, WITHOUT ROWID", True),
+        ("create table t (id integer primary key) WITHOUT ROWID", False),
+        ("create table t (id integer)", False),
+    ),
+)
+def test_table_strict(fresh_db, create_table, expected_strict):
+    fresh_db.execute(create_table)
+    table = fresh_db["t"]
+    assert table.strict == expected_strict
+
+
+@pytest.mark.parametrize(
+    "value",
+    (
+        1,
+        1.3,
+        "foo",
+        True,
+        b"binary",
+    ),
+)
+def test_table_default_values(fresh_db, value):
+    fresh_db["default_values"].insert(
+        {"nodefault": 1, "value": value}, defaults={"value": value}
+    )
+    default_values = fresh_db["default_values"].default_values
+    assert default_values == {"value": value}
+
+
+def test_pks_use_primary_key_declaration_order(fresh_db):
+    # PRIMARY KEY (a, b) declared against columns stored in order (b, a) -
+    # pks must follow the declaration order, which is what SQLite uses to
+    # resolve implicit foreign key references and compound pk lookups
+    fresh_db.execute("create table t (b text, a text, primary key (a, b))")
+    assert fresh_db["t"].pks == ["a", "b"]
+
+
+def test_transform_preserves_compound_pk_declaration_order(fresh_db):
+    fresh_db.execute("create table t (a text, b text, c text, primary key (b, a))")
+    fresh_db["t"].transform(drop={"c"})
+    assert fresh_db["t"].pks == ["b", "a"]
+    assert 'PRIMARY KEY ("b", "a")' in fresh_db["t"].schema

tests/test_list_mode.py

@@ -0,0 +1,288 @@
+"""
+Tests for list-based iteration in insert_all and upsert_all
+"""
+
+import pytest
+from sqlite_utils import Database
+
+
+def test_insert_all_list_mode_basic():
+    """Test basic insert_all with list-based iteration"""
+    db = Database(memory=True)
+
+    def data_generator():
+        # First yield column names
+        yield ["id", "name", "age"]
+        # Then yield data rows
+        yield [1, "Alice", 30]
+        yield [2, "Bob", 25]
+        yield [3, "Charlie", 35]
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows)
+    assert len(rows) == 3
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": 25}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35}
+
+
+def test_insert_all_list_mode_with_pk():
+    """Test insert_all with list mode and primary key"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield ["id", "name", "score"]
+        yield [1, "Alice", 95]
+        yield [2, "Bob", 87]
+
+    db["scores"].insert_all(data_generator(), pk="id")
+
+    assert db["scores"].pks == ["id"]
+    rows = list(db["scores"].rows)
+    assert len(rows) == 2
+
+
+def test_upsert_all_list_mode():
+    """Test upsert_all with list-based iteration"""
+    db = Database(memory=True)
+
+    # Initial insert
+    def initial_data():
+        yield ["id", "name", "value"]
+        yield [1, "Alice", 100]
+        yield [2, "Bob", 200]
+
+    db["data"].insert_all(initial_data(), pk="id")
+
+    # Upsert with some updates and new records
+    def upsert_data():
+        yield ["id", "name", "value"]
+        yield [1, "Alice", 150]  # Update existing
+        yield [3, "Charlie", 300]  # Insert new
+
+    db["data"].upsert_all(upsert_data(), pk="id")
+
+    rows = list(db["data"].rows_where(order_by="id"))
+    assert len(rows) == 3
+    assert rows[0] == {"id": 1, "name": "Alice", "value": 150}
+    assert rows[1] == {"id": 2, "name": "Bob", "value": 200}
+    assert rows[2] == {"id": 3, "name": "Charlie", "value": 300}
+
+
+def test_list_mode_with_various_types():
+    """Test list mode with different data types"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield ["id", "name", "score", "active"]
+        yield [1, "Alice", 95.5, True]
+        yield [2, "Bob", 87.3, False]
+        yield [3, "Charlie", None, True]
+
+    db["mixed"].insert_all(data_generator())
+
+    rows = list(db["mixed"].rows)
+    assert len(rows) == 3
+    assert rows[0]["score"] == 95.5
+    assert rows[1]["active"] == 0  # SQLite stores boolean as int
+    assert rows[2]["score"] is None
+
+
+def test_list_mode_error_non_string_columns():
+    """Test that non-string column names raise an error"""
+    db = Database(memory=True)
+
+    def bad_data():
+        yield [1, 2, 3]  # Non-string column names
+        yield ["a", "b", "c"]
+
+    with pytest.raises(ValueError, match="must be a list of column name strings"):
+        db["bad"].insert_all(bad_data())
+
+
+def test_list_mode_error_mixed_types():
+    """Test that mixing list and dict raises an error"""
+    db = Database(memory=True)
+
+    def bad_data():
+        yield ["id", "name"]
+        yield {"id": 1, "name": "Alice"}  # Should be a list, not dict
+
+    with pytest.raises(ValueError, match="must also be lists"):
+        db["bad"].insert_all(bad_data())
+
+
+def test_list_mode_empty_after_headers():
+    """Test that only headers without data works gracefully"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield ["id", "name", "age"]
+        # No data rows
+
+    result = db["people"].insert_all(data_generator())
+    assert result is not None
+    assert not db["people"].exists()
+
+
+def test_list_mode_batch_processing():
+    """Test list mode with large dataset requiring batching"""
+    db = Database(memory=True)
+
+    def large_data():
+        yield ["id", "value"]
+        for i in range(1000):
+            yield [i, f"value_{i}"]
+
+    db["large"].insert_all(large_data(), batch_size=100)
+
+    count = db.execute("SELECT COUNT(*) as c FROM large").fetchone()[0]
+    assert count == 1000
+
+
+def test_list_mode_shorter_rows():
+    """Test that rows shorter than column list get NULL values"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield ["id", "name", "age", "city"]
+        yield [1, "Alice", 30, "NYC"]
+        yield [2, "Bob"]  # Missing age and city
+        yield [3, "Charlie", 35]  # Missing city
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows_where(order_by="id"))
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None}
+
+
+def test_backwards_compatibility_dict_mode():
+    """Ensure dict mode still works (backward compatibility)"""
+    db = Database(memory=True)
+
+    # Traditional dict-based insert
+    data = [
+        {"id": 1, "name": "Alice", "age": 30},
+        {"id": 2, "name": "Bob", "age": 25},
+    ]
+
+    db["people"].insert_all(data)
+
+    rows = list(db["people"].rows)
+    assert len(rows) == 2
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30}
+
+
+def test_insert_all_tuple_mode_basic():
+    """Test basic insert_all with tuple-based iteration"""
+    db = Database(memory=True)
+
+    def data_generator():
+        # First yield column names as tuple
+        yield ("id", "name", "age")
+        # Then yield data rows as tuples
+        yield (1, "Alice", 30)
+        yield (2, "Bob", 25)
+        yield (3, "Charlie", 35)
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows)
+    assert len(rows) == 3
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": 25}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35}
+
+
+def test_insert_all_mixed_list_tuple():
+    """Test insert_all with mixed lists and tuples for data rows"""
+    db = Database(memory=True)
+
+    def data_generator():
+        # Column names as list
+        yield ["id", "name", "age"]
+        # Mix of list and tuple data rows
+        yield [1, "Alice", 30]
+        yield (2, "Bob", 25)
+        yield [3, "Charlie", 35]
+        yield (4, "Diana", 40)
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows)
+    assert len(rows) == 4
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": 25}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35}
+    assert rows[3] == {"id": 4, "name": "Diana", "age": 40}
+
+
+def test_upsert_all_tuple_mode():
+    """Test upsert_all with tuple-based iteration"""
+    db = Database(memory=True)
+
+    # Initial insert with tuples
+    def initial_data():
+        yield ("id", "name", "value")
+        yield (1, "Alice", 100)
+        yield (2, "Bob", 200)
+
+    db["data"].insert_all(initial_data(), pk="id")
+
+    # Upsert with tuples
+    def upsert_data():
+        yield ("id", "name", "value")
+        yield (1, "Alice", 150)  # Update existing
+        yield (3, "Charlie", 300)  # Insert new
+
+    db["data"].upsert_all(upsert_data(), pk="id")
+
+    rows = list(db["data"].rows_where(order_by="id"))
+    assert len(rows) == 3
+    assert rows[0] == {"id": 1, "name": "Alice", "value": 150}
+    assert rows[1] == {"id": 2, "name": "Bob", "value": 200}
+    assert rows[2] == {"id": 3, "name": "Charlie", "value": 300}
+
+
+def test_tuple_mode_shorter_rows():
+    """Test that tuple rows shorter than column list get NULL values"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield "id", "name", "age", "city"
+        yield 1, "Alice", 30, "NYC"
+        yield 2, "Bob"  # Missing age and city
+        yield 3, "Charlie", 35  # Missing city
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows_where(order_by="id"))
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None}
+
+
+def test_list_mode_single_record_upsert_last_pk():
+    """Test that last_pk is populated correctly for single-record upserts in list mode"""
+    db = Database(memory=True)
+
+    # Create table first
+    db["data"].insert({"id": 1, "name": "Alice", "value": 100}, pk="id")
+
+    # Now upsert a single record using list mode
+    def upsert_data():
+        yield ["id", "name", "value"]
+        yield [1, "Alice", 150]  # Update existing
+
+    table = db["data"]
+    table.upsert_all(upsert_data(), pk="id")
+
+    # Verify the data was updated
+    rows = list(db["data"].rows)
+    assert rows == [{"id": 1, "name": "Alice", "value": 150}]
+
+    # Verify last_pk is populated correctly
+    assert table.last_pk == 1

tests/test_lookup.py

@@ -66,3 +66,118 @@ def test_lookup_fails_if_constraint_cannot_be_added(fresh_db):
     # This will fail because the name column is not unique
     with pytest.raises(Exception, match="UNIQUE constraint failed"):
         species.lookup({"name": "Palm"})
+
+
+def test_lookup_with_extra_values(fresh_db):
+    species = fresh_db["species"]
+    id = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2020-01-01"})
+    assert species.get(id) == {
+        "id": 1,
+        "name": "Palm",
+        "type": "Tree",
+        "first_seen": "2020-01-01",
+    }
+    # A subsequent lookup() should ignore the second dictionary
+    id2 = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2021-02-02"})
+    assert id2 == id
+    assert species.get(id2) == {
+        "id": 1,
+        "name": "Palm",
+        "type": "Tree",
+        "first_seen": "2020-01-01",
+    }
+
+
+def test_lookup_with_extra_insert_parameters(fresh_db):
+    other_table = fresh_db["other_table"]
+    other_table.insert({"id": 1, "name": "Name"}, pk="id")
+    species = fresh_db["species"]
+    id = species.lookup(
+        {"name": "Palm", "type": "Tree"},
+        {
+            "first_seen": "2020-01-01",
+            "make_not_null": 1,
+            "fk_to_other": 1,
+            "default_is_dog": "cat",
+            "extract_this": "This is extracted",
+            "convert_to_upper": "upper",
+            "make_this_integer": "2",
+            "this_at_front": 1,
+        },
+        pk="renamed_id",
+        foreign_keys=(("fk_to_other", "other_table", "id"),),
+        column_order=("this_at_front",),
+        not_null={"make_not_null"},
+        defaults={"default_is_dog": "dog"},
+        extracts=["extract_this"],
+        conversions={"convert_to_upper": "upper(?)"},
+        columns={"make_this_integer": int},
+    )
+    assert species.schema == (
+        'CREATE TABLE "species" (\n'
+        '   "renamed_id" INTEGER PRIMARY KEY,\n'
+        '   "this_at_front" INTEGER,\n'
+        '   "name" TEXT,\n'
+        '   "type" TEXT,\n'
+        '   "first_seen" TEXT,\n'
+        '   "make_not_null" INTEGER NOT NULL,\n'
+        '   "fk_to_other" INTEGER REFERENCES "other_table"("id"),\n'
+        "   \"default_is_dog\" TEXT DEFAULT 'dog',\n"
+        '   "extract_this" INTEGER REFERENCES "extract_this"("id"),\n'
+        '   "convert_to_upper" TEXT,\n'
+        '   "make_this_integer" INTEGER\n'
+        ")"
+    )
+    assert species.get(id) == {
+        "renamed_id": id,
+        "this_at_front": 1,
+        "name": "Palm",
+        "type": "Tree",
+        "first_seen": "2020-01-01",
+        "make_not_null": 1,
+        "fk_to_other": 1,
+        "default_is_dog": "cat",
+        "extract_this": 1,
+        "convert_to_upper": "UPPER",
+        "make_this_integer": 2,
+    }
+    assert species.indexes == [
+        Index(
+            seq=0,
+            name="idx_species_name_type",
+            unique=1,
+            origin="c",
+            partial=0,
+            columns=["name", "type"],
+        )
+    ]
+
+
+@pytest.mark.parametrize("strict", (False, True))
+def test_lookup_new_table_strict(fresh_db, strict):
+    fresh_db["species"].lookup({"name": "Palm"}, strict=strict)
+    assert fresh_db["species"].strict == strict or not fresh_db.supports_strict
+
+
+def test_lookup_null_value_idempotent(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/186
+    # Repeated lookups of a null value should return the same row,
+    # not insert a duplicate row each time
+    species = fresh_db["species"]
+    first_id = species.lookup({"name": None})
+    second_id = species.lookup({"name": None})
+    assert first_id == second_id
+    assert list(species.rows) == [{"id": first_id, "name": None}]
+
+
+def test_lookup_compound_key_with_null_idempotent(fresh_db):
+    species = fresh_db["species"]
+    palm_id = species.lookup({"name": "Palm", "type": None})
+    oak_id = species.lookup({"name": "Oak", "type": "Tree"})
+    assert palm_id == species.lookup({"name": "Palm", "type": None})
+    assert oak_id == species.lookup({"name": "Oak", "type": "Tree"})
+    assert palm_id != oak_id
+    assert list(species.rows) == [
+        {"id": palm_id, "name": "Palm", "type": None},
+        {"id": oak_id, "name": "Oak", "type": "Tree"},
+    ]

tests/test_m2m.py

@@ -109,9 +109,9 @@ def test_m2m_with_table_objects(fresh_db):
     )
     expected_tables = {"dogs", "humans", "dogs_humans"}
     assert expected_tables == set(fresh_db.table_names())
-    assert 1 == dogs.count
-    assert 2 == humans.count
-    assert 2 == fresh_db["dogs_humans"].count
+    assert dogs.count == 1
+    assert humans.count == 2
+    assert fresh_db["dogs_humans"].count == 2
 
 
 def test_m2m_lookup(fresh_db):
@@ -139,9 +139,9 @@ def test_m2m_lookup(fresh_db):
 
 def test_m2m_requires_either_records_or_lookup(fresh_db):
     people = fresh_db.table("people", pk="id").insert({"name": "Wahyu"})
-    with pytest.raises(AssertionError):
+    with pytest.raises(ValueError):
         people.m2m("tags")
-    with pytest.raises(AssertionError):
+    with pytest.raises(ValueError):
         people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"})
 
 

tests/test_migrations.py

@@ -0,0 +1,246 @@
+import pytest
+import sqlite_utils
+from sqlite_utils import Migrations
+
+
+@pytest.fixture
+def migrations():
+    migrations = Migrations("test")
+
+    @migrations()
+    def m001(db):
+        db["dogs"].insert({"name": "Cleo"})
+
+    @migrations()
+    def m002(db):
+        db["cats"].create({"name": str})
+        db.execute("insert into dogs (name) values ('Pancakes')")
+
+    return migrations
+
+
+@pytest.fixture
+def migrations_not_ordered_alphabetically():
+    # Names order alphabetically in the wrong direction but this
+    # should still be applied correctly.
+    migrations = Migrations("test")
+
+    @migrations()
+    def m002(db):
+        db["dogs"].insert({"name": "Cleo"})
+
+    @migrations()
+    def m001(db):
+        db["cats"].create({"name": str})
+        db.execute("insert into dogs (name) values ('Pancakes')")
+
+    return migrations
+
+
+@pytest.fixture
+def migrations2():
+    migrations = Migrations("test2")
+
+    @migrations()
+    def m001(db):
+        db["dogs2"].insert({"name": "Cleo"})
+
+    return migrations
+
+
+def test_basic(migrations):
+    db = sqlite_utils.Database(memory=True)
+    assert db.table_names() == []
+    migrations.apply(db)
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"}
+
+
+def test_stop_before(migrations):
+    db = sqlite_utils.Database(memory=True)
+    assert db.table_names() == []
+    migrations.apply(db, stop_before="m002")
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs"}
+    migrations.apply(db)
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"}
+
+
+def test_two_migration_sets(migrations, migrations2):
+    db = sqlite_utils.Database(memory=True)
+    assert db.table_names() == []
+    migrations.apply(db)
+    migrations2.apply(db)
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats", "dogs2"}
+
+
+def test_order_does_not_matter(migrations, migrations_not_ordered_alphabetically):
+    db1 = sqlite_utils.Database(memory=True)
+    db2 = sqlite_utils.Database(memory=True)
+    migrations.apply(db1)
+    migrations_not_ordered_alphabetically.apply(db2)
+    assert db1.schema == db2.schema
+
+
+def test_applied_at_is_a_string(migrations):
+    db = sqlite_utils.Database(memory=True)
+    migrations.apply(db)
+    applied = migrations.applied(db)
+    assert len(applied) == 2
+    for migration in applied:
+        # applied_at is the TEXT timestamp straight from the
+        # _sqlite_migrations table, e.g. "2026-07-04 12:00:00.000000+00:00"
+        assert isinstance(migration.applied_at, str)
+        assert migration.applied_at.endswith("+00:00")
+
+
+def test_failing_migration_rolls_back(migrations):
+    @migrations()
+    def m003(db):
+        db["birds"].create({"name": str})
+        db.execute("insert into dogs (name) values ('Dozer')")
+        raise ValueError("boom")
+
+    db = sqlite_utils.Database(memory=True)
+    with pytest.raises(ValueError):
+        migrations.apply(db)
+    # m001 and m002 committed before the failure and stay applied
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"}
+    assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"]
+    assert [m.name for m in migrations.applied(db)] == ["m001", "m002"]
+    # Everything m003 did was rolled back and it is still pending
+    assert [m.name for m in migrations.pending(db)] == ["m003"]
+
+
+def test_rerun_after_failure_applies_each_migration_once():
+    state = {"fail": True}
+    migrations = Migrations("test")
+
+    @migrations()
+    def m001(db):
+        db["dogs"].insert({"name": "Cleo"})
+
+    @migrations()
+    def m002(db):
+        db["dogs"].insert({"name": "Pancakes"})
+        if state["fail"]:
+            raise ValueError("boom")
+
+    db = sqlite_utils.Database(memory=True)
+    with pytest.raises(ValueError):
+        migrations.apply(db)
+    state["fail"] = False
+    migrations.apply(db)
+    # m001 must not have been re-applied, m002 applied exactly once
+    assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"]
+    assert [m.name for m in migrations.applied(db)] == ["m001", "m002"]
+
+
+def test_non_transactional_migration_allows_vacuum(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = sqlite_utils.Database(path)
+    migrations = Migrations("test")
+
+    @migrations()
+    def m001(db):
+        db["dogs"].insert({"name": "Cleo"})
+
+    @migrations(transactional=False)
+    def m002(db):
+        db.execute("VACUUM")
+
+    migrations.apply(db)
+    assert [m.name for m in migrations.applied(db)] == ["m001", "m002"]
+    db.close()
+
+
+def test_apply_composes_inside_outer_transaction(migrations):
+    db = sqlite_utils.Database(memory=True)
+    with pytest.raises(ZeroDivisionError):
+        with db.atomic():
+            migrations.apply(db)
+            raise ZeroDivisionError
+    # The outer transaction rolled back, taking the migrations with it
+    assert db.table_names() == []
+
+
+@pytest.mark.parametrize(
+    "create_table,pk",
+    (
+        (
+            {
+                "migration_set": str,
+                "name": str,
+                "applied_at": str,
+            },
+            "name",
+        ),
+        (
+            {
+                "migration_set": str,
+                "name": str,
+                "applied_at": str,
+            },
+            ("migration_set", "name"),
+        ),
+    ),
+)
+def test_upgrades_sqlite_migrations(migrations, create_table, pk):
+    db = sqlite_utils.Database(memory=True)
+    db["_sqlite_migrations"].create(create_table, pk=pk)
+    assert db.table_names() == ["_sqlite_migrations"]
+    assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk))
+    migrations.apply(db)
+    assert db["_sqlite_migrations"].pks == ["id"]
+
+
+def test_pending_and_applied_are_read_only(migrations):
+    db = sqlite_utils.Database(memory=True)
+    assert [m.name for m in migrations.pending(db)] == ["m001", "m002"]
+    assert migrations.applied(db) == []
+    # Neither call should have created the tracking table
+    assert db.table_names() == []
+
+
+def test_duplicate_migration_name_errors():
+    migrations = Migrations("test")
+
+    @migrations()
+    def m001(db):
+        pass
+
+    with pytest.raises(ValueError) as ex:
+
+        @migrations(name="m001")
+        def m001_again(db):
+            pass
+
+    assert "m001" in str(ex.value)
+
+
+def test_stop_before_applied_migration_errors(migrations):
+    # Stopping before a migration that has already been applied is
+    # impossible to honor - previously the stop name was only checked
+    # against pending migrations, so everything after it was applied
+    db = sqlite_utils.Database(memory=True)
+    migrations.apply(db, stop_before="m002")  # applies m001 only
+    with pytest.raises(ValueError) as ex:
+        migrations.apply(db, stop_before="m001")
+    assert "m001" in str(ex.value)
+    assert "already been applied" in str(ex.value)
+    # Nothing else was applied
+    assert not db["cats"].exists()
+
+
+def test_stop_before_applied_migration_errors_before_any_apply(migrations):
+    # The error fires before any pending migration runs, even those that
+    # come before the already-applied stop target in registration order
+    db = sqlite_utils.Database(memory=True)
+    only_second = Migrations("test")
+
+    @only_second()
+    def m002(db):
+        db["cats"].create({"name": str})
+
+    only_second.apply(db)  # m002 applied, m001 still pending
+    with pytest.raises(ValueError):
+        migrations.apply(db, stop_before="m002")
+    assert not db["dogs"].exists()

tests/test_plugins.py

@@ -0,0 +1,128 @@
+from click.testing import CliRunner
+import click
+import importlib
+import pytest
+import sys
+from sqlite_utils import cli, Database, hookimpl, plugins
+
+
+def _supports_pragma_function_list():
+    db = Database(memory=True)
+    try:
+        db.execute("select * from pragma_function_list()")
+        return True
+    except Exception:
+        return False
+    finally:
+        db.close()
+
+
+def test_get_plugins_loads_setuptools_entrypoints_once(monkeypatch):
+    calls = []
+    monkeypatch.delattr(sys, "_called_from_test", raising=False)
+    monkeypatch.setattr(plugins, "_plugins_loaded", False)
+    monkeypatch.setattr(
+        plugins.pm,
+        "load_setuptools_entrypoints",
+        lambda group: calls.append(group) or 0,
+    )
+
+    plugins.get_plugins()
+    plugins.get_plugins()
+
+    assert calls == ["sqlite_utils"]
+
+
+def test_get_plugins_does_not_load_setuptools_entrypoints_in_tests(monkeypatch):
+    calls = []
+    monkeypatch.setattr(sys, "_called_from_test", True, raising=False)
+    monkeypatch.setattr(plugins, "_plugins_loaded", False)
+    monkeypatch.setattr(
+        plugins.pm,
+        "load_setuptools_entrypoints",
+        lambda group: calls.append(group) or 0,
+    )
+
+    assert plugins.get_plugins() == []
+    assert calls == []
+
+
+def test_register_commands():
+    importlib.reload(cli)
+    assert plugins.get_plugins() == []
+
+    class HelloWorldPlugin:
+        __name__ = "HelloWorldPlugin"
+
+        @hookimpl
+        def register_commands(self, cli):
+            @cli.command(name="hello-world")
+            def hello_world():
+                "Print hello world"
+                click.echo("Hello world!")
+
+    try:
+        plugins.pm.register(HelloWorldPlugin(), name="HelloWorldPlugin")
+        importlib.reload(cli)
+
+        assert plugins.get_plugins() == [
+            {"name": "HelloWorldPlugin", "hooks": ["register_commands"]}
+        ]
+
+        runner = CliRunner()
+        result = runner.invoke(cli.cli, ["hello-world"])
+        assert result.exit_code == 0
+        assert result.output == "Hello world!\n"
+
+    finally:
+        plugins.pm.unregister(name="HelloWorldPlugin")
+        importlib.reload(cli)
+        assert plugins.get_plugins() == []
+
+
+@pytest.mark.skipif(
+    not _supports_pragma_function_list(),
+    reason="Needs SQLite version that supports pragma_function_list()",
+)
+def test_prepare_connection():
+    importlib.reload(cli)
+    assert plugins.get_plugins() == []
+
+    class HelloFunctionPlugin:
+        __name__ = "HelloFunctionPlugin"
+
+        @hookimpl
+        def prepare_connection(self, conn):
+            conn.create_function("hello", 1, lambda name: f"Hello, {name}!")
+
+    db = Database(memory=True)
+
+    def _functions(db):
+        return [
+            row[0]
+            for row in db.execute(
+                "select distinct name from pragma_function_list() order by 1"
+            ).fetchall()
+        ]
+
+    assert "hello" not in _functions(db)
+
+    try:
+        plugins.pm.register(HelloFunctionPlugin(), name="HelloFunctionPlugin")
+
+        assert plugins.get_plugins() == [
+            {"name": "HelloFunctionPlugin", "hooks": ["prepare_connection"]}
+        ]
+
+        db = Database(memory=True)
+        assert "hello" in _functions(db)
+        result = db.execute('select hello("world")').fetchone()[0]
+        assert result == "Hello, world!"
+
+        # Test execute_plugins=False
+        db2 = Database(memory=True, execute_plugins=False)
+        assert "hello" not in _functions(db2)
+
+    finally:
+        plugins.pm.unregister(name="HelloFunctionPlugin")
+        assert plugins.get_plugins() == []

tests/test_query.py

@@ -1,5 +1,8 @@
+import pytest
 import types
 
+from sqlite_utils.utils import sqlite3
+
 
 def test_query(fresh_db):
     fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}])
@@ -8,6 +11,268 @@ def test_query(fresh_db):
     assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}]
 
 
+def test_query_executes_eagerly(fresh_db):
+    # The SQL runs when query() is called, not when the result is iterated,
+    # so errors are raised at the call site
+    with pytest.raises(sqlite3.OperationalError):
+        fresh_db.query("select * from missing_table")
+
+
+def test_query_rejects_statements_that_return_no_rows(fresh_db):
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    with pytest.raises(ValueError) as ex:
+        fresh_db.query("update dogs set name = 'Cleopaws'")
+    assert "execute()" in str(ex.value)
+    # The rejected update was rolled back, and no transaction is left open
+    assert not fresh_db.conn.in_transaction
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
+
+
+def test_query_rejected_ddl_is_rolled_back(fresh_db):
+    with pytest.raises(ValueError):
+        fresh_db.query("create table dogs (id integer primary key)")
+    assert not fresh_db.conn.in_transaction
+    assert fresh_db.table_names() == []
+
+
+def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db):
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    fresh_db.begin()
+    fresh_db.execute("insert into dogs (name) values ('Pancakes')")
+    with pytest.raises(ValueError):
+        fresh_db.query("update dogs set name = 'Cleopaws'")
+    # The transaction is still open and the earlier insert is intact
+    assert fresh_db.conn.in_transaction
+    fresh_db.commit()
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo", "Pancakes"]
+
+
+@pytest.mark.parametrize(
+    "sql",
+    [
+        "begin",
+        "commit",
+        "rollback",
+        "vacuum",
+        "detach database foo",
+        "/* comment */ commit",
+        "-- comment\nbegin",
+        "/* multi\nline */ -- and another\n  vacuum",
+        "\t /* a */ /* b */ savepoint s1",
+        "; commit",
+        ";;\n ; rollback",
+        "; /* comment */ vacuum",
+        "\ufeffbegin",
+    ],
+)
+def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql):
+    with pytest.raises(ValueError) as ex:
+        fresh_db.query(sql)
+    assert "execute()" in str(ex.value)
+    assert not fresh_db.conn.in_transaction
+
+
+def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db):
+    # A COMMIT hidden behind a leading comment must not slip past the
+    # keyword check - previously it committed the caller's open
+    # transaction before the ValueError was raised
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    fresh_db.begin()
+    fresh_db.execute("insert into dogs (name) values ('Pancakes')")
+    with pytest.raises(ValueError):
+        fresh_db.query("/* comment */ COMMIT")
+    # The explicit transaction is still open and can still be rolled back
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
+
+
+@pytest.mark.parametrize("sql", ["; COMMIT", "\ufeffCOMMIT"])
+def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql):
+    # sqlite3 tolerates empty statements and a UTF-8 BOM before the first
+    # real token, so the keyword scanner must skip them too - previously
+    # '; COMMIT' slipped past the check and committed the caller's open
+    # transaction before raising OperationalError
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    fresh_db.begin()
+    fresh_db.execute("insert into dogs (name) values ('Pancakes')")
+    with pytest.raises(ValueError):
+        fresh_db.query(sql)
+    # The explicit transaction is still open and can still be rolled back
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
+
+
+def test_query_error_leaves_no_transaction_open(fresh_db):
+    with pytest.raises(sqlite3.OperationalError):
+        fresh_db.query("select * from missing_table")
+    assert not fresh_db.conn.in_transaction
+
+
+def test_query_pragma(tmpdir):
+    from sqlite_utils import Database
+
+    db = Database(str(tmpdir / "test.db"))
+    # A row-returning PRAGMA works, including one that cannot run in a transaction
+    assert list(db.query("pragma journal_mode = wal")) == [{"journal_mode": "wal"}]
+    # A PRAGMA that returns no rows raises ValueError
+    with pytest.raises(ValueError):
+        db.query("pragma user_version = 5")
+    db.close()
+
+
+def test_query_rejected_pragma_still_takes_effect(fresh_db):
+    # Documented limitation: PRAGMAs run outside the savepoint guard,
+    # because some of them refuse to run inside a transaction - so a
+    # row-less PRAGMA takes effect even though it raises ValueError.
+    # If this test starts failing because the pragma was rolled back,
+    # the limitation has been fixed - update the docs in python-api.rst
+    # and the query() docstring to remove the carve-out
+    with pytest.raises(ValueError):
+        fresh_db.query("pragma user_version = 5")
+    assert fresh_db.execute("pragma user_version").fetchone()[0] == 5
+
+
+def test_query_comment_prefixed_pragma(tmpdir):
+    from sqlite_utils import Database
+
+    db = Database(str(tmpdir / "test.db"))
+    # A leading comment must not stop a PRAGMA being recognized as one -
+    # previously it was executed inside the savepoint guard, where
+    # journal mode changes are refused
+    assert list(db.query("-- set WAL mode\npragma journal_mode = wal")) == [
+        {"journal_mode": "wal"}
+    ]
+    db.close()
+
+
+def test_query_comment_prefixed_pragma_inside_transaction(fresh_db):
+    fresh_db.begin()
+    assert list(fresh_db.query("-- check version\npragma user_version")) == [
+        {"user_version": 0}
+    ]
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+
+
+@pytest.mark.parametrize(
+    "sql,expected",
+    [
+        ("select 1", "SELECT"),
+        ("  \t\n select 1", "SELECT"),
+        ("-- comment\nbegin", "BEGIN"),
+        ("/* one */ /* two */ pragma user_version", "PRAGMA"),
+        ("/* multi\nline */vacuum", "VACUUM"),
+        ("insert into t values (1)", "INSERT"),
+        ("-- only a comment", ""),
+        ("/* unterminated", ""),
+        ("", ""),
+        ("   ", ""),
+        ("123", ""),
+        ("; commit", "COMMIT"),
+        (";;\n ; rollback", "ROLLBACK"),
+        ("; -- comment\n begin", "BEGIN"),
+        ("\ufeffcommit", "COMMIT"),
+        ("\ufeff ; select 1", "SELECT"),
+        (";", ""),
+    ],
+)
+def test_first_keyword(sql, expected):
+    from sqlite_utils.db import _first_keyword
+
+    assert _first_keyword(sql) == expected
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_insert_returning(fresh_db):
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    rows = list(
+        fresh_db.query("insert into dogs (name) values ('Pancakes') returning name")
+    )
+    assert rows == [{"name": "Pancakes"}]
+    assert fresh_db["dogs"].count == 2
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_insert_returning_commits_without_iteration(tmpdir):
+    from sqlite_utils import Database
+
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["dogs"].insert({"name": "Cleo"})
+    # Never iterate over the results
+    db.query("insert into dogs (name) values ('Pancakes') returning name")
+    assert not db.conn.in_transaction
+    # A completely separate connection sees the new row straight away
+    other = sqlite3.connect(path)
+    assert other.execute("select count(*) from dogs").fetchone()[0] == 2
+    other.close()
+    db.close()
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_insert_returning_partial_iteration_still_commits(tmpdir):
+    from sqlite_utils import Database
+
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["dogs"].insert({"name": "Cleo"})
+    row = next(
+        db.query(
+            "insert into dogs (name) values ('Pancakes'), ('Marnie') returning name"
+        )
+    )
+    assert row == {"name": "Pancakes"}
+    assert not db.conn.in_transaction
+    other = sqlite3.connect(path)
+    assert other.execute("select count(*) from dogs").fetchone()[0] == 3
+    other.close()
+    db.close()
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_insert_returning_respects_explicit_transaction(fresh_db):
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    fresh_db.begin()
+    rows = list(
+        fresh_db.query("insert into dogs (name) values ('Pancakes') returning name")
+    )
+    assert rows == [{"name": "Pancakes"}]
+    # Still inside the explicit transaction - not committed
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
+
+
+def test_query_duplicate_column_names_are_deduped(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/624
+    fresh_db["one"].insert({"id": 1, "value": "left"})
+    fresh_db["two"].insert({"id": 2, "value": "right"})
+    rows = list(
+        fresh_db.query("select one.id, two.id, one.value, two.value from one, two")
+    )
+    assert rows == [{"id": 1, "id_2": 2, "value": "left", "value_2": "right"}]
+
+
+def test_query_deduped_column_avoids_existing_names(fresh_db):
+    # The renamed duplicate must not overwrite a real column called id_2
+    rows = list(fresh_db.query("select 1 as id, 2 as id, 3 as id_2"))
+    assert rows == [{"id": 1, "id_3": 2, "id_2": 3}]
+
+
 def test_execute_returning_dicts(fresh_db):
     # Like db.query() but returns a list, included for backwards compatibility
     # see https://github.com/simonw/sqlite-utils/issues/290
@@ -15,3 +280,24 @@ def test_execute_returning_dicts(fresh_db):
     assert fresh_db.execute_returning_dicts("select * from test") == [
         {"id": 1, "bar": 2}
     ]
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_preserves_error_from_transaction_destroying_trigger(fresh_db):
+    # RAISE(ROLLBACK) destroys the savepoint guard - the original
+    # IntegrityError must propagate, not "no such savepoint"
+    fresh_db.execute("create table t (id integer primary key, v text)")
+    fresh_db.execute("""
+        create trigger no_bad before insert on t
+        when new.v = 'bad'
+        begin
+            select raise(rollback, 'trigger says no');
+        end
+    """)
+    with pytest.raises(sqlite3.IntegrityError, match="trigger says no"):
+        fresh_db.query("insert into t (id, v) values (1, 'bad') returning id")
+    assert not fresh_db.conn.in_transaction
+    assert fresh_db.execute("select count(*) from t").fetchone()[0] == 0

tests/test_recipes.py

@@ -1,4 +1,5 @@
 from sqlite_utils import recipes
+from sqlite_utils.utils import sqlite3
 import json
 import pytest
 
@@ -61,6 +62,39 @@ def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected):
     ]
 
 
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
+@pytest.mark.parametrize("fn", ("parsedate", "parsedatetime"))
+def test_dateparse_errors_raises(fresh_db, fn):
+    """Test that invalid dates raise errors when errors=None"""
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "dt": "invalid"},
+        ],
+        pk="id",
+    )
+    # Exception in SQLite callback surfaces as OperationalError
+    with pytest.raises(sqlite3.OperationalError):
+        fresh_db["example"].convert("dt", lambda value: getattr(recipes, fn)(value))
+
+
+@pytest.mark.parametrize("fn", ("parsedate", "parsedatetime"))
+@pytest.mark.parametrize("errors", (recipes.SET_NULL, recipes.IGNORE))
+def test_dateparse_errors_handled(fresh_db, fn, errors):
+    """Test error handling modes for invalid dates"""
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "dt": "invalid"},
+        ],
+        pk="id",
+    )
+    fresh_db["example"].convert(
+        "dt", lambda value: getattr(recipes, fn)(value, errors=errors)
+    )
+    rows = list(fresh_db["example"].rows)
+    expected = [{"id": 1, "dt": None if errors is recipes.SET_NULL else "invalid"}]
+    assert rows == expected
+
+
 @pytest.mark.parametrize("delimiter", [None, ";", "-"])
 def test_jsonsplit(fresh_db, delimiter):
     fresh_db["example"].insert_all(
@@ -70,12 +104,14 @@ def test_jsonsplit(fresh_db, delimiter):
         ],
         pk="id",
     )
-    fn = recipes.jsonsplit
     if delimiter is not None:
 
         def fn(value):
             return recipes.jsonsplit(value, delimiter=delimiter)
 
+    else:
+        fn = recipes.jsonsplit
+
     fresh_db["example"].convert("tags", fn)
     assert list(fresh_db["example"].rows) == [
         {"id": 1, "tags": '["foo", "bar"]'},
@@ -98,11 +134,13 @@ def test_jsonsplit_type(fresh_db, type, expected):
         ],
         pk="id",
     )
-    fn = recipes.jsonsplit
     if type is not None:
 
         def fn(value):
             return recipes.jsonsplit(value, type=type)
 
+    else:
+        fn = recipes.jsonsplit
+
     fresh_db["example"].convert("records", fn)
     assert json.loads(fresh_db["example"].get(1)["records"]) == expected

tests/test_recreate.py

@@ -14,19 +14,25 @@ def test_recreate_ignored_for_in_memory():
 
 def test_recreate_not_allowed_for_connection():
     conn = sqlite3.connect(":memory:")
-    with pytest.raises(AssertionError):
-        Database(conn, recreate=True)
+    try:
+        with pytest.raises(ValueError):
+            Database(conn, recreate=True)
+    finally:
+        conn.close()
 
 
 @pytest.mark.parametrize(
-    "use_path,file_exists", [(True, True), (True, False), (False, True), (False, False)]
+    "use_path,create_file_first",
+    [(True, True), (True, False), (False, True), (False, False)],
 )
-def test_recreate(tmpdir, use_path, file_exists):
-    filepath = str(tmpdir / "data.db")
+def test_recreate(tmp_path, use_path, create_file_first):
+    filepath = str(tmp_path / "data.db")
     if use_path:
         filepath = pathlib.Path(filepath)
-    if file_exists:
-        Database(filepath)["t1"].insert({"foo": "bar"})
-        assert ["t1"] == Database(filepath).table_names()
+    if create_file_first:
+        db = Database(filepath)
+        db["t1"].insert({"foo": "bar"})
+        assert ["t1"] == db.table_names()
+        db.close()
     Database(filepath, recreate=True)["t2"].insert({"foo": "bar"})
     assert ["t2"] == Database(filepath).table_names()

tests/test_register_function.py

@@ -1,7 +1,8 @@
 # flake8: noqa
 import pytest
 import sys
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, call
+from sqlite_utils.utils import sqlite3
 
 
 def test_register_function(fresh_db):
@@ -13,6 +14,15 @@ def test_register_function(fresh_db):
     assert result == "olleh"
 
 
+def test_register_function_custom_name(fresh_db):
+    @fresh_db.register_function(name="revstr")
+    def reverse_string(s):
+        return "".join(reversed(list(s)))
+
+    result = fresh_db.execute('select revstr("hello")').fetchone()[0]
+    assert result == "olleh"
+
+
 def test_register_function_multiple_arguments(fresh_db):
     @fresh_db.register_function
     def a_times_b_plus_c(a, b, c):
@@ -31,20 +41,47 @@ def test_register_function_deterministic(fresh_db):
     assert result == "bob"
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 8), reason="deterministic=True was added in Python 3.8"
-)
-def test_register_function_deterministic_registered(fresh_db):
+def test_register_function_deterministic_tries_again_if_exception_raised(fresh_db):
+    # Save the original connection so we can close it later
+    original_conn = fresh_db.conn
     fresh_db.conn = MagicMock()
     fresh_db.conn.create_function = MagicMock()
 
-    @fresh_db.register_function(deterministic=True)
-    def to_lower_2(s):
-        return s.lower()
+    try:
 
-    fresh_db.conn.create_function.assert_called_with(
-        "to_lower_2", 1, to_lower_2, deterministic=True
-    )
+        @fresh_db.register_function(deterministic=True)
+        def to_lower_2(s):
+            return s.lower()
+
+        fresh_db.conn.create_function.assert_called_with(
+            "to_lower_2", 1, to_lower_2, deterministic=True
+        )
+
+        first = True
+
+        def side_effect(*args, **kwargs):
+            # Raise exception only first time this is called
+            nonlocal first
+            if first:
+                first = False
+                raise sqlite3.NotSupportedError()
+
+        # But if sqlite3.NotSupportedError is raised, it tries again
+        fresh_db.conn.create_function.reset_mock()
+        fresh_db.conn.create_function.side_effect = side_effect
+
+        @fresh_db.register_function(deterministic=True)
+        def to_lower_3(s):
+            return s.lower()
+
+        # Should have been called once with deterministic=True and once without
+        assert fresh_db.conn.create_function.call_args_list == [
+            call("to_lower_3", 1, to_lower_3, deterministic=True),
+            call("to_lower_3", 1, to_lower_3),
+        ]
+    finally:
+        # Close the original connection that was replaced with the mock
+        original_conn.close()
 
 
 def test_register_function_replace(fresh_db):
@@ -54,7 +91,7 @@ def test_register_function_replace(fresh_db):
 
     assert "one" == fresh_db.execute("select one()").fetchone()[0]
 
-    # This will fail to replace the function:
+    # This will silently fail to replaec the function
     @fresh_db.register_function()
     def one():  # noqa
         return "two"

tests/test_rows.py

@@ -104,3 +104,37 @@ def test_pks_and_rows_where_compound_pk(fresh_db):
         (("number", 1), {"type": "number", "number": 1, "plusone": 2}),
         (("number", 2), {"type": "number", "number": 2, "plusone": 3}),
     ]
+
+
+def test_rows_where_duplicate_select_columns_are_deduped(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/624
+    fresh_db["t"].insert({"id": 1, "name": "Cleo"})
+    rows = list(fresh_db["t"].rows_where(select="id, id, name"))
+    assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}]
+
+
+def test_pks_and_rows_where_view(fresh_db):
+    # pks_and_rows_where() lives on Queryable so views expose it, but
+    # SQLite views have no rowid. Modern SQLite (3.36+) raises an
+    # OperationalError from the generated SQL; older versions returned
+    # NULL for a view's rowid. Either way it must not fail earlier with
+    # an AttributeError from View lacking Table-only properties
+    from sqlite_utils.utils import sqlite3
+
+    fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db.create_view("dog_names", "select name from dogs")
+    try:
+        result = list(fresh_db["dog_names"].pks_and_rows_where())
+    except sqlite3.OperationalError:
+        pass  # SQLite 3.36+: no such column: rowid
+    else:
+        # Older SQLite returns NULL rowids for views
+        assert result == [(None, {"rowid": None, "name": "Cleo"})]
+
+
+def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db):
+    # Compound pks are returned in PRIMARY KEY declaration order
+    fresh_db.execute("create table t (b text, a text, primary key (a, b))")
+    fresh_db["t"].insert({"a": "A", "b": "B"})
+    pks_and_rows = list(fresh_db["t"].pks_and_rows_where())
+    assert pks_and_rows == [(("A", "B"), {"b": "B", "a": "A"})]

tests/test_rows_from_file.py

@@ -0,0 +1,54 @@
+from sqlite_utils.utils import rows_from_file, Format, RowError
+from io import BytesIO, StringIO
+import pytest
+
+
+@pytest.mark.parametrize(
+    "input,expected_format",
+    (
+        (b"id,name\n1,Cleo", Format.CSV),
+        (b"id\tname\n1\tCleo", Format.TSV),
+        (b'[{"id": "1", "name": "Cleo"}]', Format.JSON),
+    ),
+)
+def test_rows_from_file_detect_format(input, expected_format):
+    rows, format = rows_from_file(BytesIO(input))
+    assert format == expected_format
+    rows_list = list(rows)
+    assert rows_list == [{"id": "1", "name": "Cleo"}]
+
+
+@pytest.mark.parametrize(
+    "ignore_extras,extras_key,expected",
+    (
+        (True, None, [{"id": "1", "name": "Cleo"}]),
+        (False, "_rest", [{"id": "1", "name": "Cleo", "_rest": ["oops"]}]),
+        # expected of None means expect an error:
+        (False, False, None),
+    ),
+)
+def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expected):
+    try:
+        rows, format = rows_from_file(
+            BytesIO(b"id,name\r\n1,Cleo,oops"),
+            format=Format.CSV,
+            ignore_extras=ignore_extras,
+            extras_key=extras_key,
+        )
+        list_rows = list(rows)
+    except RowError:
+        if expected is None:
+            # This is fine,
+            return
+        else:
+            # We did not expect an error
+            raise
+    assert list_rows == expected
+
+
+def test_rows_from_file_error_on_string_io():
+    with pytest.raises(TypeError) as ex:
+        rows_from_file(StringIO("id,name\r\n1,Cleo"))  # type: ignore[arg-type]
+    assert ex.value.args == (
+        "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO",
+    )

tests/test_sniff.py

@@ -6,13 +6,13 @@ import pytest
 sniff_dir = pathlib.Path(__file__).parent / "sniff"
 
 
-@pytest.mark.parametrize("filepath", sniff_dir.glob("example*"))
+@pytest.mark.parametrize("filepath", sorted(sniff_dir.glob("example*")))
 def test_sniff(tmpdir, filepath):
     db_path = str(tmpdir / "test.db")
     runner = CliRunner()
     result = runner.invoke(
         cli.cli,
-        ["insert", db_path, "creatures", str(filepath), "--sniff"],
+        ["insert", db_path, "creatures", str(filepath), "--sniff", "--no-detect-types"],
         catch_exceptions=False,
     )
     assert result.exit_code == 0, result.stdout

tests/test_tracer.py

@@ -6,26 +6,29 @@ def test_tracer():
     db = Database(
         memory=True, tracer=lambda sql, params: collected.append((sql, params))
     )
-    db["dogs"].insert({"name": "Cleopaws"})
-    db["dogs"].enable_fts(["name"])
-    db["dogs"].search("Cleopaws")
+    dogs = db.table("dogs")
+    dogs.insert({"name": "Cleopaws"})
+    dogs.enable_fts(["name"])
+    dogs.search("Cleopaws")
     assert collected == [
         ("PRAGMA recursive_triggers=on;", None),
         ("select name from sqlite_master where type = 'view'", None),
         ("select name from sqlite_master where type = 'table'", None),
-        ("CREATE TABLE [dogs] (\n   [name] TEXT\n);\n        ", None),
         ("select name from sqlite_master where type = 'view'", None),
-        ("INSERT INTO [dogs] ([name]) VALUES (?);", ["Cleopaws"]),
         ("select name from sqlite_master where type = 'view'", None),
+        ("select name from sqlite_master where type = 'table'", None),
+        ("select name from sqlite_master where type = 'view'", None),
+        ('CREATE TABLE "dogs" (\n   "name" TEXT\n);\n        ', None),
+        ("select name from sqlite_master where type = 'view'", None),
+        ('INSERT INTO "dogs" ("name") VALUES (?)', ["Cleopaws"]),
         (
-            "CREATE VIRTUAL TABLE [dogs_fts] USING FTS5 (\n    [name],\n    content=[dogs]\n)",
+            'CREATE VIRTUAL TABLE "dogs_fts" USING FTS5 (\n    "name",\n    content="dogs"\n)',
             None,
         ),
         (
-            "INSERT INTO [dogs_fts] (rowid, [name])\n    SELECT rowid, [name] FROM [dogs];",
+            'INSERT INTO "dogs_fts" (rowid, "name")\n    SELECT rowid, "name" FROM "dogs";',
             None,
         ),
-        ("select name from sqlite_master where type = 'view'", None),
     ]
 
 
@@ -37,55 +40,57 @@ def test_with_tracer():
 
     db = Database(memory=True)
 
-    db["dogs"].insert({"name": "Cleopaws"})
-    db["dogs"].enable_fts(["name"])
+    dogs = db.table("dogs")
+
+    dogs.insert({"name": "Cleopaws"})
+    dogs.enable_fts(["name"])
 
     assert len(collected) == 0
 
     with db.tracer(tracer):
-        list(db["dogs"].search("Cleopaws"))
+        list(dogs.search("Cleopaws"))
 
-    assert len(collected) == 5
+    assert len(collected) == 4
     assert collected == [
-        ("select name from sqlite_master where type = 'view'", None),
         (
-            (
-                "SELECT name FROM sqlite_master\n"
-                "    WHERE rootpage = 0\n"
-                "    AND (\n"
-                "        sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n"
-                "        OR (\n"
-                '            tbl_name = "dogs"\n'
-                "            AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n"
-                "        )\n"
-                "    )"
-            ),
-            None,
+            "SELECT name FROM sqlite_master\n"
+            "    WHERE rootpage = 0\n"
+            "    AND (\n"
+            "        sql LIKE :like\n"
+            "        OR sql LIKE :like2\n"
+            "        OR (\n"
+            "            tbl_name = :table\n"
+            "            AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n"
+            "        )\n"
+            "    )",
+            {
+                "like": "%VIRTUAL TABLE%USING FTS%content=[dogs]%",
+                "like2": '%VIRTUAL TABLE%USING FTS%content="dogs"%',
+                "table": "dogs",
+            },
         ),
         ("select name from sqlite_master where type = 'view'", None),
         ("select sql from sqlite_master where name = ?", ("dogs_fts",)),
         (
-            (
-                "with original as (\n"
-                "    select\n"
-                "        rowid,\n"
-                "        *\n"
-                "    from [dogs]\n"
-                ")\n"
-                "select\n"
-                "    [original].*\n"
-                "from\n"
-                "    [original]\n"
-                "    join [dogs_fts] on [original].rowid = [dogs_fts].rowid\n"
-                "where\n"
-                "    [dogs_fts] match :query\n"
-                "order by\n"
-                "    [dogs_fts].rank"
-            ),
+            'with "original" as (\n'
+            "    select\n"
+            "        rowid,\n"
+            "        *\n"
+            '    from "dogs"\n'
+            ")\n"
+            "select\n"
+            '    "original".*\n'
+            "from\n"
+            '    "original"\n'
+            '    join "dogs_fts" on "original".rowid = "dogs_fts".rowid\n'
+            "where\n"
+            '    "dogs_fts" match :query\n'
+            "order by\n"
+            '    "dogs_fts".rank',
             {"query": "Cleopaws"},
         ),
     ]
 
     # Outside the with block collected should not be appended to
-    db["dogs"].insert({"name": "Cleopaws"})
-    assert len(collected) == 5
+    dogs.insert({"name": "Cleopaws"})
+    assert len(collected) == 4

tests/test_transform.py

@@ -1,4 +1,6 @@
-from sqlite_utils.db import ForeignKey
+import sqlite3
+
+from sqlite_utils.db import ForeignKey, TransactionError, TransformError
 from sqlite_utils.utils import OperationalError
 import pytest
 
@@ -10,80 +12,90 @@ import pytest
         (
             {},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] TEXT\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Change column type
         (
             {"types": {"age": int}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] INTEGER\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" INTEGER\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Rename a column
         (
             {"rename": {"age": "dog_age"}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [dog_age] TEXT\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" TEXT\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Drop a column
         (
             {"drop": ["age"]},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name])\n   SELECT [id], [name] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n   SELECT "rowid", "id", "name" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Convert type AND rename column
         (
             {"types": {"age": int}, "rename": {"age": "dog_age"}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [dog_age] INTEGER\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" INTEGER\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Change primary key
         (
             {"pk": "age"},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT PRIMARY KEY\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT PRIMARY KEY\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Change primary key to a compound pk
         (
             {"pk": ("age", "name")},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT,\n   PRIMARY KEY ([age], [name])\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT,\n   PRIMARY KEY ("age", "name")\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Remove primary key, creating a rowid table
         (
             {"pk": None},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+            ],
+        ),
+        # Keeping the table
+        (
+            {"drop": ["age"], "keep_table": "kept_table"},
+            [
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n   SELECT "rowid", "id", "name" FROM "dogs";',
+                'ALTER TABLE "dogs" RENAME TO "kept_table";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
     ],
@@ -123,40 +135,40 @@ def test_transform_sql_table_with_primary_key(
         (
             {},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Change column type
         (
             {"types": {"age": int}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] INTEGER\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" INTEGER\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Rename a column
         (
             {"rename": {"age": "dog_age"}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [dog_age] TEXT\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "dog_age" TEXT\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Make ID a primary key
         (
             {"pk": "id"},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] TEXT\n);",
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
-                "DROP TABLE [dogs];",
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'DROP TABLE "dogs";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
     ],
@@ -194,13 +206,13 @@ def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db):
     dogs.insert({"id": 1, "name": "Cleo", "age": "5"})
     assert (
         dogs.schema
-        == "CREATE TABLE [dogs] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT\n)"
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT\n)'
     )
     dogs.transform(pk="id")
     # Slight oddity: [dogs] becomes "dogs" during the rename:
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] TEXT\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n)'
     )
 
 
@@ -210,17 +222,51 @@ def test_transform_rename_pk(fresh_db):
     dogs.transform(rename={"id": "pk"})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [pk] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] TEXT\n)'
+        == 'CREATE TABLE "dogs" (\n   "pk" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n)'
     )
 
 
+def test_transform_preserves_keyword_literal_defaults(fresh_db):
+    # transform() used to requote keyword-literal defaults (DEFAULT TRUE became
+    # DEFAULT 'TRUE'), so a default insert stored the text 'TRUE' instead of the
+    # integer 1 -- silent value corruption on every rebuilt table.
+    fresh_db.execute(
+        "CREATE TABLE t ("
+        " id INTEGER PRIMARY KEY,"
+        " is_active INTEGER DEFAULT TRUE,"
+        " flag INTEGER DEFAULT FALSE,"
+        " note TEXT DEFAULT NULL"
+        ")"
+    )
+    table = fresh_db["t"]
+    table.insert({"id": 1})
+    before = fresh_db.execute("SELECT is_active, flag, note FROM t").fetchone()
+    assert before == (1, 0, None)
+
+    # Rebuild the table via an unrelated change.
+    table.transform(rename={"note": "note2"})
+
+    # The keyword literals stay unquoted in the schema ...
+    assert "DEFAULT TRUE" in table.schema
+    assert "DEFAULT FALSE" in table.schema
+    assert "DEFAULT NULL" in table.schema
+    assert "'TRUE'" not in table.schema
+
+    # ... and a fresh default insert still yields 1 / 0 / NULL, not strings.
+    table.insert({"id": 2})
+    after = fresh_db.execute(
+        "SELECT is_active, flag, note2 FROM t WHERE id = 2"
+    ).fetchone()
+    assert after == (1, 0, None)
+
+
 def test_transform_not_null(fresh_db):
     dogs = fresh_db["dogs"]
     dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id")
     dogs.transform(not_null={"name"})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT NOT NULL,\n   [age] TEXT\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT NOT NULL,\n   "age" TEXT\n)'
     )
 
 
@@ -230,7 +276,7 @@ def test_transform_remove_a_not_null(fresh_db):
     dogs.transform(not_null={"name": True, "age": False})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT NOT NULL,\n   [age] TEXT\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT NOT NULL,\n   "age" TEXT\n)'
     )
 
 
@@ -241,7 +287,7 @@ def test_transform_add_not_null_with_rename(fresh_db, not_null):
     dogs.transform(not_null=not_null, rename={"age": "dog_age"})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [dog_age] TEXT NOT NULL\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" TEXT NOT NULL\n)'
     )
 
 
@@ -251,7 +297,7 @@ def test_transform_defaults(fresh_db):
     dogs.transform(defaults={"age": 1})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] INTEGER DEFAULT 1\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" INTEGER DEFAULT 1\n)'
     )
 
 
@@ -261,7 +307,7 @@ def test_transform_defaults_and_rename_column(fresh_db):
     dogs.transform(rename={"age": "dog_age"}, defaults={"age": 1})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [dog_age] INTEGER DEFAULT 1\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" INTEGER DEFAULT 1\n)'
     )
 
 
@@ -271,7 +317,7 @@ def test_remove_defaults(fresh_db):
     dogs.transform(defaults={"age": None})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] INTEGER\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" INTEGER\n)'
     )
 
 
@@ -319,22 +365,29 @@ def test_transform_foreign_keys_survive_renamed_column(
     ]
 
 
+def _add_country_city_continent(db):
+    db["country"].insert({"id": 1, "name": "France"}, pk="id")
+    db["continent"].insert({"id": 2, "name": "Europe"}, pk="id")
+    db["city"].insert({"id": 24, "name": "Paris"}, pk="id")
+
+
+_CAVEAU = {
+    "id": 32,
+    "name": "Caveau de la Huchette",
+    "country": 1,
+    "continent": 2,
+    "city": 24,
+}
+
+
 @pytest.mark.parametrize("use_pragma_foreign_keys", [False, True])
 def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys):
     if use_pragma_foreign_keys:
         fresh_db.conn.execute("PRAGMA foreign_keys=ON")
     # Create table with three foreign keys so we can drop two of them
-    fresh_db["country"].insert({"id": 1, "name": "France"}, pk="id")
-    fresh_db["continent"].insert({"id": 2, "name": "Europe"}, pk="id")
-    fresh_db["city"].insert({"id": 24, "name": "Paris"}, pk="id")
+    _add_country_city_continent(fresh_db)
     fresh_db["places"].insert(
-        {
-            "id": 32,
-            "name": "Caveau de la Huchette",
-            "country": 1,
-            "continent": 2,
-            "city": 24,
-        },
+        _CAVEAU,
         foreign_keys=("country", "continent", "city"),
     )
     assert fresh_db["places"].foreign_keys == [
@@ -374,6 +427,480 @@ def test_transform_verify_foreign_keys(fresh_db):
     # This should have rolled us back
     assert (
         fresh_db["authors"].schema
-        == "CREATE TABLE [authors] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT\n)"
+        == 'CREATE TABLE "authors" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n)'
     )
     assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True])
+def test_transform_on_delete_cascade_does_not_delete_records(
+    fresh_db, use_pragma_foreign_keys
+):
+    # Transforming a table drops and recreates it - if another table references
+    # it with ON DELETE CASCADE and PRAGMA foreign_keys is on, that drop must
+    # not cascade and delete the referencing records
+    if use_pragma_foreign_keys:
+        fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE
+        );
+        """)
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    # Transform the table on the other end of the cascading foreign key
+    fresh_db["authors"].transform(rename={"name": "author_name"})
+    assert list(fresh_db["authors"].rows) == [
+        {"id": 1, "author_name": "Ursula K. Le Guin"}
+    ]
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "The Dispossessed", "author_id": 1}
+    ]
+    # Transforming the table with the cascading foreign key should not
+    # delete its records either
+    fresh_db["books"].transform(rename={"title": "book_title"})
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "book_title": "The Dispossessed", "author_id": 1}
+    ]
+    if use_pragma_foreign_keys:
+        assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+@pytest.mark.parametrize("on_delete", ["CASCADE", "SET NULL", "SET DEFAULT", "cascade"])
+def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_delete):
+    # PRAGMA foreign_keys is a no-op inside a transaction, so transforming a
+    # table referenced by ON DELETE CASCADE / SET NULL / SET DEFAULT foreign
+    # keys inside an open transaction would fire those actions when the old
+    # table is dropped - transform() should refuse instead
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id) ON DELETE {}
+        );
+        """.format(on_delete))
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    previous_schema = fresh_db["authors"].schema
+    with fresh_db.atomic():
+        with pytest.raises(TransactionError) as excinfo:
+            fresh_db["authors"].transform(rename={"name": "author_name"})
+    message = str(excinfo.value)
+    assert "books" in message
+    assert "ON DELETE {}".format(on_delete.upper()) in message
+    # Nothing should have changed
+    assert fresh_db["authors"].schema == previous_schema
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "The Dispossessed", "author_id": 1}
+    ]
+    assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db):
+    # The copied table carries a foreign key referencing the original table
+    # name, so a self-referential cascade would wipe the copy too
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE categories (
+            id INTEGER PRIMARY KEY,
+            name TEXT,
+            parent_id INTEGER REFERENCES categories(id) ON DELETE CASCADE
+        );
+        """)
+    fresh_db["categories"].insert_all(
+        [
+            {"id": 1, "name": "Fiction", "parent_id": None},
+            {"id": 2, "name": "Science Fiction", "parent_id": 1},
+        ]
+    )
+    with fresh_db.atomic():
+        with pytest.raises(TransactionError) as excinfo:
+            fresh_db["categories"].transform(rename={"name": "title"})
+    assert "categories" in str(excinfo.value)
+    assert fresh_db["categories"].count == 2
+
+
+def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db):
+    # An inbound foreign key without a destructive ON DELETE action is safe
+    # inside a transaction thanks to PRAGMA defer_foreign_keys
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id)
+        );
+        """)
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    with fresh_db.atomic():
+        fresh_db["authors"].transform(rename={"name": "author_name"})
+    assert list(fresh_db["authors"].rows) == [
+        {"id": 1, "author_name": "Ursula K. Le Guin"}
+    ]
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "The Dispossessed", "author_id": 1}
+    ]
+    assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+def test_transform_in_transaction_allowed_for_child_table(fresh_db):
+    # The table being transformed only has an outbound foreign key - dropping
+    # it fires no ON DELETE actions, so this is allowed inside a transaction
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE
+        );
+        """)
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    with fresh_db.atomic():
+        fresh_db["books"].transform(rename={"title": "book_title"})
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "book_title": "The Dispossessed", "author_id": 1}
+    ]
+
+
+def test_transform_in_transaction_allowed_with_foreign_keys_off(fresh_db):
+    # With PRAGMA foreign_keys off (the default) no cascades can fire, so
+    # transform inside a transaction is safe even with a CASCADE schema
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE
+        );
+        """)
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    with fresh_db.atomic():
+        fresh_db["authors"].transform(rename={"name": "author_name"})
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "The Dispossessed", "author_id": 1}
+    ]
+
+
+def test_transform_add_foreign_keys_from_scratch(fresh_db):
+    _add_country_city_continent(fresh_db)
+    fresh_db["places"].insert(_CAVEAU)
+    # Should have no foreign keys
+    assert fresh_db["places"].foreign_keys == []
+    # Now add them using .transform()
+    fresh_db["places"].transform(add_foreign_keys=("country", "continent", "city"))
+    # Should now have all three:
+    assert fresh_db["places"].foreign_keys == [
+        ForeignKey(
+            table="places", column="city", other_table="city", other_column="id"
+        ),
+        ForeignKey(
+            table="places",
+            column="continent",
+            other_table="continent",
+            other_column="id",
+        ),
+        ForeignKey(
+            table="places", column="country", other_table="country", other_column="id"
+        ),
+    ]
+    assert fresh_db["places"].schema == (
+        'CREATE TABLE "places" (\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT,\n'
+        '   "country" INTEGER REFERENCES "country"("id"),\n'
+        '   "continent" INTEGER REFERENCES "continent"("id"),\n'
+        '   "city" INTEGER REFERENCES "city"("id")\n'
+        ")"
+    )
+
+
+@pytest.mark.parametrize(
+    "add_foreign_keys",
+    (
+        ("country", "continent"),
+        # Fully specified
+        (
+            ("country", "country", "id"),
+            ("continent", "continent", "id"),
+        ),
+    ),
+)
+def test_transform_add_foreign_keys_from_partial(fresh_db, add_foreign_keys):
+    _add_country_city_continent(fresh_db)
+    fresh_db["places"].insert(
+        _CAVEAU,
+        foreign_keys=("city",),
+    )
+    # Should have one foreign keys
+    assert fresh_db["places"].foreign_keys == [
+        ForeignKey(table="places", column="city", other_table="city", other_column="id")
+    ]
+    # Now add three more using .transform()
+    fresh_db["places"].transform(add_foreign_keys=add_foreign_keys)
+    # Should now have all three:
+    assert fresh_db["places"].foreign_keys == [
+        ForeignKey(
+            table="places", column="city", other_table="city", other_column="id"
+        ),
+        ForeignKey(
+            table="places",
+            column="continent",
+            other_table="continent",
+            other_column="id",
+        ),
+        ForeignKey(
+            table="places", column="country", other_table="country", other_column="id"
+        ),
+    ]
+
+
+@pytest.mark.parametrize(
+    "foreign_keys",
+    (
+        ("country", "continent"),
+        # Fully specified
+        (
+            ("country", "country", "id"),
+            ("continent", "continent", "id"),
+        ),
+    ),
+)
+def test_transform_replace_foreign_keys(fresh_db, foreign_keys):
+    _add_country_city_continent(fresh_db)
+    fresh_db["places"].insert(
+        _CAVEAU,
+        foreign_keys=("city",),
+    )
+    assert len(fresh_db["places"].foreign_keys) == 1
+    # Replace with two different ones
+    fresh_db["places"].transform(foreign_keys=foreign_keys)
+    assert fresh_db["places"].schema == (
+        'CREATE TABLE "places" (\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT,\n'
+        '   "country" INTEGER REFERENCES "country"("id"),\n'
+        '   "continent" INTEGER REFERENCES "continent"("id"),\n'
+        '   "city" INTEGER\n'
+        ")"
+    )
+
+
+@pytest.mark.parametrize("table_type", ("id_pk", "rowid", "compound_pk"))
+def test_transform_preserves_rowids(fresh_db, table_type):
+    pk = None
+    if table_type == "id_pk":
+        pk = "id"
+    elif table_type == "compound_pk":
+        pk = ("id", "name")
+    elif table_type == "rowid":
+        pk = None
+    fresh_db["places"].insert_all(
+        [
+            {"id": "1", "name": "Paris", "country": "France"},
+            {"id": "2", "name": "London", "country": "UK"},
+            {"id": "3", "name": "New York", "country": "USA"},
+        ],
+        pk=pk,
+    )
+    # Now delete and insert a row to mix up the `rowid` sequence
+    fresh_db["places"].delete_where("id = ?", ["2"])
+    fresh_db["places"].insert({"id": "4", "name": "London", "country": "UK"})
+    previous_rows = list(
+        tuple(row) for row in fresh_db.execute("select rowid, id, name from places")
+    )
+    # Transform it
+    fresh_db["places"].transform(column_order=("country", "name"))
+    # Should be the same
+    next_rows = list(
+        tuple(row) for row in fresh_db.execute("select rowid, id, name from places")
+    )
+    assert previous_rows == next_rows
+
+
+@pytest.mark.parametrize(
+    "initial_strict,transform_strict,expected_strict",
+    (
+        (False, None, False),
+        (True, None, True),
+        (False, True, True),
+        (True, False, False),
+    ),
+)
+def test_transform_strict(fresh_db, initial_strict, transform_strict, expected_strict):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    dogs = fresh_db.table("dogs", strict=initial_strict)
+    dogs.insert({"id": 1, "name": "Cleo"})
+    assert dogs.strict is initial_strict
+    dogs.transform(strict=transform_strict)
+    assert dogs.strict is expected_strict
+
+
+def test_transform_to_strict_with_invalid_data(fresh_db):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    dogs = fresh_db["dogs"]
+    dogs.create({"id": int})
+    dogs.insert({"id": "not-an-integer"})
+
+    with pytest.raises(sqlite3.IntegrityError):
+        dogs.transform(strict=True)
+
+    assert dogs.strict is False
+    assert list(dogs.rows) == [{"id": "not-an-integer"}]
+    assert fresh_db.table_names() == ["dogs"]
+
+
+def test_transform_strict_updates_default(fresh_db):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    table = fresh_db.table("items", strict=True)
+    table.create({"id": int})
+
+    table.transform(strict=False)
+    assert table.strict is False
+
+    table.create({"id": int}, replace=True)
+    assert table.strict is False
+
+
+@pytest.mark.parametrize("method_name", ("transform", "transform_sql"))
+def test_transform_to_strict_not_supported(fresh_db, method_name):
+    table = fresh_db["items"]
+    table.create({"id": int})
+    fresh_db._supports_strict = False
+
+    with pytest.raises(TransformError, match="SQLite does not support STRICT tables"):
+        getattr(table, method_name)(strict=True)
+
+    assert table.strict is False
+
+
+@pytest.mark.parametrize(
+    "indexes, transform_params",
+    [
+        ([["name"]], {"types": {"age": str}}),
+        ([["name"], ["age", "breed"]], {"types": {"age": str}}),
+        ([], {"types": {"age": str}}),
+        ([["name"]], {"types": {"age": str}, "keep_table": "old_dogs"}),
+    ],
+)
+def test_transform_indexes(fresh_db, indexes, transform_params):
+    # https://github.com/simonw/sqlite-utils/issues/633
+    # New table should have same indexes as old table after transformation
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5, "breed": "Labrador"}, pk="id")
+
+    for index in indexes:
+        dogs.create_index(index)
+
+    indexes_before_transform = dogs.indexes
+
+    dogs.transform(**transform_params)
+
+    assert sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in dogs.indexes
+        ],
+        key=lambda x: x["name"],
+    ) == sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in indexes_before_transform
+        ],
+        key=lambda x: x["name"],
+    ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}"
+    if "keep_table" in transform_params:
+        assert all(
+            index.origin == "pk"
+            for index in fresh_db[transform_params["keep_table"]].indexes
+        )
+
+
+def test_transform_retains_indexes_with_foreign_keys(fresh_db):
+    dogs = fresh_db["dogs"]
+    owners = fresh_db["owners"]
+
+    dogs.insert({"id": 1, "name": "Cleo", "owner_id": 1}, pk="id")
+    owners.insert({"id": 1, "name": "Alice"}, pk="id")
+
+    dogs.create_index(["name"])
+
+    indexes_before_transform = dogs.indexes
+
+    fresh_db.add_foreign_keys([("dogs", "owner_id", "owners", "id")])  # calls transform
+
+    assert sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in dogs.indexes
+        ],
+        key=lambda x: x["name"],
+    ) == sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in indexes_before_transform
+        ],
+        key=lambda x: x["name"],
+    ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}"
+
+
+@pytest.mark.parametrize(
+    "transform_params",
+    [
+        {"rename": {"age": "dog_age"}},
+        {"drop": ["age"]},
+    ],
+)
+def test_transform_with_indexes_errors(fresh_db, transform_params):
+    # Should error with a compound (name, age) index if age is renamed or dropped
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id")
+
+    dogs.create_index(["name", "age"])
+
+    with pytest.raises(TransformError) as excinfo:
+        dogs.transform(**transform_params)
+
+    assert (
+        "Index 'idx_dogs_name_age' column 'age' is not in updated table 'dogs'. "
+        "You must manually drop this index prior to running this transformation"
+        in str(excinfo.value)
+    )
+
+
+def test_transform_with_unique_constraint_implicit_index(fresh_db):
+    dogs = fresh_db["dogs"]
+    # Create a table with a UNIQUE constraint on 'name', which creates an implicit index
+    fresh_db.execute("""
+        CREATE TABLE dogs (
+            id INTEGER PRIMARY KEY,
+            name TEXT UNIQUE,
+            age INTEGER
+        );
+    """)
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5})
+
+    # Attempt to transform the table without modifying 'name'
+    with pytest.raises(TransformError) as excinfo:
+        dogs.transform(types={"age": str})
+
+    assert (
+        "Index 'sqlite_autoindex_dogs_1' on table 'dogs' does not have a CREATE INDEX statement."
+        in str(excinfo.value)
+    )
+    assert (
+        "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation."
+        in str(excinfo.value)
+    )

tests/test_update.py

@@ -70,20 +70,21 @@ def test_update_alter(fresh_db):
     ] == list(table.rows)
 
 
-def test_update_alter_with_invalid_column_characters(fresh_db):
+def test_update_alter_with_special_column_characters(fresh_db):
+    # With double-quote escaping, columns with special characters are now valid
     table = fresh_db["table"]
     rowid = table.insert({"foo": "bar"}).last_pk
-    with pytest.raises(AssertionError):
-        table.update(rowid, {"new_col[abc]": 1.2}, alter=True)
+    table.update(rowid, {"new_col[abc]": 1.2}, alter=True)
+    assert list(table.rows) == [{"foo": "bar", "new_col[abc]": 1.2}]
 
 
 def test_update_with_no_values_sets_last_pk(fresh_db):
     table = fresh_db.table("dogs", pk="id")
     table.insert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Pancakes"}])
     table.update(1)
-    assert 1 == table.last_pk
+    assert table.last_pk == 1
     table.update(2)
-    assert 2 == table.last_pk
+    assert table.last_pk == 2
     with pytest.raises(NotFoundError):
         table.update(3)
 

tests/test_upsert.py

@@ -1,33 +1,46 @@
 from sqlite_utils.db import PrimaryKeyRequired
+from sqlite_utils import Database
 import pytest
 
 
-def test_upsert(fresh_db):
-    table = fresh_db["table"]
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert(use_old_upsert):
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    table = db["table"]
     table.insert({"id": 1, "name": "Cleo"}, pk="id")
     table.upsert({"id": 1, "age": 5}, pk="id", alter=True)
-    assert [{"id": 1, "name": "Cleo", "age": 5}] == list(table.rows)
-    assert 1 == table.last_pk
+    assert list(table.rows) == [{"id": 1, "name": "Cleo", "age": 5}]
+    assert table.last_pk == 1
 
 
 def test_upsert_all(fresh_db):
     table = fresh_db["table"]
     table.upsert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Nixie"}], pk="id")
     table.upsert_all([{"id": 1, "age": 5}, {"id": 2, "age": 5}], pk="id", alter=True)
-    assert [
+    assert list(table.rows) == [
         {"id": 1, "name": "Cleo", "age": 5},
         {"id": 2, "name": "Nixie", "age": 5},
-    ] == list(table.rows)
+    ]
     assert table.last_pk is None
 
 
 def test_upsert_all_single_column(fresh_db):
     table = fresh_db["table"]
     table.upsert_all([{"name": "Cleo"}], pk="name")
-    assert [{"name": "Cleo"}] == list(table.rows)
+    assert list(table.rows) == [{"name": "Cleo"}]
     assert table.pks == ["name"]
 
 
+def test_upsert_all_not_null(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/538
+    fresh_db["comments"].upsert_all(
+        [{"id": 1, "name": "Cleo"}],
+        pk="id",
+        not_null=["name"],
+    )
+    assert list(fresh_db["comments"].rows) == [{"id": 1, "name": "Cleo"}]
+
+
 def test_upsert_error_if_no_pk(fresh_db):
     table = fresh_db["table"]
     with pytest.raises(PrimaryKeyRequired):
@@ -36,6 +49,89 @@ def test_upsert_error_if_no_pk(fresh_db):
         table.upsert({"id": 1, "name": "Cleo"})
 
 
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_empty_record_errors(use_old_upsert):
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    table = db["table"]
+    table.insert({"id": 1, "name": "Cleo"}, pk="id")
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert({}, pk="id")
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert_all([{}, {}], pk="id")
+    # No rows can have been inserted
+    assert table.count == 1
+
+
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_missing_pk_value_errors(use_old_upsert):
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    table = db["table"]
+    table.insert({"id": 1, "name": "Cleo"}, pk="id")
+    # Records that omit the pk column entirely
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert_all([{"name": "Pancakes"}, {"name": "Marnie"}], pk="id")
+    # A record with an explicit None pk value can never conflict
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert({"id": None, "name": "Pancakes"}, pk="id")
+    assert list(table.rows) == [{"id": 1, "name": "Cleo"}]
+
+
+def test_upsert_missing_compound_pk_value_errors(fresh_db):
+    table = fresh_db["table"]
+    table.insert({"a": "x", "b": "y", "v": 1}, pk=("a", "b"))
+    # Missing one component of the detected compound primary key
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert({"a": "x", "v": 2})
+    assert list(table.rows) == [{"a": "x", "b": "y", "v": 1}]
+
+
+def test_upsert_error_if_existing_table_has_no_pk(fresh_db):
+    table = fresh_db.create_table("table", {"id": int, "name": str})
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert({"id": 1, "name": "Cleo"})
+
+
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_uses_compound_pk_from_existing_table(use_old_upsert):
+    # https://github.com/simonw/sqlite-utils/issues/629
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    db.execute("""
+        create table summary (
+            Source text,
+            Object text,
+            Category text,
+            Count integer,
+            primary key (Source, Object, Category)
+        )
+        """)
+    table = db["summary"]
+    table.upsert(
+        {
+            "Source": "Client A",
+            "Object": "Accounts",
+            "Category": "All",
+            "Count": 3,
+        }
+    )
+    assert table.last_pk == ("Client A", "Accounts", "All")
+    table.upsert(
+        {
+            "Source": "Client A",
+            "Object": "Accounts",
+            "Category": "All",
+            "Count": 4,
+        }
+    )
+    assert list(table.rows) == [
+        {
+            "Source": "Client A",
+            "Object": "Accounts",
+            "Category": "All",
+            "Count": 4,
+        }
+    ]
+
+
 def test_upsert_with_hash_id(fresh_db):
     table = fresh_db["table"]
     table.upsert({"foo": "bar"}, hash_id="pk")
@@ -45,6 +141,30 @@ def test_upsert_with_hash_id(fresh_db):
     assert "a5e744d0164540d33b1d7ea616c28f2fa97e754a" == table.last_pk
 
 
+@pytest.mark.parametrize("hash_id", (None, "custom_id"))
+def test_upsert_with_hash_id_columns(fresh_db, hash_id):
+    table = fresh_db["table"]
+    table.upsert({"a": 1, "b": 2, "c": 3}, hash_id=hash_id, hash_id_columns=("a", "b"))
+    assert list(table.rows) == [
+        {
+            hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49",
+            "a": 1,
+            "b": 2,
+            "c": 3,
+        }
+    ]
+    assert table.last_pk == "4acc71e0547112eb432f0a36fb1924c4a738cb49"
+    table.upsert({"a": 1, "b": 2, "c": 4}, hash_id=hash_id, hash_id_columns=("a", "b"))
+    assert list(table.rows) == [
+        {
+            hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49",
+            "a": 1,
+            "b": 2,
+            "c": 4,
+        }
+    ]
+
+
 def test_upsert_compound_primary_key(fresh_db):
     table = fresh_db["table"]
     table.upsert_all(

tests/test_utils.py

@@ -1,4 +1,6 @@
 from sqlite_utils import utils
+import csv
+import io
 import pytest
 
 
@@ -22,6 +24,79 @@ def test_decode_base64_values(input, expected, should_be_is):
         assert actual == expected
 
 
-def test_find_spatialite():
-    spatialite = utils.find_spatialite()
-    assert spatialite is None or isinstance(spatialite, str)
+@pytest.mark.parametrize(
+    "size,expected",
+    (
+        (1, [["a"], ["b"], ["c"], ["d"]]),
+        (2, [["a", "b"], ["c", "d"]]),
+        (3, [["a", "b", "c"], ["d"]]),
+        (4, [["a", "b", "c", "d"]]),
+    ),
+)
+def test_chunks(size, expected):
+    input = ["a", "b", "c", "d"]
+    chunks = list(map(list, utils.chunks(input, size)))
+    assert chunks == expected
+
+
+def test_hash_record():
+    expected = "d383e7c0ba88f5ffcdd09be660de164b3847401a"
+    assert utils.hash_record({"name": "Cleo", "twitter": "CleoPaws"}) == expected
+    assert (
+        utils.hash_record(
+            {"name": "Cleo", "twitter": "CleoPaws", "age": 7}, keys=("name", "twitter")
+        )
+        == expected
+    )
+    assert (
+        utils.hash_record({"name": "Cleo", "twitter": "CleoPaws", "age": 7}) != expected
+    )
+
+
+def test_maximize_csv_field_size_limit():
+    # Reset to default in case other tests have changed it
+    csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT)
+    long_value = "a" * 131073
+    long_csv = "id,text\n1,{}".format(long_value)
+    fp = io.BytesIO(long_csv.encode("utf-8"))
+    # Using rows_from_file should error
+    with pytest.raises(csv.Error):
+        rows, _ = utils.rows_from_file(fp, utils.Format.CSV)
+        list(rows)
+    # But if we call maximize_csv_field_size_limit() first it should be OK:
+    utils.maximize_csv_field_size_limit()
+    fp2 = io.BytesIO(long_csv.encode("utf-8"))
+    rows2, _ = utils.rows_from_file(fp2, utils.Format.CSV)
+    rows_list2 = list(rows2)
+    assert len(rows_list2) == 1
+    assert rows_list2[0]["id"] == "1"
+    assert rows_list2[0]["text"] == long_value
+
+
+@pytest.mark.parametrize(
+    "input,expected",
+    (
+        ({"foo": {"bar": 1}}, {"foo_bar": 1}),
+        ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}),
+        ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}),
+    ),
+)
+def test_flatten(input, expected):
+    assert utils.flatten(input) == expected
+
+
+@pytest.mark.parametrize(
+    "input,expected",
+    (
+        ([], []),
+        (["id", "name"], ["id", "name"]),
+        (["id", "id"], ["id", "id_2"]),
+        (["id", "id", "id"], ["id", "id_2", "id_3"]),
+        # A renamed duplicate must not clobber a real column called id_2
+        (["id", "id", "id_2"], ["id", "id_3", "id_2"]),
+        (["id_2", "id", "id"], ["id_2", "id", "id_3"]),
+        (["id", "id", "id_2", "id_2"], ["id", "id_3", "id_2", "id_2_2"]),
+    ),
+)
+def test_dedupe_keys(input, expected):
+    assert utils.dedupe_keys(input) == expected

tests/test_wal.py

@@ -1,5 +1,6 @@
 import pytest
 from sqlite_utils import Database
+from sqlite_utils.db import TransactionError
 
 
 @pytest.fixture
@@ -21,3 +22,67 @@ def test_enable_disable_wal(db_path_tmpdir):
     db.disable_wal()
     assert "delete" == db.journal_mode
     assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()]
+
+
+def test_enable_wal_inside_transaction_raises(db_path_tmpdir):
+    db, path, tmpdir = db_path_tmpdir
+    db["test"].insert({"id": 1}, pk="id")
+    with pytest.raises(TransactionError):
+        with db.atomic():
+            db["test"].insert({"id": 2}, pk="id")
+            db.enable_wal()
+    # The atomic() block must have rolled back cleanly and the
+    # journal mode must be unchanged
+    assert db.journal_mode == "delete"
+    assert [r["id"] for r in db["test"].rows] == [1]
+
+
+def test_disable_wal_inside_transaction_raises(db_path_tmpdir):
+    db, path, tmpdir = db_path_tmpdir
+    db.enable_wal()
+    db["test"].insert({"id": 1}, pk="id")
+    with pytest.raises(TransactionError):
+        with db.atomic():
+            db["test"].insert({"id": 2}, pk="id")
+            db.disable_wal()
+    assert db.journal_mode == "wal"
+    assert [r["id"] for r in db["test"].rows] == [1]
+
+
+def test_ensure_autocommit_on(db_path_tmpdir):
+    db, path, tmpdir = db_path_tmpdir
+    previous_isolation_level = db.conn.isolation_level
+    assert previous_isolation_level is not None
+    with db.ensure_autocommit_on():
+        # isolation_level of None means driver-level autocommit mode
+        assert db.conn.isolation_level is None
+    # Restored afterwards
+    assert db.conn.isolation_level == previous_isolation_level
+
+
+def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir):
+    # Calling enable_wal() when WAL is already enabled is a no-op,
+    # so it is fine inside a transaction
+    db, path, tmpdir = db_path_tmpdir
+    db.enable_wal()
+    with db.atomic():
+        db["test"].insert({"id": 1}, pk="id")
+        db.enable_wal()
+    assert [r["id"] for r in db["test"].rows] == [1]
+
+
+def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir):
+    # Setting isolation_level commits any pending transaction as a side
+    # effect, silently breaking the caller's rollback guarantee - so
+    # entering autocommit mode with a transaction open is an error
+    db, path, tmpdir = db_path_tmpdir
+    db["test"].insert({"id": 1}, pk="id")
+    db.begin()
+    db.execute("insert into test (id) values (2)")
+    with pytest.raises(TransactionError):
+        with db.ensure_autocommit_on():
+            pass
+    # The transaction is still open and can still be rolled back
+    assert db.conn.in_transaction
+    db.rollback()
+    assert [r["id"] for r in db["test"].rows] == [1]

tests/utils.py

@@ -1,7 +0,0 @@
-import re
-
-COLLAPSE_RE = re.compile(r"\s+")
-
-
-def collapse_whitespace(s):
-    return COLLAPSE_RE.sub(" ", s)