Changelog now links to CLI and Python API in most recent entry

Refs #791, #792, #794, #795

.github/actions/setup-sqlite-version/action.yml

@@ -0,0 +1,39 @@
+name: "Setup SQLite version"
+description: "Build and activate a specific SQLite version from its amalgamation archive"
+inputs:
+  version:
+    description: "The SQLite version to install"
+    required: true
+  cflags:
+    description: "CFLAGS to use when compiling SQLite"
+    required: false
+    default: ""
+  skip-activate:
+    description: "Set to true to skip modifying the library path"
+    required: false
+    default: "false"
+  fallback-urls:
+    description: "Whitespace-separated fallback download URLs to try after sqlite.org"
+    required: false
+    default: ""
+outputs:
+  sqlite-location:
+    description: "Directory containing the compiled SQLite library"
+    value: ${{ steps.build.outputs.sqlite-location }}
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads"
+    - uses: actions/cache@v6
+      with:
+        path: ${{ runner.temp }}/sqlite-versions/downloads
+        key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1
+    - id: build
+      shell: bash
+      run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh"
+      env:
+        SQLITE_VERSION: ${{ inputs.version }}
+        SQLITE_CFLAGS: ${{ inputs.cflags }}
+        SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }}
+        SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }}

.github/actions/setup-sqlite-version/setup-sqlite-version.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}"
+cflags="${SQLITE_CFLAGS:-}"
+skip_activate="${SQLITE_SKIP_ACTIVATE:-false}"
+extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}"
+
+case "$version_spec" in
+  3.46 | 3.46.0)
+    sqlite_version="3.46.0"
+    sqlite_year="2024"
+    amalgamation_id="3460000"
+    builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip"
+    ;;
+  3.23.1)
+    sqlite_version="3.23.1"
+    sqlite_year="2018"
+    amalgamation_id="3230100"
+    builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3230100.zip"
+    ;;
+  *)
+    echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh."
+    exit 1
+    ;;
+esac
+
+case "$(uname -s)" in
+  Linux)
+    library_name="libsqlite3.so.0"
+    library_path_var="LD_LIBRARY_PATH"
+    ;;
+  Darwin)
+    library_name="libsqlite3.dylib"
+    library_path_var="DYLD_LIBRARY_PATH"
+    ;;
+  *)
+    echo "::error::Unsupported platform $(uname -s)"
+    exit 1
+    ;;
+esac
+
+runner_temp="${RUNNER_TEMP:-}"
+if [ -z "$runner_temp" ]; then
+  runner_temp="$(mktemp -d)"
+fi
+
+filename="sqlite-amalgamation-${amalgamation_id}"
+official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip"
+download_dir="${runner_temp}/sqlite-versions/downloads"
+source_root="${runner_temp}/sqlite-versions/source"
+source_dir="${source_root}/${filename}"
+build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}"
+archive_path="${download_dir}/${filename}.zip"
+
+mkdir -p "$download_dir" "$source_root" "$build_dir"
+
+download_archive() {
+  local url
+  local candidate_path="${archive_path}.tmp"
+  local urls=("$official_url")
+
+  for url in $builtin_fallback_urls $extra_fallback_urls; do
+    urls+=("$url")
+  done
+
+  rm -f "$candidate_path"
+  for url in "${urls[@]}"; do
+    echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}"
+    if curl \
+      --fail \
+      --location \
+      --show-error \
+      --retry 5 \
+      --retry-delay 2 \
+      --retry-max-time 180 \
+      --retry-all-errors \
+      --connect-timeout 20 \
+      --max-time 240 \
+      --output "$candidate_path" \
+      "$url"; then
+      mv "$candidate_path" "$archive_path"
+      return 0
+    fi
+
+    echo "::warning::Download failed from ${url}"
+    rm -f "$candidate_path"
+  done
+
+  echo "::error::Could not download SQLite ${sqlite_version} amalgamation"
+  return 1
+}
+
+if [ ! -f "${source_dir}/sqlite3.c" ]; then
+  if [ ! -f "$archive_path" ]; then
+    download_archive
+  fi
+
+  rm -rf "$source_dir"
+  unzip -q "$archive_path" -d "$source_root"
+fi
+
+if [ ! -f "${source_dir}/sqlite3.c" ]; then
+  echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}"
+  exit 1
+fi
+
+read -r -a cflag_args <<< "$cflags"
+
+echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}"
+gcc \
+  -fPIC \
+  -shared \
+  "${cflag_args[@]}" \
+  "${source_dir}/sqlite3.c" \
+  "-I${source_dir}" \
+  -o "${build_dir}/${library_name}"
+
+if [ "$library_name" = "libsqlite3.so.0" ]; then
+  ln -sf "$library_name" "${build_dir}/libsqlite3.so"
+fi
+
+if [ -n "${GITHUB_OUTPUT:-}" ]; then
+  echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT"
+else
+  echo "sqlite-location=${build_dir}"
+fi
+
+case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in
+  true | 1 | yes)
+    echo "Skipping ${library_path_var} activation"
+    ;;
+  *)
+    existing_value="${!library_path_var:-}"
+    if [ -n "${GITHUB_ENV:-}" ]; then
+      if [ -n "$existing_value" ]; then
+        echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV"
+      else
+        echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV"
+      fi
+    fi
+    echo "Added ${build_dir} to ${library_path_var}"
+    ;;
+esac

.github/workflows/documentation-links.yml

@@ -0,0 +1,16 @@
+name: Read the Docs Pull Request Preview
+on:
+  pull_request_target:
+    types:
+      - opened
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: "sqlite-utils"

.github/workflows/publish.yml

@@ -9,24 +9,19 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: actions/cache@v2
+        cache: pip
-      name: Configure pip caching
+        cache-dependency-path: pyproject.toml
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install . --group dev
     - name: Run tests
       run: |
         pytest
@@ -34,25 +29,20 @@ jobs:
     runs-on: ubuntu-latest
     needs: [test]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v7
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
-        python-version: '3.9'
+        python-version: '3.14'
-    - uses: actions/cache@v2
+        cache: pip
-      name: Configure pip caching
+        cache-dependency-path: pyproject.toml
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-publish-pip-
     - name: Install dependencies
       run: |
-        pip install setuptools wheel twine
+        pip install build twine
     - name: Publish
       env:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
       run: |
-        python setup.py sdist bdist_wheel
+        python -m build
         twine upload dist/*

.github/workflows/spellcheck.yml

@@ -0,0 +1,22 @@
+name: Check spelling in documentation
+
+on: [push, pull_request]
+
+jobs:
+  spellcheck:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Install dependencies
+      run: |
+        pip install . --group docs
+    - name: Check spelling
+      run: |
+        codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
+        codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt

.github/workflows/test-coverage.yml

@@ -12,22 +12,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v7
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
-        python-version: 3.9
+        python-version: "3.11"
-    - uses: actions/cache@v2
+        cache: pip
-      name: Configure pip caching
+        cache-dependency-path: pyproject.toml
-      with:
+    - name: Install SpatiaLite
-        path: ~/.cache/pip
+      run: sudo apt-get install libsqlite3-mod-spatialite
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -e .[test]
+        python -m pip install . --group dev
         python -m pip install pytest-cov
     - name: Run tests
       run: |-

.github/workflows/test-sqlite-support.yml

@@ -0,0 +1,41 @@
+name: Test SQLite versions
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ${{ matrix.platform }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        platform: [ubuntu-latest]
+        python-version: ["3.10"]
+        sqlite-version: [
+          "3.46",
+          "3.23.1", # 2018-04-10, before UPSERT
+        ]
+    steps:
+    - uses: actions/checkout@v7
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Set up SQLite ${{ matrix.sqlite-version }}
+      uses: ./.github/actions/setup-sqlite-version
+      with:
+        version: ${{ matrix.sqlite-version }}
+        cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1"
+    - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)"
+    - name: Install dependencies
+      run: |
+        pip install . --group dev
+        pip freeze
+    - name: Run tests
+      run: |
+        python -m pytest

.github/workflows/test.yml

@@ -1,36 +1,57 @@
 name: Test
 
-on: [push]
+on: [push, pull_request]
+
+env:
+  FORCE_COLOR: 1
 
 jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15-dev"]
         numpy: [0, 1]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest, macos-14]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: actions/cache@v2
+        allow-prereleases: true
-      name: Configure pip caching
+        cache: pip
-      with:
+        cache-dependency-path: pyproject.toml
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install . --group dev
     - name: Optionally install numpy
       if: matrix.numpy == 1
       run: pip install numpy
+    - name: Install SpatiaLite
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install libsqlite3-mod-spatialite
+    - name: Build extension for --load-extension test
+      if: matrix.os == 'ubuntu-latest'
+      run: |-
+        (cd tests && gcc ext.c -fPIC -shared -o ext.so && ls -lah)
     - name: Run tests
       run: |
-        pytest
+        pytest -v
+    - name: Run autocommit tests just on 3.14/Ubuntu
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14'
+      run: pytest --sqlite-autocommit
+    - name: run mypy
+      run: mypy sqlite_utils tests
+    - name: run flake8
+      run: flake8
+    - name: run ty
+      if: matrix.os != 'windows-latest' && matrix.python-version == '3.14'
+      run: |
+        pip install uv
+        uv run ty check sqlite_utils
     - name: Check formatting
       run: black . --check
+    - name: Check if cog needs to be run
+      run: |
+        cog --check --diff README.md docs/*.rst

.gitignore

@@ -1,4 +1,6 @@
 .venv
+dist
+build
 *.db
 __pycache__/
 *.py[cod]
@@ -12,3 +14,10 @@ venv
 .coverage
 .schema
 .vscode
+.hypothesis
+Pipfile
+Pipfile.lock
+uv.lock
+tests/*.dylib
+tests/*.so
+tests/*.dll

.readthedocs.yaml

@@ -0,0 +1,17 @@
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+  jobs:
+    install:
+    - pip install --upgrade pip
+    - pip install . --group docs
+
+formats:
+- pdf
+- epub

Justfile

@@ -0,0 +1,33 @@
+# Run tests and linters
+@default: test lint
+
+# Run pytest with supplied options
+@test *options:
+  uv run pytest {{options}}
+
+@run *options:
+  uv run -- {{options}}
+
+# Run linters: black, flake8, mypy, ty, cog
+@lint:
+  just run black . --check
+  uv run flake8
+  uv run mypy sqlite_utils tests
+  uv run ty check sqlite_utils
+  uv run cog --check README.md docs/*.rst
+  uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
+
+# Rebuild docs with cog
+@cog:
+  uv run --group docs cog -r README.md docs/*.rst
+
+# Serve live docs on localhost:8000
+@docs: cog
+  #!/usr/bin/env bash
+  cd docs
+  uv run --group docs make livehtml
+
+
+# Apply Black
+@black:
+  uv run black .

README.md

@@ -1,24 +1,29 @@
 # sqlite-utils
 
 [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/)
-[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/latest/changelog.html)
+[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/stable/changelog.html)
 [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/)
 [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest)
-[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest)
+[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable)
 [![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE)
+[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/Ass7bCAMDw)
 
 Python CLI utility and library for manipulating SQLite databases.
 
 ## Some feature highlights
 
 - [Pipe JSON](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema
+- [Run in-memory SQL queries](https://sqlite-utils.datasette.io/en/stable/cli.html#querying-data-directly-using-an-in-memory-database), including joins, directly against data in CSV, TSV or JSON files and view the results
 - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance
-- Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as dropping columns
+- Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as changing the type of a column
 - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data
+- [Manage database migrations](https://sqlite-utils.datasette.io/en/stable/migrations.html) using Python migration files and the `sqlite-utils migrate` command
+- [Install plugins](https://sqlite-utils.datasette.io/en/stable/plugins.html) to add custom SQL functions and additional features
 
-Read more on my blog: [
+Upgrading from sqlite-utils 3.x? See the [4.0 upgrade guide](https://sqlite-utils.datasette.io/en/stable/upgrading.html#upgrading-from-3-x-to-4-0).
-sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/).
+
+Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqlite-utils](https://simonwillison.net/tags/sqlite-utils/).
 
 ## Installation
 
@@ -32,12 +37,19 @@ Or if you use [Homebrew](https://brew.sh/) for macOS:
 
 Now you can do things with the CLI utility like this:
 
+    $ sqlite-utils memory dogs.csv "select * from t"
+    [{"id": 1, "age": 4, "name": "Cleo"},
+     {"id": 2, "age": 2, "name": "Pancakes"}]
+
+    $ sqlite-utils insert dogs.db dogs dogs.csv --csv
+    [####################################]  100%
+
     $ sqlite-utils tables dogs.db --counts
     [{"table": "dogs", "count": 2}]
 
-    $ sqlite-utils dogs.db "select * from dogs"
+    $ sqlite-utils dogs.db "select id, name from dogs"
-    [{"id": 1, "age": 4, "name": "Cleo"},
+    [{"id": 1, "name": "Cleo"},
-     {"id": 2, "age": 2, "name": "Pancakes"}]
+     {"id": 2, "name": "Pancakes"}]
 
     $ sqlite-utils dogs.db "select * from dogs" --csv
     id,age,name
@@ -61,7 +73,7 @@ Or for data in a CSV file:
 
 `sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command:
 
-    $ cat dogs.csv | sqlite-utils memory - "select name, age from dogs"
+    $ cat dogs.csv | sqlite-utils memory - "select name, age from stdin"
 
 See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands.
 

docs/Makefile

@@ -20,4 +20,4 @@ help:
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 livehtml:
-	sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0)
+	sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils

docs/_static/js/custom.js

@@ -0,0 +1,23 @@
+jQuery(function ($) {
+  // Show banner linking to /stable/ if this is a /latest/ page
+  if (!/\/latest\//.test(location.pathname)) {
+    return;
+  }
+  var stableUrl = location.pathname.replace("/latest/", "/stable/");
+  // Check it's not a 404
+  fetch(stableUrl, { method: "HEAD" }).then((response) => {
+    if (response.status == 200) {
+      var warning = $(
+        `<div class="admonition warning">
+           <p class="first admonition-title">Note</p>
+           <p class="last">
+             This documentation covers the <strong>development version</strong> of <code>sqlite-utils</code>.</p>
+             <p>See <a href="${stableUrl}">this page</a> for the current stable release.
+           </p>
+        </div>`
+      );
+      warning.find("a").attr("href", stableUrl);
+      $("article[role=main]").prepend(warning);
+    }
+  });
+});

docs/_templates/base.html

@@ -0,0 +1,42 @@
+{%- extends "!base.html" %}
+
+{% block site_meta %}
+{{ super() }}
+<script defer data-domain="sqlite-utils.datasette.io" src="https://plausible.io/js/plausible.js"></script>
+{% endblock %}
+
+{% block scripts %}
+{{ super() }}
+<style type="text/css">
+.highlight-output .highlight {
+  border-left: 9px solid #30c94f;
+}
+</style>
+<script>
+document.addEventListener("DOMContentLoaded", function() {
+  // Show banner linking to /stable/ if this is a /latest/ page
+  if (!/\/latest\//.test(location.pathname)) {
+    return;
+  }
+  var stableUrl = location.pathname.replace("/latest/", "/stable/");
+  // Check it's not a 404
+  fetch(stableUrl, { method: "HEAD" }).then((response) => {
+    if (response.status === 200) {
+      var warning = document.createElement("div");
+      warning.className = "admonition warning";
+      warning.innerHTML = `
+        <p class="first admonition-title">Note</p>
+        <p class="last">
+          This documentation covers the <strong>development version</strong> of Datasette.
+        </p>
+        <p>
+          See <a href="${stableUrl}">this page</a> for the current stable release.
+        </p>
+      `;
+      var mainArticle = document.querySelector("article[role=main]");
+      mainArticle.insertBefore(warning, mainArticle.firstChild);
+    }
+  });
+});
+</script>
+{% endblock %}

docs/codespell-ignore-words.txt

@@ -0,0 +1 @@
+doub

docs/conf.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from subprocess import Popen, PIPE
+import inspect
+from pathlib import Path
+from subprocess import Popen, PIPE, check_output
+import sys
 
 # This file is execfile()d with the current directory set to its
 # containing dir.
@@ -30,7 +33,68 @@ from subprocess import Popen, PIPE
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = []
+extensions = [
+    "sphinx.ext.extlinks",
+    "sphinx.ext.autodoc",
+    "sphinx_copybutton",
+    "sphinx.ext.linkcode",
+]
+autodoc_member_order = "bysource"
+autodoc_typehints = "description"
+
+extlinks = {
+    "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#%s"),
+}
+
+
+def _linkcode_git_ref():
+    try:
+        return check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip()
+    except Exception:
+        return "main"
+
+
+def linkcode_resolve(domain, info):
+    if domain != "py":
+        return None
+
+    module_name = info.get("module")
+    if not module_name or module_name.split(".")[0] != "sqlite_utils":
+        return None
+
+    module = sys.modules.get(module_name)
+    if module is None:
+        return None
+
+    obj = module
+    for part in info.get("fullname", "").split("."):
+        obj = getattr(obj, part, None)
+        if obj is None:
+            return None
+
+    if isinstance(obj, property):
+        obj = obj.fget
+
+    try:
+        obj = inspect.unwrap(obj)
+        source_file = inspect.getsourcefile(obj)
+        _, line_number = inspect.getsourcelines(obj)
+    except Exception:
+        return None
+
+    if source_file is None:
+        return None
+
+    try:
+        filename = Path(source_file).resolve().relative_to(Path(__file__).parent.parent)
+    except ValueError:
+        return None
+
+    return (
+        "https://github.com/simonw/sqlite-utils/blob/"
+        f"{_linkcode_git_ref()}/{filename}#L{line_number}"
+    )
+
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
@@ -46,7 +110,7 @@ master_doc = "index"
 
 # General information about the project.
 project = "sqlite-utils"
-copyright = "2018-2021, Simon Willison"
+copyright = "2018-2022, Simon Willison"
 author = "Simon Willison"
 
 # The version info for the project you're documenting, acts as replacement for
@@ -55,7 +119,7 @@ author = "Simon Willison"
 #
 # The short X.Y version.
 pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True)
-git_version = pipe.stdout.read().decode("utf8")
+git_version = pipe.stdout.read().decode("utf8") if pipe.stdout else ""
 
 if git_version:
     version = git_version.rsplit("-", 1)[0]
@@ -69,7 +133,7 @@ else:
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -79,6 +143,9 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "sphinx"
 
+# Only syntax highlight of code-block is used:
+highlight_language = "none"
+
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
@@ -88,7 +155,8 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = "sphinx_rtd_theme"
+html_theme = "furo"
+html_title = "sqlite-utils"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -101,18 +169,7 @@ html_theme = "sphinx_rtd_theme"
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
-# Custom sidebar templates, must be a dictionary that maps document names
+html_js_files = ["js/custom.js"]
-# to template names.
-#
-# This is required for the alabaster theme
-# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
-html_sidebars = {
-    "**": [
-        "relations.html",  # needs 'show_related': True theme option to display
-        "searchbox.html",
-    ]
-}
-
 
 # -- Options for HTMLHelp output ------------------------------------------
 
@@ -170,7 +227,7 @@ texinfo_documents = [
         "sqlite-utils documentation",
         author,
         "sqlite-utils",
-        "Python utility functions for manipulating SQLite databases",
+        "Python library for manipulating SQLite databases",
         "Miscellaneous",
     )
 ]

docs/contributing.rst

@@ -0,0 +1,137 @@
+.. _contributing:
+
+==============
+ Contributing
+==============
+
+Development of ``sqlite-utils`` takes place in the `sqlite-utils GitHub repository <https://github.com/simonw/sqlite-utils>`__.
+
+All improvements to the software should start with an issue. Read `How I build a feature <https://simonwillison.net/2022/Jan/12/how-i-build-a-feature/>`__ for a detailed description of the recommended process for building bug fixes or enhancements.
+
+.. _contributing_checkout:
+
+Obtaining the code
+==================
+
+To work on this library locally, first checkout the code::
+
+    git clone git@github.com:simonw/sqlite-utils
+    cd sqlite-utils
+
+Use ``uv run`` to run the development version of the tool::
+
+    uv run sqlite-utils --help
+
+.. _contributing_tests:
+
+Running the tests
+=================
+
+Use ``uv run`` to run the tests::
+
+    uv run pytest
+
+.. _contributing_docs:
+
+Building the documentation
+==========================
+
+To build the documentation run this command::
+
+    uv run make livehtml --directory docs
+
+This will start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file.
+
+The `cog <https://github.com/nedbat/cog>`__ tool is used to maintain portions of the documentation. You can run it like so::
+
+    uv run cog -r docs/*.rst
+
+.. _contributing_linting:
+
+Linting and formatting
+======================
+
+``sqlite-utils`` uses `Black <https://black.readthedocs.io/>`__ for code formatting, and `flake8 <https://flake8.pycqa.org/>`__ and `mypy <https://mypy.readthedocs.io/>`__ for linting and type checking::
+
+    uv run black .
+
+Linting tools can be run like this::
+
+    uv run flake8
+    uv run mypy sqlite_utils
+
+All three of these tools are run by our CI mechanism against every commit and pull request.
+
+.. _contributing_just:
+
+Using Just
+==========
+
+If you install `Just <https://github.com/casey/just>`__ you can use it to manage your local development environment.
+
+To run all of the tests and linters::
+
+    just
+
+To run tests, or run a specific test module or test by name::
+
+    just test # All tests
+    just test tests/test_cli_memory.py # Just this module
+    just test -k test_memory_no_detect_types # Just this test
+
+To run just the linters::
+
+    just lint
+
+To apply Black to your code::
+
+    just black
+
+To update documentation using Cog::
+
+    just cog
+
+To run the live documentation server (this will run Cog first)::
+
+    just docs
+
+And to list all available commands::
+
+    just -l
+
+.. _release_process:
+
+Release process
+===============
+
+Releases are performed using tags. When a new release is published on GitHub, a `GitHub Actions workflow <https://github.com/simonw/sqlite-utils/blob/main/.github/workflows/publish.yml>`__ will perform the following:
+
+* Run the unit tests against all supported Python versions. If the tests pass...
+* Build a wheel bundle of the underlying Python source code
+* Push that new wheel up to PyPI: https://pypi.org/project/sqlite-utils/
+
+To deploy new releases you will need to have push access to the GitHub repository.
+
+``sqlite-utils`` follows `Semantic Versioning <https://semver.org/>`__::
+
+    major.minor.patch
+
+We increment ``major`` for backwards-incompatible releases.
+
+We increment ``minor`` for new features.
+
+We increment ``patch`` for bugfix releass.
+
+To release a new version, first create a commit that updates the version number in ``pyproject.toml`` and the :ref:`the changelog <changelog>` with highlights of the new version. An example `commit can be seen here <https://github.com/simonw/sqlite-utils/commit/b491f22d817836829965516983a3f4c3c72c05fc>`__::
+
+    # Update changelog
+    git commit -m " Release 3.29
+
+    Refs #423, #458, #467, #469, #470, #471, #472, #475" -a
+    git push
+
+Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here <https://github.com/simonw/sqlite-utils/issues/458#ref-commit-b491f22>`__.
+
+You can generate the list of issue references for a specific release by copying and pasting text from the release notes or GitHub changes-since-last-release view into this `Extract issue numbers from pasted text <https://observablehq.com/@simonw/extract-issue-numbers-from-pasted-text>`__ tool.
+
+To create the tag for the release, create `a new release <https://github.com/simonw/sqlite-utils/releases/new>`__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool <https://euangoddard.github.io/clipboard2markdown/>`__.

docs/index.rst

@@ -2,7 +2,7 @@
  sqlite-utils |version|
 =======================
 
-|PyPI| |Changelog| |CI| |License|
+|PyPI| |Changelog| |CI| |License| |discord|
 
 .. |PyPI| image:: https://img.shields.io/pypi/v/sqlite-utils.svg
    :target: https://pypi.org/project/sqlite-utils/
@@ -12,8 +12,10 @@
    :target: https://github.com/simonw/sqlite-utils/actions
 .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
    :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE
+.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=discord
+   :target: https://discord.gg/Ass7bCAMDw
 
-*Python utility functions for manipulating SQLite databases*
+*CLI tool and Python library for manipulating SQLite databases*
 
 This library and command-line utility helps create SQLite databases from an existing collection of data.
 
@@ -23,6 +25,8 @@ sqlite-utils is not intended to be a full ORM: the focus is utility helpers to m
 
 It is designed as a useful complement to `Datasette <https://datasette.io/>`_.
 
+`Cleaning data with sqlite-utils and Datasette <https://datasette.io/tutorials/clean-data>`_ provides a tutorial introduction (and accompanying ten minute video) about using this tool.
+
 Contents
 --------
 
@@ -32,6 +36,10 @@ Contents
    installation
    cli
    python-api
+   migrations
+   plugins
+   reference
+   cli-reference
+   upgrading
+   contributing
    changelog
-
-Take a look at `this script <https://github.com/simonw/russian-ira-facebook-ads-datasette/blob/master/fetch_and_build_russian_ads.py>`_ for an example of this library in action.

docs/installation.rst

@@ -11,11 +11,11 @@
 Using Homebrew
 ==============
 
-The :ref:`sqlite-utils commad-line tool <cli>` can be installed on macOS using Homebrew::
+The :ref:`sqlite-utils command-line tool <cli>` can be installed on macOS using Homebrew::
 
     brew install sqlite-utils
 
-If you have it installed and want to upgrade to the most recent release, you ran run::
+If you have it installed and want to upgrade to the most recent release, you can run::
 
     brew upgrade sqlite-utils
 
@@ -38,3 +38,53 @@ Using pipx
 `pipx <https://pypi.org/project/pipx/>`__ is a tool for installing Python command-line applications in their own isolated environments. You can use ``pipx`` to install the ``sqlite-utils`` command-line tool like this::
 
     pipx install sqlite-utils
+
+.. _installation_sqlite3_alternatives:
+
+Alternatives to sqlite3
+=======================
+
+By default, ``sqlite-utils`` uses the ``sqlite3`` package bundled with the Python standard library.
+
+Depending on your operating system, this may come with some limitations.
+
+On some platforms the ability to load additional extensions (via ``conn.load_extension(...)`` or ``--load-extension=/path/to/extension``) may be disabled.
+
+You may also see the error ``sqlite3.OperationalError: table sqlite_master may not be modified`` when trying to alter an existing table.
+
+You can work around these limitations by installing the `pysqlite3 <https://pypi.org/project/pysqlite3/>`__ package, which provides a drop-in replacement for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions.
+
+To install ``pysqlite3`` run the following:
+
+.. code-block:: bash
+
+    sqlite-utils install pysqlite3
+
+``pysqlite3`` does not provide an implementation of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package:
+
+.. code-block:: bash
+
+    sqlite-utils install sqlite-dump
+
+.. _installation_completion:
+
+Setting up shell completion
+===========================
+
+You can configure shell tab completion for the ``sqlite-utils`` command using these commands.
+
+For ``bash``:
+
+.. code-block:: bash
+
+    eval "$(_SQLITE_UTILS_COMPLETE=bash_source sqlite-utils)"
+
+For ``zsh``:
+
+.. code-block:: zsh
+
+    eval "$(_SQLITE_UTILS_COMPLETE=zsh_source sqlite-utils)"
+
+Add this code to ``~/.zshrc`` or ``~/.bashrc`` to automatically run it when you start a new shell.
+
+See `the Click documentation <https://click.palletsprojects.com/en/8.1.x/shell-completion/>`__ for more details.

docs/migrations.rst

@@ -0,0 +1,194 @@
+.. _migrations:
+
+=====================
+ Database migrations
+=====================
+
+``sqlite-utils`` includes a migration system for applying repeatable changes to SQLite database files.
+
+A migration is a Python function that receives a :class:`sqlite_utils.Database` instance and then executes Python code to modify that database - creating or transforming tables, adding indexes, inserting rows, or any other operation supported by SQLite.
+
+Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database.
+
+This means you can run the migrate operation multiple times and it will only apply migrations that have not previously been recorded.
+
+.. _migrations_define:
+
+Defining migrations
+===================
+
+Ordered migration sets are defined by first creating a :class:`sqlite_utils.Migrations` object.
+
+Individual migrations are Python functions that are then registered with that migration set. Each migration function is passed a single argument that is a :ref:`sqlite_utils.Database <reference_db_database>` instance.
+
+The name passed to ``Migrations("creatures")`` identifies that set of migrations. Use a name that is unique for your project, since multiple migration sets can be applied to the same database.
+
+Here is a simple example of a ``migrations.py`` file which creates a table, then adds an extra column to that table in a second migration:
+
+.. code-block:: python
+
+    from sqlite_utils import Migrations
+
+    migrations = Migrations("creatures")
+
+    @migrations()
+    def create_table(db):
+        db["creatures"].create(
+            {"id": int, "name": str, "species": str},
+            pk="id",
+        )
+
+    @migrations()
+    def add_weight(db):
+        db["creatures"].add_column("weight", float)
+
+.. _migrations_python:
+
+Applying migrations in Python
+=============================
+
+Once you have a ``Migrations(name)`` collection with one or more migrations registered to it, you can execute them in Python code like this:
+
+.. code-block:: python
+
+    from sqlite_utils import Database
+
+    db = Database("creatures.db")
+    migrations.apply(db)
+
+Running ``migrations.apply(db)`` repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped.
+
+Migration functions are applied in the order that they were registered. The function name is used as the migration name unless you pass one explicitly:
+
+.. code-block:: python
+
+    @migrations(name="001_create_table")
+    def create_table(db):
+        db["creatures"].create({"id": int, "name": str}, pk="id")
+
+When you apply a set of migrations you can stop part way through by specifying a ``stop_before=`` migration name:
+
+.. code-block:: python
+
+    migrations.apply(db, stop_before="add_weight")
+
+.. _migrations_transactions:
+
+Migrations and transactions
+===========================
+
+Each migration runs inside a transaction, together with the ``_sqlite_migrations`` record of it having been applied. If a migration function raises an exception, everything it did is rolled back, no record is written and the migration stays pending - so fixing the error and re-applying will run that migration again from a clean state. Migrations that completed earlier in the same ``apply()`` run stay applied.
+
+Some operations cannot run inside a transaction, for example ``VACUUM`` or changing the journal mode with ``db.enable_wal()``. Register migrations like these with ``transactional=False``:
+
+.. code-block:: python
+
+    @migrations(transactional=False)
+    def compact(db):
+        db.execute("VACUUM")
+
+A migration registered with ``transactional=False`` runs without a wrapping transaction, so if it fails part way through any changes it already made will not be rolled back, and re-applying will run the whole function again.
+
+Avoid calling ``db.commit()`` or otherwise managing transactions manually inside a transactional migration - register the migration with ``transactional=False`` if it needs to control its own transactions. Using ``with db.atomic():`` blocks inside a migration is fine: they nest as savepoints within the migration's transaction, so the migration as a whole still commits or rolls back as a single unit. See :ref:`python_api_transactions`.
+
+Applying migrations using the CLI
+=================================
+
+Run migrations using the ``sqlite-utils migrate`` command:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db path/to/migrations.py
+
+The first argument is the database file. The remaining arguments can be paths to migration files or directories containing migration files.
+
+If you omit migration paths, ``sqlite-utils`` searches the current directory and subdirectories for files called ``migrations.py``:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db
+
+You can also pass a directory. Every ``migrations.py`` file in that directory tree will be considered:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db path/to/project/
+
+Running the command repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped.
+
+Listing migrations
+==================
+
+Use ``--list`` to show applied and pending migrations without running them. This is a read-only operation - it will not create the database file or the ``_sqlite_migrations`` table:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db --list
+
+Example output:
+
+.. code-block:: output
+
+    Migrations for: creatures
+
+      Applied:
+        create_table - 2026-06-09 17:23:12.048092+00:00
+        add_weight - 2026-06-09 17:23:12.051249+00:00
+
+      Pending:
+        add_age
+
+Stopping before a migration
+===========================
+
+When applying migrations using the CLI, you can stop before a named migration:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db path/to/migrations.py --stop-before add_weight
+
+This applies any pending migrations before ``add_weight`` and leaves ``add_weight`` and later migrations pending. An unqualified migration name matches in any migration set.
+
+You can also target a specific migration set using ``migration_set:migration_name``. This is useful if a migrations file contains more than one migration set, or if multiple sets use the same migration name:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db path/to/migrations.py \
+      --stop-before creatures:add_weight \
+      --stop-before sales:drop_index
+
+The ``--stop-before`` option can be passed more than once.
+
+If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. Naming a migration that has already been applied is also an error - stopping before it is impossible to honor - and no pending migrations are applied.
+
+Verbose output
+==============
+
+Use ``--verbose`` or ``-v`` to show the schema before and after migrations are applied, plus a unified diff when the schema changes:
+
+.. code-block:: bash
+
+    sqlite-utils migrate creatures.db --verbose
+
+Migrating from sqlite-migrate
+=============================
+
+This system uses the same migration table format as the older `sqlite-migrate <https://github.com/simonw/sqlite-migrate>`__ package. To use existing migration files directly with ``sqlite-utils``, update their import from ``sqlite_migrate`` to ``sqlite_utils``:
+
+.. code-block:: python
+
+    from sqlite_utils import Migrations
+
+    migration = Migrations("creatures")
+
+    @migration()
+    def create_table(db):
+        db["creatures"].create({"id": int, "name": str}, pk="id")
+
+Python API
+==========
+
+.. autoclass:: sqlite_utils.migrations.Migrations
+   :members:
+   :undoc-members:
+   :exclude-members: _Migration, _AppliedMigration

docs/plugins.rst

@@ -0,0 +1,159 @@
+.. _plugins:
+
+=========
+ Plugins
+=========
+
+``sqlite-utils`` supports plugins, which can be used to add extra features to the software.
+
+Plugins can add new commands, for example ``sqlite-utils some-command ...``
+
+Plugins can be installed using the ``sqlite-utils install`` command:
+
+.. code-block:: bash
+
+    sqlite-utils install sqlite-utils-name-of-plugin
+
+You can see a JSON list of plugins that have been installed by running this:
+
+.. code-block:: bash
+
+    sqlite-utils plugins
+
+Plugin hooks such as :ref:`plugins_hooks_prepare_connection` affect each instance of the ``Database`` class. You can opt-out of these plugins by creating that class instance like so:
+
+.. code-block:: python
+
+    db = Database(memory=True, execute_plugins=False)
+
+.. _plugins_building:
+
+Building a plugin
+-----------------
+
+Plugins are created in a directory named after the plugin. To create a "hello world" plugin, first create a ``hello-world`` directory:
+
+.. code-block:: bash
+
+    mkdir hello-world
+    cd hello-world
+
+In that folder create two files. The first is a ``pyproject.toml`` file describing the plugin:
+
+.. code-block:: toml
+
+    [project]
+    name = "sqlite-utils-hello-world"
+    version = "0.1"
+
+    [project.entry-points.sqlite_utils]
+    hello_world = "sqlite_utils_hello_world"
+
+The ``[project.entry-points.sqlite_utils]`` section tells ``sqlite-utils`` which module to load when executing the plugin.
+
+Then create ``sqlite_utils_hello_world.py`` with the following content:
+
+.. code-block:: python
+
+    import click
+    import sqlite_utils
+
+    @sqlite_utils.hookimpl
+    def register_commands(cli):
+        @cli.command()
+        def hello_world():
+            "Say hello world"
+            click.echo("Hello world!")
+
+Install the plugin in "editable" mode - so you can make changes to the code and have them picked up instantly by ``sqlite-utils`` - like this:
+
+.. code-block:: bash
+
+    sqlite-utils install -e .
+
+Or pass the path to your plugin directory:
+
+.. code-block:: bash
+
+    sqlite-utils install -e /dev/sqlite-utils-hello-world
+
+Now, running this should execute your new command:
+
+.. code-block:: bash
+
+    sqlite-utils hello-world
+
+Your command will also be listed in the output of ``sqlite-utils --help``.
+
+See the `LLM plugin documentation <https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#distributing-your-plugin>`__ for tips on distributing your plugin.
+
+.. _plugins_hooks:
+
+Plugin hooks
+------------
+
+Plugin hooks allow ``sqlite-utils`` to be customized.
+
+.. _plugins_hooks_register_commands:
+
+register_commands(cli)
+~~~~~~~~~~~~~~~~~~~~~~
+
+This hook can be used to register additional commands with the ``sqlite-utils`` CLI. It is called with the ``cli`` object, which is a ``click.Group`` instance.
+
+Example implementation:
+
+.. code-block:: python
+
+    import click
+    import sqlite_utils
+
+    @sqlite_utils.hookimpl
+    def register_commands(cli):
+        @cli.command()
+        def hello_world():
+            "Say hello world"
+            click.echo("Hello world!")
+
+New commands implemented by plugins can invoke existing commands using the `context.invoke <https://click.palletsprojects.com/en/stable/api/#click.Context.invoke>`__ mechanism.
+
+As a special niche feature, if your plugin needs to import some files and then act against an in-memory database containing those files you can forward to the :ref:`sqlite-utils memory command <cli_memory>` and pass it ``return_db=True``:
+
+.. code-block:: python
+
+    @cli.command()
+    @click.pass_context
+    @click.argument(
+        "paths",
+        type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+        required=False,
+        nargs=-1,
+    )
+    def show_schema_for_files(ctx, paths):
+        from sqlite_utils.cli import memory
+        db = ctx.invoke(memory, paths=paths, return_db=True)
+        # Now do something with that database
+        click.echo(db.schema)
+
+.. _plugins_hooks_prepare_connection:
+
+prepare_connection(conn)
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This hook is called when a new SQLite database connection is created. You can use it to `register custom SQL functions <https://docs.python.org/2/library/sqlite3.html#sqlite3.Connection.create_function>`_, aggregates and collations. For example:
+
+.. code-block:: python
+
+    import sqlite_utils
+
+    @sqlite_utils.hookimpl
+    def prepare_connection(conn):
+        conn.create_function(
+            "hello", 1, lambda name: f"Hello, {name}!"
+        )
+
+This registers a SQL function called ``hello`` which takes a single argument and can be called like this:
+
+.. code-block:: sql
+
+    select hello("world"); -- "Hello, world!"

docs/reference.rst

@@ -0,0 +1,117 @@
+.. _reference:
+
+===============
+ API reference
+===============
+
+.. contents:: :local:
+   :class: this-will-duplicate-information-and-it-is-still-useful-here
+
+.. _reference_db_database:
+
+sqlite_utils.db.Database
+========================
+
+.. autoclass:: sqlite_utils.db.Database
+    :members:
+    :undoc-members:
+    :special-members: __getitem__
+    :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys
+
+.. _reference_db_queryable:
+
+sqlite_utils.db.Queryable
+=========================
+
+:ref:`Table <reference_db_table>` and :ref:`View <reference_db_view>` are  both subclasses of ``Queryable``, providing access to the following methods:
+
+.. autoclass:: sqlite_utils.db.Queryable
+    :members:
+    :undoc-members:
+    :exclude-members: execute_count
+
+.. _reference_db_table:
+
+sqlite_utils.db.Table
+=====================
+
+.. autoclass:: sqlite_utils.db.Table
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :exclude-members: guess_foreign_column, value_or_default, build_insert_queries_and_params, insert_chunk, add_missing_columns
+
+.. _reference_db_view:
+
+sqlite_utils.db.View
+====================
+
+.. autoclass:: sqlite_utils.db.View
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. _reference_db_other:
+
+Other
+=====
+
+.. _reference_db_other_column:
+
+sqlite_utils.db.Column
+----------------------
+
+.. autoclass:: sqlite_utils.db.Column
+
+.. _reference_db_other_column_details:
+
+sqlite_utils.db.ColumnDetails
+-----------------------------
+
+.. autoclass:: sqlite_utils.db.ColumnDetails
+
+.. _reference_db_other_foreign_key:
+
+sqlite_utils.db.ForeignKey
+--------------------------
+
+.. autoclass:: sqlite_utils.db.ForeignKey
+
+sqlite_utils.utils
+==================
+
+.. _reference_utils_hash_record:
+
+sqlite_utils.utils.hash_record
+------------------------------
+
+.. autofunction:: sqlite_utils.utils.hash_record
+
+.. _reference_utils_rows_from_file:
+
+sqlite_utils.utils.rows_from_file
+---------------------------------
+
+.. autofunction:: sqlite_utils.utils.rows_from_file
+
+.. _reference_utils_typetracker:
+
+sqlite_utils.utils.TypeTracker
+------------------------------
+
+.. autoclass:: sqlite_utils.utils.TypeTracker
+   :members: wrap, types
+
+.. _reference_utils_chunks:
+
+sqlite_utils.utils.chunks
+-------------------------
+
+.. autofunction:: sqlite_utils.utils.chunks
+
+.. _reference_utils_flatten:
+
+sqlite_utils.utils.flatten
+--------------------------
+
+.. autofunction:: sqlite_utils.utils.flatten

docs/upgrading.rst

@@ -0,0 +1,146 @@
+.. _upgrading:
+
+===========
+ Upgrading
+===========
+
+This page describes the changes you may need to make to your own code or scripts when upgrading between major versions of ``sqlite-utils``.
+
+For the full list of changes in every release see the :ref:`changelog`.
+
+.. _upgrading_3_to_4:
+
+Upgrading from 3.x to 4.0
+=========================
+
+Requirements
+------------
+
+- Python 3.10 or higher is required.
+- The ``click`` dependency must be version 8.3.1 or later.
+
+Command-line changes
+--------------------
+
+**Type detection is now the default for CSV and TSV imports.** ``sqlite-utils insert`` and ``sqlite-utils upsert`` now detect column types when importing CSV or TSV data - previously every column was created as ``TEXT`` unless you passed ``--detect-types``. To restore the old behavior pass the new ``--no-detect-types`` flag:
+
+.. code-block:: bash
+
+    sqlite-utils insert data.db rows data.csv --csv --no-detect-types
+
+Two related things have been removed:
+
+- The ``SQLITE_UTILS_DETECT_TYPES`` environment variable.
+- The old ``-d/--detect-types`` flag itself. Since detection is now the default the flag did nothing - remove it from any scripts that used it.
+
+**The convert command no longer skips falsey values.** ``sqlite-utils convert`` previously skipped values that evaluated to ``False`` (empty strings, ``0``) unless you passed ``--no-skip-false``. All values are now converted and the ``--no-skip-false`` flag has been removed.
+
+**drop-table and drop-view check the object type.** ``sqlite-utils drop-table`` now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. If you relied on that (unlikely), use the matching command instead.
+
+**sqlite-utils tui has moved to a plugin.** The optional terminal interface is now provided by the `sqlite-utils-tui <https://github.com/simonw/sqlite-utils-tui>`__ plugin:
+
+.. code-block:: bash
+
+    sqlite-utils install sqlite-utils-tui
+
+Python API changes
+------------------
+
+**db.query() now rejects SQL that does not return rows.** This is likely the most common change you will need to make to existing code. ``db.query()`` used to accept any SQL statement - passing one that returns no rows, such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause or a ``CREATE TABLE``, did nothing at all, silently. Those statements now raise a ``ValueError``, and are rolled back so they have no effect on the database. Transaction control statements (``BEGIN``, ``COMMIT``, ``END``, ``ROLLBACK``, ``SAVEPOINT``, ``RELEASE``) plus ``VACUUM``, ``ATTACH`` and ``DETACH`` are also rejected with a ``ValueError``, without being executed at all. Use ``db.execute()`` for statements that do not return rows:
+
+.. code-block:: python
+
+    # 3.x accepted this but silently did nothing:
+    db.query("update dogs set name = 'Cleopaws'")
+
+    # In 4.0 use execute() for SQL that does not return rows:
+    db.execute("update dogs set name = 'Cleopaws'")
+
+**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily, but errors in your SQL raise at the ``db.query()`` call site rather than on first iteration, and a write with a ``RETURNING`` clause takes effect even if you never iterate over its results.
+
+**db.table() no longer returns views.** ``db.table(name)`` now raises a ``sqlite_utils.db.NoTable`` exception if ``name`` is a SQL view. Use the new ``db.view(name)`` method for views:
+
+.. code-block:: python
+
+    table = db.table("my_table")
+    view = db.view("my_view")
+
+``db["name"]`` still returns either a ``Table`` or a ``View`` depending on what exists in the database.
+
+**Upserts use INSERT ... ON CONFLICT.** Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax rather than the previous ``INSERT OR IGNORE`` followed by ``UPDATE``. If your code depends on the old behavior, pass ``use_old_upsert=True`` to the ``Database()`` constructor - see :ref:`python_api_old_upsert`.
+
+**Upsert records must include their primary keys.** ``table.upsert()`` and ``table.upsert_all()`` now raise ``sqlite_utils.db.PrimaryKeyRequired`` if a record is missing a value for any primary key column (or has ``None`` for one). Previously such records were quietly inserted as new rows. Relatedly, ``pk=`` is now optional when the table already exists with a primary key - it is detected automatically.
+
+**Floating point columns are now REAL.** Auto-detected floating point columns are created with the correct SQLite type ``REAL`` instead of ``FLOAT``. Code that inspects column types should expect ``REAL``.
+
+**Generated schemas use double quotes.** Tables created by this library now wrap table and column names in ``"double-quotes"`` where they previously used ``[square-braces]``. If you compare ``table.schema`` strings against expected values you will need to update them.
+
+**table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values.
+
+**Null values are no longer extracted into lookup tables.** ``table.extract()`` and the ``sqlite-utils extract`` command leave rows alone if every extracted column is ``null`` - the new foreign key column is left as ``null`` instead of pointing at an all-``null`` record in the lookup table. The ``extracts=`` insert option similarly keeps ``None`` values as ``null``. Relatedly, ``table.lookup()`` now compares values using ``IS`` so that looking up a value containing ``None`` returns the existing matching row - previously it inserted a duplicate row on every call.
+
+**ensure_autocommit_off() is now ensure_autocommit_on().** The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``. The old name described the opposite of what the method did: it temporarily puts the connection into driver-level autocommit mode (by setting ``isolation_level = None``), so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. The behavior is unchanged - update any calls to use the new name.
+
+**View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method.
+
+**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained new fields - ``columns``, ``other_columns``, ``is_compound``, ``on_delete`` and ``on_update`` - so that compound (multi-column) foreign keys and foreign key actions can be represented. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead:
+
+.. code-block:: python
+
+    # 3.x - tuple unpacking, no longer works:
+    for table, column, other_table, other_column in db["courses"].foreign_keys:
+        ...
+
+    # 4.0 - access fields by name:
+    for fk in db["courses"].foreign_keys:
+        fk.table, fk.column, fk.other_table, fk.other_column
+
+Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. Like the old namedtuple, ``ForeignKey`` instances are immutable and hashable - they can be collected into sets and used as dictionary keys. Note that equality now includes the ``on_delete`` and ``on_update`` actions: a ``ForeignKey`` with ``ON DELETE CASCADE`` is not equal to one without.
+
+Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` tuples instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are one-item tuples.
+
+Two related behavior changes to ``table.transform()``: compound foreign keys now survive a transform (previously they were split into separate single-column keys), and ``ON DELETE``/``ON UPDATE`` actions such as ``ON DELETE CASCADE`` are now preserved (previously they were silently stripped from the schema).
+
+**Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``.
+
+**Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() <python_api_atomic>` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice:
+
+- Write statements executed with raw ``db.execute()`` calls now commit automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that nothing committed - if your code used ``db.execute()`` for writes and relied on ``db.conn.rollback()`` to undo them, open an explicit transaction with the new ``db.begin()`` method first.
+- Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead.
+- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, instead of silently committing it.
+- Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - a transaction you explicitly opened with ``db.begin()`` and did not commit is rolled back.
+- ``Database()`` rejects connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options, raising ``sqlite_utils.db.TransactionError``. On those connections every write the library made was silently discarded when the connection closed.
+
+Packaging changes
+-----------------
+
+- ``sqlite-utils`` now uses ``pyproject.toml`` in place of ``setup.py``.
+- ``pip`` is now a runtime dependency, used by the ``sqlite-utils install`` and ``uninstall`` commands.
+
+New features to be aware of
+---------------------------
+
+Not breaking changes, but new in 4.0 and worth knowing about when you upgrade:
+
+- A :ref:`database migrations system <migrations>`, incorporating the functionality of the ``sqlite-migrate`` plugin. If you used that plugin, the built-in system reads the same ``_sqlite_migrations`` table - your applied migrations will not run again. Update your migration files to use ``from sqlite_utils import Migrations``.
+- :ref:`db.atomic() <python_api_atomic>` for nested transaction support.
+- ``table.insert_all()`` and ``table.upsert_all()`` accept an iterator of lists or tuples as an alternative to dictionaries - see :ref:`python_api_insert_lists`.
+
+.. _upgrading_2_to_3:
+
+Upgrading from 2.x to 3.0
+=========================
+
+The 3.0 release redesigned search. The breaking changes were minor:
+
+- ``table.search()`` returns a generator of dictionaries, sorted by relevance. It previously returned a list of tuples sorted by ``rowid``.
+- The ``-c`` shortcut for ``--csv`` and the ``-f`` shortcut for ``--fmt`` were removed from the CLI - use the full option names.
+
+.. _upgrading_1_to_2:
+
+Upgrading from 1.x to 2.0
+=========================
+
+The 2.0 release changed the meaning of *upsert*. In 1.x, ``table.upsert()`` and ``table.upsert_all()`` actually performed ``INSERT OR REPLACE`` operations - entirely replacing the existing row. Since 2.0 an upsert updates only the columns you provide, leaving other columns untouched.
+
+If you want the 1.x behavior, use ``table.insert(..., replace=True)`` or ``table.insert_all(..., replace=True)`` instead.

mypy.ini

@@ -0,0 +1,32 @@
+[mypy]
+python_version = 3.10
+warn_return_any = False
+warn_unused_configs = True
+warn_redundant_casts = False
+warn_unused_ignores = False
+check_untyped_defs = True
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+no_implicit_optional = True
+strict_equality = True
+
+[mypy-sqlite_utils.cli]
+ignore_errors = True
+
+[mypy-pysqlite3.*]
+ignore_missing_imports = True
+
+[mypy-sqlite_dump.*]
+ignore_missing_imports = True
+
+[mypy-sqlite_fts4.*]
+ignore_missing_imports = True
+
+[mypy-pandas.*]
+ignore_missing_imports = True
+
+[mypy-numpy.*]
+ignore_missing_imports = True
+
+[mypy-tests.*]
+ignore_errors = True

pyproject.toml

@@ -0,0 +1,85 @@
+[project]
+name = "sqlite-utils"
+version = "4.1.1"
+description = "CLI tool and Python library for manipulating SQLite databases"
+readme = { file = "README.md", content-type = "text/markdown" }
+authors = [
+    { name = "Simon Willison" },
+]
+license = "Apache-2.0"
+requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: End Users/Desktop",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Database",
+]
+
+dependencies = [
+    "click>=8.3.1",
+    "click-default-group>=1.2.3",
+    "pluggy",
+    "python-dateutil",
+    "sqlite-fts4",
+    "tabulate",
+    "pip",
+]
+
+[dependency-groups]
+dev = [
+    "black>=26.3.1",
+    "click>=8.4.2",
+    "cogapp",
+    "hypothesis",
+    "pytest",
+    # mypy
+    "data-science-types",
+    "mypy",
+    "types-click",
+    "types-pluggy",
+    "types-python-dateutil",
+    "types-tabulate",
+    # flake8
+    "flake8",
+    "flake8-pyproject",
+    "ty>=0.0.37",
+    # For stable cog:
+    "tabulate>=0.10.0",
+]
+docs = [
+    "codespell",
+    "furo",
+    "pygments-csv-lexer",
+    "sphinx-autobuild",
+    "sphinx-copybutton",
+]
+
+[project.urls]
+Homepage = "https://github.com/simonw/sqlite-utils"
+Documentation = "https://sqlite-utils.datasette.io/en/stable/"
+Changelog = "https://sqlite-utils.datasette.io/en/stable/changelog.html"
+Issues = "https://github.com/simonw/sqlite-utils/issues"
+CI = "https://github.com/simonw/sqlite-utils/actions"
+
+[project.scripts]
+sqlite-utils = "sqlite_utils.cli:cli"
+
+[build-system]
+# setuptools 77+ is needed for the PEP 639 license = "Apache-2.0" expression
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[tool.flake8]
+max-line-length = 160
+# Black compatibility, E203 whitespace before ':':
+extend-ignore = ["E203"]
+extend-exclude = [".venv", "build", "dist", "docs", "sqlite_utils.egg-info"]
+
+[tool.setuptools.package-data]
+sqlite_utils = ["py.typed"]

setup.py

@@ -1,57 +0,0 @@
-from setuptools import setup, find_packages
-import io
-import os
-
-VERSION = "3.11"
-
-
-def get_long_description():
-    with io.open(
-        os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"),
-        encoding="utf8",
-    ) as fp:
-        return fp.read()
-
-
-setup(
-    name="sqlite-utils",
-    description="CLI tool and Python utility functions for manipulating SQLite databases",
-    long_description=get_long_description(),
-    long_description_content_type="text/markdown",
-    author="Simon Willison",
-    version=VERSION,
-    license="Apache License, Version 2.0",
-    packages=find_packages(exclude=["tests", "tests.*"]),
-    install_requires=["sqlite-fts4", "click", "click-default-group", "tabulate"],
-    setup_requires=["pytest-runner"],
-    extras_require={
-        "test": ["pytest", "black", "hypothesis"],
-        "docs": ["sphinx_rtd_theme", "sphinx-autobuild"],
-    },
-    entry_points="""
-        [console_scripts]
-        sqlite-utils=sqlite_utils.cli:cli
-    """,
-    tests_require=["sqlite-utils[test]"],
-    url="https://github.com/simonw/sqlite-utils",
-    project_urls={
-        "Documentation": "https://sqlite-utils.datasette.io/en/stable/",
-        "Changelog": "https://sqlite-utils.datasette.io/en/stable/changelog.html",
-        "Source code": "https://github.com/simonw/sqlite-utils",
-        "Issues": "https://github.com/simonw/sqlite-utils/issues",
-        "CI": "https://github.com/simonw/sqlite-utils/actions",
-    },
-    python_requires=">=3.6",
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Intended Audience :: Developers",
-        "Intended Audience :: Science/Research",
-        "Intended Audience :: End Users/Desktop",
-        "Topic :: Database",
-        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-    ],
-)

sqlite_utils/__init__.py

@@ -1,4 +1,7 @@
-from .db import Database
 from .utils import suggest_column_types
+from .hookspecs import hookimpl
+from .hookspecs import hookspec
+from .db import Database
+from .migrations import Migrations
 
-__all__ = ["Database", "suggest_column_types"]
+__all__ = ["Database", "Migrations", "suggest_column_types", "hookimpl", "hookspec"]

sqlite_utils/__main__.py

@@ -0,0 +1,4 @@
+from .cli import cli
+
+if __name__ == "__main__":
+    cli()

sqlite_utils/hookspecs.py

@@ -0,0 +1,18 @@
+import sqlite3
+
+import click
+from pluggy import HookimplMarker
+from pluggy import HookspecMarker
+
+hookspec = HookspecMarker("sqlite_utils")
+hookimpl = HookimplMarker("sqlite_utils")
+
+
+@hookspec
+def register_commands(cli: click.Group) -> None:
+    """Register additional CLI commands, e.g. 'sqlite-utils mycommand ...'"""
+
+
+@hookspec
+def prepare_connection(conn: sqlite3.Connection) -> None:
+    """Modify SQLite connection in some way e.g. register custom SQL functions"""

sqlite_utils/migrations.py

@@ -0,0 +1,177 @@
+from collections.abc import Iterable
+from dataclasses import dataclass
+import datetime
+from typing import Callable, cast, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from sqlite_utils.db import Database, Table
+
+
+class Migrations:
+    migrations_table = "_sqlite_migrations"
+
+    @dataclass
+    class _Migration:
+        name: str
+        fn: Callable
+        transactional: bool = True
+
+    @dataclass
+    class _AppliedMigration:
+        name: str
+        # A string timestamp such as "2026-07-04 12:00:00.000000+00:00" -
+        # stored as TEXT in the _sqlite_migrations table
+        applied_at: str
+
+    def __init__(self, name: str):
+        """
+        :param name: The name of the migration set. This should be unique.
+        """
+        self.name = name
+        self._migrations: list[Migrations._Migration] = []
+
+    def __call__(
+        self, *, name: str | None = None, transactional: bool = True
+    ) -> Callable:
+        """
+        :param name: The name to use for this migration - if not provided,
+          the name of the function will be used.
+        :param transactional: If ``True`` (the default) the migration and the
+          record of it having been applied are wrapped in a transaction, which
+          will be rolled back if the migration raises an exception. Pass
+          ``False`` for migrations that cannot run inside a transaction, for
+          example those that execute ``VACUUM``.
+        """
+
+        def inner(func: Callable) -> Callable:
+            migration_name = name or getattr(func, "__name__")
+            if any(m.name == migration_name for m in self._migrations):
+                raise ValueError(
+                    "Migration '{}' is already registered in set '{}'".format(
+                        migration_name, self.name
+                    )
+                )
+            self._migrations.append(
+                self._Migration(migration_name, func, transactional)
+            )
+            return func
+
+        return inner
+
+    def pending(self, db: "Database") -> list["Migrations._Migration"]:
+        """
+        Return a list of pending migrations.
+
+        This is a read-only operation - it does not write to the database.
+        """
+        already_applied = {migration.name for migration in self.applied(db)}
+        return [
+            migration
+            for migration in self._migrations
+            if migration.name not in already_applied
+        ]
+
+    def applied(self, db: "Database") -> list["Migrations._AppliedMigration"]:
+        """
+        Return a list of applied migrations, in the order they were applied.
+
+        This is a read-only operation - it does not write to the database.
+        """
+        table = _table(db, self.migrations_table)
+        if not table.exists():
+            return []
+        return [
+            self._AppliedMigration(name=row["name"], applied_at=row["applied_at"])
+            for row in table.rows_where(
+                "migration_set = ?", [self.name], order_by="rowid"
+            )
+        ]
+
+    def apply(self, db: "Database", *, stop_before: str | Iterable[str] | None = None):
+        """
+        Apply any pending migrations to the database.
+
+        Each migration runs inside a transaction, together with the record of
+        it having been applied - if the migration raises an exception its
+        changes are rolled back, no record is written and the migration stays
+        pending. Migrations registered with ``transactional=False`` run
+        outside of a transaction.
+
+        :raises ValueError: if a ``stop_before`` name matches a migration in
+          this set that has already been applied - stopping before it is
+          impossible to honor, and no pending migrations are applied
+        """
+        if stop_before is None:
+            stop_before_names = set()
+        elif isinstance(stop_before, str):
+            stop_before_names = {stop_before}
+        else:
+            stop_before_names = set(stop_before)
+        # A stop_before naming an already-applied migration cannot be
+        # honored - error rather than applying everything after it. Names
+        # not in this set at all are ignored, because unqualified CLI
+        # values are offered to every migration set
+        already_applied = stop_before_names.intersection(
+            migration.name for migration in self.applied(db)
+        )
+        if already_applied:
+            raise ValueError(
+                "Cannot stop before migration{} {} in set '{}' - already "
+                "been applied".format(
+                    "s" if len(already_applied) > 1 else "",
+                    ", ".join(sorted(already_applied)),
+                    self.name,
+                )
+            )
+        self.ensure_migrations_table(db)
+        for migration in self.pending(db):
+            name = migration.name
+            if name in stop_before_names:
+                return
+            if migration.transactional:
+                with db.atomic():
+                    migration.fn(db)
+                    self._record_applied(db, name)
+            else:
+                migration.fn(db)
+                self._record_applied(db, name)
+
+    def _record_applied(self, db: "Database", name: str):
+        _table(db, self.migrations_table).insert(
+            {
+                "migration_set": self.name,
+                "name": name,
+                "applied_at": str(datetime.datetime.now(datetime.timezone.utc)),
+            }
+        )
+
+    def ensure_migrations_table(self, db: "Database"):
+        """
+        Ensure the _sqlite_migrations table exists and has the correct schema.
+        """
+        table = _table(db, self.migrations_table)
+        if not table.exists():
+            table.create(
+                {
+                    "id": int,
+                    "migration_set": str,
+                    "name": str,
+                    "applied_at": str,
+                },
+                pk="id",
+            )
+            table.create_index(["migration_set", "name"], unique=True)
+        elif table.pks != ["id"]:
+            table.transform(pk="id")
+            unique_indexes = {tuple(index.columns) for index in table.indexes}
+            if ("migration_set", "name") not in unique_indexes:
+                table.create_index(["migration_set", "name"], unique=True)
+
+    def __repr__(self):
+        return "<Migrations '{}': [{}]>".format(
+            self.name, ", ".join(m.name for m in self._migrations)
+        )
+
+
+def _table(db: "Database", name: str) -> "Table":
+    return cast("Table", db[name])

sqlite_utils/plugins.py

@@ -0,0 +1,35 @@
+from typing import Dict, List, Union
+
+import pluggy
+import sys
+from . import hookspecs
+
+pm: pluggy.PluginManager = pluggy.PluginManager("sqlite_utils")
+pm.add_hookspecs(hookspecs)
+_plugins_loaded = False
+
+
+def ensure_plugins_loaded() -> None:
+    global _plugins_loaded
+    if _plugins_loaded or getattr(sys, "_called_from_test", False):
+        return
+    pm.load_setuptools_entrypoints("sqlite_utils")
+    _plugins_loaded = True
+
+
+def get_plugins() -> List[Dict[str, Union[str, List[str]]]]:
+    ensure_plugins_loaded()
+    plugins: List[Dict[str, Union[str, List[str]]]] = []
+    plugin_to_distinfo = dict(pm.list_plugin_distinfo())
+    for plugin in pm.get_plugins():
+        hookcallers = pm.get_hookcallers(plugin) or []
+        plugin_info: Dict[str, Union[str, List[str]]] = {
+            "name": plugin.__name__,
+            "hooks": [h.name for h in hookcallers],
+        }
+        distinfo = plugin_to_distinfo.get(plugin)
+        if distinfo:
+            plugin_info["version"] = distinfo.version
+            plugin_info["name"] = distinfo.project_name
+        plugins.append(plugin_info)
+    return plugins

sqlite_utils/recipes.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from typing import Callable, Optional
+
+from dateutil import parser
+import json
+
+IGNORE: object = object()
+SET_NULL: object = object()
+
+
+def parsedate(
+    value: str,
+    dayfirst: bool = False,
+    yearfirst: bool = False,
+    errors: Optional[object] = None,
+) -> Optional[str]:
+    """
+    Parse a date and convert it to ISO date format: yyyy-mm-dd
+    \b
+    - dayfirst=True: treat xx as the day in xx/yy/zz
+    - yearfirst=True: treat xx as the year in xx/yy/zz
+    - errors=r.IGNORE to ignore values that cannot be parsed
+    - errors=r.SET_NULL to set values that cannot be parsed to null
+    """
+    if not value:
+        return value
+    try:
+        return (
+            parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst)
+            .date()
+            .isoformat()
+        )
+    except parser.ParserError:
+        if errors is IGNORE:
+            return value
+        elif errors is SET_NULL:
+            return None
+        else:
+            raise
+
+
+def parsedatetime(
+    value: str,
+    dayfirst: bool = False,
+    yearfirst: bool = False,
+    errors: Optional[object] = None,
+) -> Optional[str]:
+    """
+    Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS
+    \b
+    - dayfirst=True: treat xx as the day in xx/yy/zz
+    - yearfirst=True: treat xx as the year in xx/yy/zz
+    - errors=r.IGNORE to ignore values that cannot be parsed
+    - errors=r.SET_NULL to set values that cannot be parsed to null
+    """
+    if not value:
+        return value
+    try:
+        return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat()
+    except parser.ParserError:
+        if errors is IGNORE:
+            return value
+        elif errors is SET_NULL:
+            return None
+        else:
+            raise
+
+
+def jsonsplit(
+    value: str, delimiter: str = ",", type: Callable[[str], object] = str
+) -> str:
+    """
+    Convert a string like a,b,c into a JSON array ["a", "b", "c"]
+    """
+    return json.dumps([type(s.strip()) for s in value.split(delimiter)])

sqlite_utils/utils.py

@@ -2,40 +2,154 @@ import base64
 import contextlib
 import csv
 import enum
+import hashlib
+import importlib
 import io
+import itertools
 import json
 import os
-from typing import Generator
+import sys
+from typing import (
+    Any,
+    BinaryIO,
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    TYPE_CHECKING,
+    TypeVar,
+    Union,
+    cast,
+)
 
 import click
 
-try:
+from . import recipes
-    import pysqlite3 as sqlite3
-    import pysqlite3.dbapi2
 
-    OperationalError = pysqlite3.dbapi2.OperationalError
+if TYPE_CHECKING:
-except ImportError:
+    import sqlite3  # noqa: F401
-    import sqlite3
+    from sqlite3 import dbapi2  # noqa: F401
+
+    OperationalError = dbapi2.OperationalError
+else:
+    try:
+        sqlite3 = importlib.import_module("pysqlite3")
+        dbapi2 = importlib.import_module("pysqlite3.dbapi2")
+        OperationalError = dbapi2.OperationalError
+    except ImportError:
+        import sqlite3  # noqa: F401
+        from sqlite3 import dbapi2  # noqa: F401
+
+        OperationalError = dbapi2.OperationalError
 
-    OperationalError = sqlite3.OperationalError
 
 SPATIALITE_PATHS = (
     "/usr/lib/x86_64-linux-gnu/mod_spatialite.so",
+    "/usr/lib/aarch64-linux-gnu/mod_spatialite.so",
     "/usr/local/lib/mod_spatialite.dylib",
+    "/usr/local/lib/mod_spatialite.so",
+    "/opt/homebrew/lib/mod_spatialite.dylib",
 )
 
+# Mainly so we can restore it if needed in the tests:
+ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit()
 
-def suggest_column_types(records):
+# Type alias for row dictionaries - values can be various SQLite-compatible types
-    all_column_types = {}
+RowValue = Union[None, int, float, str, bytes, bool, List[str]]
+Row = Dict[str, RowValue]
+
+T = TypeVar("T")
+
+
+class _CloseableIterator(Iterator[Row]):
+    """Iterator wrapper that closes a file when iteration is complete."""
+
+    def __init__(self, iterator: Iterator[Row], closeable: io.IOBase) -> None:
+        self._iterator = iterator
+        self._closeable = closeable
+
+    def __iter__(self) -> "_CloseableIterator":
+        return self
+
+    def __next__(self) -> Row:
+        try:
+            return next(self._iterator)
+        except StopIteration:
+            self._closeable.close()
+            raise
+
+    def close(self) -> None:
+        self._closeable.close()
+
+
+def maximize_csv_field_size_limit() -> None:
+    """
+    Increase the CSV field size limit to the maximum possible.
+    """
+    # https://stackoverflow.com/a/15063941
+    field_size_limit = sys.maxsize
+
+    while True:
+        try:
+            csv.field_size_limit(field_size_limit)
+            break
+        except OverflowError:
+            field_size_limit = int(field_size_limit / 10)
+
+
+def find_spatialite() -> Optional[str]:
+    """
+    The ``find_spatialite()`` function searches for the `SpatiaLite <https://www.gaia-gis.it/fossil/libspatialite/index>`__
+    SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found.
+
+    You can use it in code like this:
+
+    .. code-block:: python
+
+        from sqlite_utils import Database
+        from sqlite_utils.utils import find_spatialite
+
+        db = Database("mydb.db")
+        spatialite = find_spatialite()
+        if spatialite:
+            db.conn.enable_load_extension(True)
+            db.conn.load_extension(spatialite)
+
+        # or use with db.init_spatialite like this
+        db.init_spatialite(find_spatialite())
+
+    """
+    for path in SPATIALITE_PATHS:
+        if os.path.exists(path):
+            return path
+    return None
+
+
+def suggest_column_types(
+    records: Iterable[Dict[str, Any]],
+) -> Dict[str, type]:
+    all_column_types: Dict[str, Set[type]] = {}
     for record in records:
         for key, value in record.items():
             all_column_types.setdefault(key, set()).add(type(value))
-    column_types = {}
+    return types_for_column_types(all_column_types)
 
+
+def types_for_column_types(
+    all_column_types: Dict[str, Set[type]],
+) -> Dict[str, type]:
+    column_types: Dict[str, type] = {}
     for key, types in all_column_types.items():
         # Ignore null values if at least one other type present:
         if len(types) > 1:
             types.discard(None.__class__)
+        t: type
         if {None.__class__} == types:
             t = str
         elif len(types) == 1:
@@ -57,7 +171,7 @@ def suggest_column_types(records):
     return column_types
 
 
-def column_affinity(column_type):
+def column_affinity(column_type: str) -> type:
     # Implementation of SQLite affinity rules from
     # https://www.sqlite.org/datatype3.html#determination_of_column_affinity
     assert isinstance(column_type, str)
@@ -76,40 +190,42 @@ def column_affinity(column_type):
     return float
 
 
-def decode_base64_values(doc):
+def decode_base64_values(doc: Dict[str, Any]) -> Dict[str, Any]:
     # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them
     to_fix = [
         k
         for k in doc
         if isinstance(doc[k], dict)
-        and doc[k].get("$base64") is True
+        and cast(dict, doc[k]).get("$base64") is True
-        and "encoded" in doc[k]
+        and "encoded" in cast(dict, doc[k])
     ]
     if not to_fix:
         return doc
-    return dict(doc, **{k: base64.b64decode(doc[k]["encoded"]) for k in to_fix})
+    return dict(
-
+        doc, **{k: base64.b64decode(cast(dict, doc[k])["encoded"]) for k in to_fix}
-
+    )
-def find_spatialite():
-    for path in SPATIALITE_PATHS:
-        if os.path.exists(path):
-            return path
-    return None
 
 
 class UpdateWrapper:
-    def __init__(self, wrapped, update):
+    def __init__(self, wrapped: io.IOBase, update: Callable[[int], None]) -> None:
         self._wrapped = wrapped
         self._update = update
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[bytes]:
         for line in self._wrapped:
             self._update(len(line))
             yield line
 
+    def read(self, size: int = -1) -> bytes:
+        data = self._wrapped.read(size)
+        self._update(len(data))
+        return data
+
 
 @contextlib.contextmanager
-def file_progress(file, silent=False, **kwargs):
+def file_progress(
+    file: io.IOBase, silent: bool = False, **kwargs: object
+) -> Generator[Union[io.IOBase, "UpdateWrapper"], None, None]:
     if silent:
         yield file
         return
@@ -122,8 +238,8 @@ def file_progress(file, silent=False, **kwargs):
     if fileno == 0:  # 0 means stdin
         yield file
     else:
-        file_length = os.path.getsize(file.name)
+        file_length = os.path.getsize(file.name)  # type: ignore
-        with click.progressbar(length=file_length, **kwargs) as bar:
+        with click.progressbar(length=file_length, **kwargs) as bar:  # type: ignore
             yield UpdateWrapper(file, bar.update)
 
 
@@ -142,13 +258,87 @@ class RowsFromFileBadJSON(RowsFromFileError):
     pass
 
 
+class RowError(Exception):
+    pass
+
+
+def _extra_key_strategy(
+    reader: Iterable[Dict[Optional[str], object]],
+    ignore_extras: Optional[bool] = False,
+    extras_key: Optional[str] = None,
+) -> Iterable[Row]:
+    # Logic for handling CSV rows with more values than there are headings
+    for row in reader:
+        # DictReader adds a 'None' key with extra row values
+        if None not in row:
+            yield cast(Row, row)
+        elif ignore_extras:
+            # ignoring row.pop(none) because of this issue:
+            # https://github.com/simonw/sqlite-utils/issues/440#issuecomment-1155358637
+            row.pop(None)
+            yield cast(Row, row)
+        elif not extras_key:
+            extras = row.pop(None)
+            raise RowError(
+                "Row {} contained these extra values: {}".format(row, extras)
+            )
+        else:
+            extras_value = row.pop(None)
+            row_out = cast(Row, row)
+            row_out[extras_key] = cast(RowValue, extras_value)
+            yield row_out
+
+
 def rows_from_file(
-    fp,
+    fp: BinaryIO,
-    format=None,
+    format: Optional[Format] = None,
-    dialect=None,
+    dialect: Optional[Type[csv.Dialect]] = None,
-    encoding=None,
+    encoding: Optional[str] = None,
-    detect_types=False,
+    ignore_extras: Optional[bool] = False,
-) -> Generator[dict, None, None]:
+    extras_key: Optional[str] = None,
+) -> Tuple[Iterable[Row], Format]:
+    """
+    Load a sequence of dictionaries from a file-like object containing one of four different formats.
+
+    .. code-block:: python
+
+        from sqlite_utils.utils import rows_from_file
+        import io
+
+        rows, format = rows_from_file(io.StringIO("id,name\\n1,Cleo")))
+        print(list(rows), format)
+        # Outputs [{'id': '1', 'name': 'Cleo'}] Format.CSV
+
+    This defaults to attempting to automatically detect the format of the data, or you can pass in an
+    explicit format using the format= option.
+
+    Returns a tuple of ``(rows_generator, format_used)`` where ``rows_generator`` can be iterated over
+    to return dictionaries, while ``format_used`` is a value from the ``sqlite_utils.utils.Format`` enum:
+
+    .. code-block:: python
+
+        class Format(enum.Enum):
+            CSV = 1
+            TSV = 2
+            JSON = 3
+            NL = 4
+
+    If a CSV or TSV file includes rows with more fields than are declared in the header a
+    ``sqlite_utils.utils.RowError`` exception will be raised when you loop over the generator.
+
+    You can instead ignore the extra data by passing ``ignore_extras=True``.
+
+    Or pass ``extras_key="rest"`` to put those additional values in a list in a key called ``rest``.
+
+    :param fp: a file-like object containing binary data
+    :param format: the format to use - omit this to detect the format
+    :param dialect: the CSV dialect to use - omit this to detect the dialect
+    :param encoding: the character encoding to use when reading CSV/TSV data
+    :param ignore_extras: ignore any extra fields on rows
+    :param extras_key: put any extra fields in a list with this key
+    """
+    if ignore_extras and extras_key:
+        raise ValueError("Cannot use ignore_extras= and extras_key= together")
     if format == Format.JSON:
         decoded = json.load(fp)
         if isinstance(decoded, dict):
@@ -159,19 +349,36 @@ def rows_from_file(
     elif format == Format.NL:
         return (json.loads(line) for line in fp if line.strip()), Format.NL
     elif format == Format.CSV:
-        decoded_fp = io.TextIOWrapper(fp, encoding=encoding or "utf-8-sig")
+        use_encoding: str = encoding or "utf-8-sig"
-        return csv.DictReader(decoded_fp, dialect=dialect), Format.CSV
+        decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding)
+        if dialect is not None:
+            reader = csv.DictReader(decoded_fp, dialect=dialect)
+        else:
+            reader = csv.DictReader(decoded_fp)
+        rows = _extra_key_strategy(reader, ignore_extras, extras_key)
+        return _CloseableIterator(iter(rows), decoded_fp), Format.CSV
     elif format == Format.TSV:
+        rows, _ = rows_from_file(
+            fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding
+        )
         return (
-            rows_from_file(
+            _extra_key_strategy(
-                fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding
+                cast(Iterable[Dict[Optional[str], object]], rows),
-            )[0],
+                ignore_extras,
+                extras_key,
+            ),
             Format.TSV,
         )
     elif format is None:
         # Detect the format, then call this recursively
-        buffered = io.BufferedReader(fp, buffer_size=4096)
+        buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096)
-        first_bytes = buffered.peek(2048).strip()
+        try:
+            first_bytes = buffered.peek(2048).strip()
+        except AttributeError:
+            # Likely the user passed a TextIO when this needs a BytesIO
+            raise TypeError(
+                "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO"
+            )
         if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"):
             # TODO: Detect newline-JSON
             return rows_from_file(buffered, format=Format.JSON)
@@ -179,18 +386,54 @@ def rows_from_file(
             dialect = csv.Sniffer().sniff(
                 first_bytes.decode(encoding or "utf-8-sig", "ignore")
             )
-            return rows_from_file(
+            rows, _ = rows_from_file(
                 buffered, format=Format.CSV, dialect=dialect, encoding=encoding
             )
+            # Make sure we return the format we detected
+            detected_format = Format.TSV if dialect.delimiter == "\t" else Format.CSV
+            return (
+                _extra_key_strategy(
+                    cast(Iterable[Dict[Optional[str], object]], rows),
+                    ignore_extras,
+                    extras_key,
+                ),
+                detected_format,
+            )
     else:
         raise RowsFromFileError("Bad format")
 
 
 class TypeTracker:
-    def __init__(self):
+    """
-        self.trackers = {}
+    Wrap an iterator of dictionaries and keep track of which SQLite column
+    types are the most likely fit for each of their keys.
 
-    def wrap(self, iterator):
+    Example usage:
+
+    .. code-block:: python
+
+        from sqlite_utils.utils import TypeTracker
+        import sqlite_utils
+
+        db = sqlite_utils.Database(memory=True)
+        tracker = TypeTracker()
+        rows = [{"id": "1", "name": "Cleo", "id": "2", "name": "Cardi"}]
+        db["creatures"].insert_all(tracker.wrap(rows))
+        print(tracker.types)
+        # Outputs {'id': 'integer', 'name': 'text'}
+        db["creatures"].transform(types=tracker.types)
+    """
+
+    def __init__(self) -> None:
+        self.trackers: Dict[str, "ValueTracker"] = {}
+
+    def wrap(self, iterator: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
+        """
+        Use this to loop through an existing iterator, tracking the column types
+        as part of the iteration.
+
+        :param iterator: The iterator to wrap
+        """
         for row in iterator:
             for key, value in row.items():
                 tracker = self.trackers.setdefault(key, ValueTracker())
@@ -198,41 +441,47 @@ class TypeTracker:
             yield row
 
     @property
-    def types(self):
+    def types(self) -> Dict[str, str]:
+        """
+        A dictionary mapping column names to their detected types. This can be passed
+        to the ``db[table_name].transform(types=tracker.types)`` method.
+        """
         return {key: tracker.guessed_type for key, tracker in self.trackers.items()}
 
 
 class ValueTracker:
-    def __init__(self):
+    couldbe: Dict[str, Callable[[object], bool]]
+
+    def __init__(self) -> None:
         self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()}
 
     @classmethod
-    def get_tests(cls):
+    def get_tests(cls) -> List[str]:
         return [
             key.split("test_")[-1]
             for key in cls.__dict__.keys()
             if key.startswith("test_")
         ]
 
-    def test_integer(self, value):
+    def test_integer(self, value: object) -> bool:
         try:
-            int(value)
+            int(cast(Any, value))
             return True
         except (ValueError, TypeError):
             return False
 
-    def test_float(self, value):
+    def test_float(self, value: object) -> bool:
         try:
-            float(value)
+            float(cast(Any, value))
             return True
         except (ValueError, TypeError):
             return False
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return self.guessed_type + ": possibilities = " + repr(self.couldbe)
 
     @property
-    def guessed_type(self):
+    def guessed_type(self) -> str:
         options = set(self.couldbe.keys())
         # Return based on precedence
         for key in self.get_tests():
@@ -240,12 +489,174 @@ class ValueTracker:
                 return key
         return "text"
 
-    def evaluate(self, value):
+    def evaluate(self, value: object) -> None:
         if not value or not self.couldbe:
             return
-        not_these = []
+        not_these: List[str] = []
         for name, test in self.couldbe.items():
             if not test(value):
                 not_these.append(name)
         for key in not_these:
             del self.couldbe[key]
+
+
+class NullProgressBar:
+    def __init__(self, *args: Iterable[T]) -> None:
+        self.args = args
+
+    def __iter__(self) -> Iterator[T]:
+        yield from self.args[0]  # type: ignore
+
+    def update(self, value: int) -> None:
+        pass
+
+
+@contextlib.contextmanager
+def progressbar(*args: Iterable[T], **kwargs: Any) -> Generator[Any, None, None]:
+    silent = kwargs.pop("silent")
+    if silent:
+        yield NullProgressBar(*args)
+    else:
+        with click.progressbar(*args, **kwargs) as bar:  # type: ignore
+            yield bar
+
+
+def _compile_code(
+    code: str, imports: Iterable[str], variable: str = "value"
+) -> Callable[..., Any]:
+    globals_dict: Dict[str, Any] = {"r": recipes, "recipes": recipes}
+    # Handle imports first so they're available for all approaches
+    for import_ in imports:
+        globals_dict[import_.split(".")[0]] = __import__(import_)
+
+    # If user defined a convert() function, return that
+    try:
+        exec(code, globals_dict)
+        return cast(Callable[..., object], globals_dict["convert"])
+    except (AttributeError, SyntaxError, NameError, KeyError, TypeError):
+        pass
+
+    # Check if code is a direct callable reference
+    # e.g. "r.parsedate" instead of "r.parsedate(value)"
+    try:
+        fn = eval(code, globals_dict)
+        if callable(fn):
+            return cast(Callable[..., object], fn)
+    except Exception:
+        pass
+
+    # Try compiling their code as a function instead
+    body_variants = [code]
+    # If single line and no 'return', try adding the return
+    if "\n" not in code and not code.strip().startswith("return "):
+        body_variants.insert(0, "return {}".format(code))
+
+    code_o = None
+    for variant in body_variants:
+        new_code = ["def fn({}):".format(variable)]
+        for line in variant.split("\n"):
+            new_code.append("    {}".format(line))
+        try:
+            code_o = compile("\n".join(new_code), "<string>", "exec")
+            break
+        except SyntaxError:
+            # Try another variant, e.g. for 'return row["column"] = 1'
+            continue
+
+    if code_o is None:
+        raise SyntaxError("Could not compile code")
+
+    exec(code_o, globals_dict)
+    return cast(Callable[..., object], globals_dict["fn"])
+
+
+def chunks(sequence: Iterable[T], size: int) -> Iterable[Iterable[T]]:
+    """
+    Iterate over chunks of the sequence of the given size.
+
+    :param sequence: Any Python iterator
+    :param size: The size of each chunk
+    """
+    iterator = iter(sequence)
+    for item in iterator:
+        yield itertools.chain([item], itertools.islice(iterator, size - 1))
+
+
+def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> str:
+    """
+    ``record`` should be a Python dictionary. Returns a sha1 hash of the
+    keys and values in that record.
+
+    If ``keys=`` is provided, uses just those keys to generate the hash.
+
+    Example usage::
+
+        from sqlite_utils.utils import hash_record
+
+        hashed = hash_record({"name": "Cleo", "twitter": "CleoPaws"})
+        # Or with the keys= option:
+        hashed = hash_record(
+            {"name": "Cleo", "twitter": "CleoPaws", "age": 7},
+            keys=("name", "twitter")
+        )
+
+    :param record: Record to generate a hash for
+    :param keys: Subset of keys to use for that hash
+    """
+    to_hash: Dict[str, Any] = record
+    if keys is not None:
+        to_hash = {key: record[key] for key in keys}
+    return hashlib.sha1(
+        json.dumps(to_hash, separators=(",", ":"), sort_keys=True, default=repr).encode(
+            "utf8"
+        )
+    ).hexdigest()
+
+
+def dedupe_keys(keys: Iterable[str]) -> List[str]:
+    """
+    Rename duplicates in a list of column names so every name is unique,
+    by appending ``_2``, ``_3``... to later occurrences - skipping any
+    suffix that would collide with another column in the list.
+
+    Used when converting SQL query rows to dictionaries, where duplicate
+    column names would otherwise silently overwrite each other.
+
+    :param keys: List of column names, possibly containing duplicates
+    """
+    keys = list(keys)
+    taken = set(keys)
+    if len(taken) == len(keys):
+        # No duplicates - the common case
+        return keys
+    seen: set = set()
+    result = []
+    for key in keys:
+        if key in seen:
+            new_key = key
+            suffix = 2
+            while new_key in seen or new_key in taken:
+                new_key = "{}_{}".format(key, suffix)
+                suffix += 1
+            key = new_key
+        seen.add(key)
+        result.append(key)
+    return result
+
+
+def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]:
+    for key, value in d.items():
+        if isinstance(value, dict):
+            for key2, value2 in _flatten(value):
+                yield key + "_" + key2, value2
+        else:
+            yield key, value
+
+
+def flatten(row: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Turn a nested dict e.g. ``{"a": {"b": 1}}`` into a flat dict: ``{"a_b": 1}``
+
+    :param row: A Python dictionary, optionally with nested dictionaries
+    """
+    return dict(_flatten(row))

tests/conftest.py

@@ -1,6 +1,63 @@
 from sqlite_utils import Database
+from sqlite_utils.utils import sqlite3
 import pytest
 
+CREATE_TABLES = """
+create table Gosh (c1 text, c2 text, c3 text);
+create table Gosh2 (c1 text, c2 text, c3 text);
+"""
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--sqlite-autocommit",
+        action="store_true",
+        default=False,
+        help=(
+            "Run every test against connections created with the Python 3.12+ "
+            "sqlite3.connect(autocommit=True) mode"
+        ),
+    )
+
+
+def pytest_configure(config):
+    import sys
+
+    sys._called_from_test = True  # type: ignore[attr-defined]
+
+    if config.getoption("--sqlite-autocommit"):
+        if sys.version_info < (3, 12):
+            raise pytest.UsageError(
+                "--sqlite-autocommit requires Python 3.12 or higher"
+            )
+        real_connect = sqlite3.connect
+
+        def autocommit_connect(*args, **kwargs):
+            kwargs.setdefault("autocommit", True)
+            return real_connect(*args, **kwargs)
+
+        sqlite3.connect = autocommit_connect
+
+
+@pytest.fixture(autouse=True)
+def close_all_databases():
+    """Automatically close all Database objects created during a test."""
+    databases = []
+    original_init = Database.__init__
+
+    def tracking_init(self, *args, **kwargs):
+        original_init(self, *args, **kwargs)
+        databases.append(self)
+
+    Database.__init__ = tracking_init  # type: ignore[method-assign]
+    yield
+    Database.__init__ = original_init  # type: ignore[method-assign]
+    for db in databases:
+        try:
+            db.close()
+        except Exception:
+            pass
+
 
 @pytest.fixture
 def fresh_db():
@@ -10,12 +67,19 @@ def fresh_db():
 @pytest.fixture
 def existing_db():
     database = Database(memory=True)
-    database.executescript(
+    database.executescript("""
-        """
         CREATE TABLE foo (text TEXT);
         INSERT INTO foo (text) values ("one");
         INSERT INTO foo (text) values ("two");
         INSERT INTO foo (text) values ("three");
-    """
+    """)
-    )
     return database
+
+
+@pytest.fixture
+def db_path(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = sqlite3.connect(path)
+    db.executescript(CREATE_TABLES)
+    db.close()
+    return path

tests/ext.c

@@ -0,0 +1,48 @@
+/*
+** This file implements a SQLite extension with multiple entrypoints.
+**
+** The default entrypoint, sqlite3_ext_init, has a single function "a".
+** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b".
+** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c".
+**
+** Compiling instructions:
+**     https://www.sqlite.org/loadext.html#compiling_a_loadable_extension
+**
+*/
+
+#include "sqlite3ext.h"
+
+SQLITE_EXTENSION_INIT1
+
+// SQL function that returns back the value supplied during sqlite3_create_function()
+static void func(sqlite3_context *context, int argc, sqlite3_value **argv) {
+  sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC);
+}
+
+
+// The default entrypoint, since it matches the "ext.dylib"/"ext.so" name
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0);
+}
+
+// Alternate entrypoint #1
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0);
+}
+
+// Alternate entrypoint #2
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0);
+}

tests/test_analyze.py

@@ -0,0 +1,51 @@
+import pytest
+
+
+@pytest.fixture
+def db(fresh_db):
+    fresh_db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db["one_index"].create_index(["name"])
+    fresh_db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id")
+    fresh_db["two_indexes"].create_index(["name"])
+    fresh_db["two_indexes"].create_index(["species"])
+    return fresh_db
+
+
+def test_analyze_whole_database(db):
+    assert set(db.table_names()) == {"one_index", "two_indexes"}
+    db.analyze()
+    assert set(db.table_names()).issuperset(
+        {"one_index", "two_indexes", "sqlite_stat1"}
+    )
+    assert list(db["sqlite_stat1"].rows) == [
+        {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"},
+        {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"},
+        {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"},
+    ]
+
+
+@pytest.mark.parametrize("method", ("db_method_with_name", "table_method"))
+def test_analyze_one_table(db, method):
+    assert set(db.table_names()).issuperset({"one_index", "two_indexes"})
+    if method == "db_method_with_name":
+        db.analyze("one_index")
+    elif method == "table_method":
+        db["one_index"].analyze()
+
+    assert set(db.table_names()).issuperset(
+        {"one_index", "two_indexes", "sqlite_stat1"}
+    )
+    assert list(db["sqlite_stat1"].rows) == [
+        {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}
+    ]
+
+
+def test_analyze_index_by_name(db):
+    assert set(db.table_names()) == {"one_index", "two_indexes"}
+    db.analyze("idx_two_indexes_species")
+    assert set(db.table_names()).issuperset(
+        {"one_index", "two_indexes", "sqlite_stat1"}
+    )
+    assert list(db["sqlite_stat1"].rows) == [
+        {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"},
+    ]

tests/test_analyze_tables.py

@@ -1,10 +1,8 @@
-from sqlite_utils.db import Database, ForeignKey, ColumnDetails
+from sqlite_utils.db import Database, ColumnDetails
 from sqlite_utils import cli
-from sqlite_utils.utils import OperationalError
 from click.testing import CliRunner
 import pytest
 import sqlite3
-import textwrap
 
 
 @pytest.fixture
@@ -26,11 +24,35 @@ def db_to_analyze(fresh_db):
     return fresh_db
 
 
+@pytest.fixture
+def big_db_to_analyze_path(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    categories = {
+        "A": 40,
+        "B": 30,
+        "C": 20,
+        "D": 10,
+    }
+    to_insert = []
+    for category, count in categories.items():
+        for _ in range(count):
+            to_insert.append(
+                {
+                    "category": category,
+                    "all_null": None,
+                }
+            )
+    db["stuff"].insert_all(to_insert)
+    return path
+
+
 @pytest.mark.parametrize(
-    "column,expected",
+    "column,extra_kwargs,expected",
     [
         (
             "id",
+            {},
             ColumnDetails(
                 table="stuff",
                 column="id",
@@ -44,6 +66,7 @@ def db_to_analyze(fresh_db):
         ),
         (
             "owner",
+            {},
             ColumnDetails(
                 table="stuff",
                 column="owner",
@@ -57,6 +80,7 @@ def db_to_analyze(fresh_db):
         ),
         (
             "size",
+            {},
             ColumnDetails(
                 table="stuff",
                 column="size",
@@ -68,11 +92,41 @@ def db_to_analyze(fresh_db):
                 least_common=None,
             ),
         ),
+        (
+            "owner",
+            {"most_common": False},
+            ColumnDetails(
+                table="stuff",
+                column="owner",
+                total_rows=8,
+                num_null=0,
+                num_blank=0,
+                num_distinct=4,
+                most_common=None,
+                least_common=[("Anne", 1), ("Terry...", 2)],
+            ),
+        ),
+        (
+            "owner",
+            {"least_common": False},
+            ColumnDetails(
+                table="stuff",
+                column="owner",
+                total_rows=8,
+                num_null=0,
+                num_blank=0,
+                num_distinct=4,
+                most_common=[("Joan", 3), ("Kumar", 2)],
+                least_common=None,
+            ),
+        ),
     ],
 )
-def test_analyze_column(db_to_analyze, column, expected):
+def test_analyze_column(db_to_analyze, column, extra_kwargs, expected):
     assert (
-        db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5)
+        db_to_analyze["stuff"].analyze_column(
+            column, common_limit=2, value_truncate=5, **extra_kwargs
+        )
         == expected
     )
 
@@ -81,16 +135,15 @@ def test_analyze_column(db_to_analyze, column, expected):
 def db_to_analyze_path(db_to_analyze, tmpdir):
     path = str(tmpdir / "test.db")
     db = sqlite3.connect(path)
-    db.executescript("\n".join(db_to_analyze.conn.iterdump()))
+    sql = "\n".join(db_to_analyze.iterdump())
+    db.executescript(sql)
+    db.close()
     return path
 
 
 def test_analyze_table(db_to_analyze_path):
     result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path])
-    assert (
+    assert result.output.strip() == ("""
-        result.output.strip()
-        == (
-            """
 stuff.id: (1/3)
 
   Total rows: 8
@@ -123,15 +176,14 @@ stuff.size: (3/3)
 
   Most common:
     5: 5
-    3: 4"""
+    3: 4""").strip()
-        ).strip()
-    )
 
 
 def test_analyze_table_save(db_to_analyze_path):
     result = CliRunner().invoke(
         cli.cli, ["analyze-tables", db_to_analyze_path, "--save"]
     )
+    assert result.exit_code == 0
     rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows)
     assert rows == [
         {
@@ -165,3 +217,101 @@ def test_analyze_table_save(db_to_analyze_path):
             "least_common": None,
         },
     ]
+
+
+@pytest.mark.parametrize(
+    "no_most,no_least",
+    (
+        (False, False),
+        (True, False),
+        (False, True),
+        (True, True),
+    ),
+)
+def test_analyze_table_save_no_most_no_least_options(
+    no_most, no_least, big_db_to_analyze_path
+):
+    args = [
+        "analyze-tables",
+        big_db_to_analyze_path,
+        "--save",
+        "--common-limit",
+        "2",
+        "--column",
+        "category",
+    ]
+    if no_most:
+        args.append("--no-most")
+    if no_least:
+        args.append("--no-least")
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 0
+    rows = list(Database(big_db_to_analyze_path)["_analyze_tables_"].rows)
+    expected = {
+        "table": "stuff",
+        "column": "category",
+        "total_rows": 100,
+        "num_null": 0,
+        "num_blank": 0,
+        "num_distinct": 4,
+        "most_common": None,
+        "least_common": None,
+    }
+    if not no_most:
+        expected["most_common"] = '[["A", 40], ["B", 30]]'
+    if not no_least:
+        expected["least_common"] = '[["D", 10], ["C", 20]]'
+
+    assert rows == [expected]
+
+
+def test_analyze_table_column_all_nulls(big_db_to_analyze_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["analyze-tables", big_db_to_analyze_path, "stuff", "--column", "all_null"],
+    )
+    assert result.exit_code == 0
+    assert result.output == (
+        "stuff.all_null: (1/1)\n\n  Total rows: 100\n"
+        "  Null rows: 100\n"
+        "  Blank rows: 0\n"
+        "\n"
+        "  Distinct values: 0\n\n"
+    )
+
+
+@pytest.mark.parametrize(
+    "args,expected_error",
+    (
+        (["-c", "bad_column"], "These columns were not found: bad_column\n"),
+        (["one", "-c", "age"], "These columns were not found: age\n"),
+        (["two", "-c", "age"], None),
+        (
+            ["one", "-c", "age", "--column", "bad"],
+            "These columns were not found: age, bad\n",
+        ),
+    ),
+)
+def test_analyze_table_validate_columns(tmpdir, args, expected_error):
+    path = str(tmpdir / "test_validate_columns.db")
+    db = Database(path)
+    db["one"].insert(
+        {
+            "id": 1,
+            "name": "one",
+        }
+    )
+    db["two"].insert(
+        {
+            "id": 1,
+            "age": 5,
+        }
+    )
+    result = CliRunner().invoke(
+        cli.cli,
+        ["analyze-tables", path] + args,
+        catch_exceptions=False,
+    )
+    assert result.exit_code == (1 if expected_error else 0)
+    if expected_error:
+        assert expected_error in result.output

tests/test_atomic.py

@@ -0,0 +1,382 @@
+import pytest
+
+from sqlite_utils.db import Database, _iter_complete_sql_statements
+from sqlite_utils.utils import sqlite3
+
+
+@pytest.mark.parametrize(
+    "sql,expected",
+    (
+        (
+            "CREATE TABLE t(id); INSERT INTO t VALUES (1)",
+            ["CREATE TABLE t(id);", "INSERT INTO t VALUES (1)"],
+        ),
+        (
+            "INSERT INTO t VALUES ('a;b');",
+            ["INSERT INTO t VALUES ('a;b');"],
+        ),
+        (
+            "-- comment;\nCREATE TABLE t(id);",
+            ["-- comment;\nCREATE TABLE t(id);"],
+        ),
+        (
+            """
+            CREATE TRIGGER t_ai AFTER INSERT ON t
+            BEGIN
+                UPDATE t SET value = 'a;b' WHERE id = new.id;
+                INSERT INTO log VALUES ('x;y');
+            END;
+            """,
+            [
+                "CREATE TRIGGER t_ai AFTER INSERT ON t\n"
+                "            BEGIN\n"
+                "                UPDATE t SET value = 'a;b' WHERE id = new.id;\n"
+                "                INSERT INTO log VALUES ('x;y');\n"
+                "            END;"
+            ],
+        ),
+    ),
+)
+def test_iter_complete_sql_statements(sql, expected):
+    assert list(_iter_complete_sql_statements(sql)) == expected
+
+
+def test_atomic_commits(fresh_db):
+    with fresh_db.atomic():
+        fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+
+    assert list(fresh_db["dogs"].rows) == [{"id": 1, "name": "Cleo"}]
+
+
+def test_atomic_rolls_back(fresh_db):
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+            raise RuntimeError("boom")
+
+    assert not fresh_db["dogs"].exists()
+
+
+def test_nested_atomic_rolls_back_to_savepoint(fresh_db):
+    fresh_db["dogs"].create({"id": int, "name": str}, pk="id")
+
+    with fresh_db.atomic():
+        fresh_db["dogs"].insert({"id": 1, "name": "Cleo"})
+        with pytest.raises(RuntimeError):
+            with fresh_db.atomic():
+                fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"})
+                raise RuntimeError("boom")
+        fresh_db["dogs"].insert({"id": 3, "name": "Marnie"})
+
+    assert list(fresh_db["dogs"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 3, "name": "Marnie"},
+    ]
+
+
+def test_outer_atomic_rolls_back_released_savepoint(fresh_db):
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+            with fresh_db.atomic():
+                fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"})
+            raise RuntimeError("boom")
+
+    assert not fresh_db["dogs"].exists()
+
+
+def test_executescript_does_not_commit_open_atomic_block(fresh_db):
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db.executescript("""
+                CREATE TABLE dogs(id INTEGER PRIMARY KEY, name TEXT);
+                CREATE TRIGGER dogs_ai AFTER INSERT ON dogs
+                BEGIN
+                    UPDATE dogs SET name = upper(new.name) || '; updated' WHERE id = new.id;
+                END;
+                -- This comment has a semicolon;
+                INSERT INTO dogs VALUES (1, 'Cleo; the first');
+            """)
+            raise RuntimeError("boom")
+
+    assert not fresh_db["dogs"].exists()
+
+
+def test_transform_does_not_commit_open_atomic_block(fresh_db):
+    fresh_db["dogs"].insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id")
+
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"})
+            fresh_db["dogs"].transform(rename={"age": "dog_age"})
+            raise RuntimeError("boom")
+
+    assert (
+        fresh_db["dogs"].schema
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n)'
+    )
+    assert list(fresh_db["dogs"].rows) == [
+        {"id": 1, "name": "Cleo", "age": "5"},
+    ]
+
+
+def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db):
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id")
+    fresh_db["books"].insert(
+        {"id": 1, "title": "Book", "author_id": 1},
+        pk="id",
+        foreign_keys={"author_id"},
+    )
+
+    with fresh_db.atomic():
+        fresh_db["authors"].transform(rename={"name": "full_name"})
+        assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+    assert (
+        fresh_db["authors"].schema
+        == 'CREATE TABLE "authors" (\n   "id" INTEGER PRIMARY KEY,\n   "full_name" TEXT\n)'
+    )
+    assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == []
+
+
+def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db):
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id")
+    fresh_db["books"].insert(
+        {"id": 1, "title": "Book", "author_id": 1},
+        pk="id",
+        foreign_keys={"author_id"},
+    )
+
+    with pytest.raises(RuntimeError):
+        with fresh_db.atomic():
+            fresh_db["authors"].transform(rename={"name": "full_name"})
+            raise RuntimeError("boom")
+
+    assert (
+        fresh_db["authors"].schema
+        == 'CREATE TABLE "authors" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n)'
+    )
+    assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+    assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == []
+
+
+def test_transform_detects_foreign_key_check_violations(fresh_db):
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 2}, pk="id")
+
+    with pytest.raises(sqlite3.IntegrityError):
+        fresh_db["books"].transform(add_foreign_keys=(("author_id", "authors", "id"),))
+
+    assert fresh_db["books"].foreign_keys == []
+    assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db):
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.execute("begin")
+    with fresh_db.atomic():
+        fresh_db["t"].insert({"id": 2}, pk="id")
+    # Nothing is committed until the user's own transaction commits
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1]
+    # And with a commit instead, the atomic block's writes persist
+    fresh_db.execute("begin")
+    with fresh_db.atomic():
+        fresh_db["t"].insert({"id": 3}, pk="id")
+    fresh_db.commit()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1, 3]
+
+
+def test_begin_commit_rollback(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["t"].insert({"id": 1}, pk="id")
+    db.begin()
+    db["t"].insert({"id": 2}, pk="id")
+    assert db.conn.in_transaction
+    db.rollback()
+    assert not db.conn.in_transaction
+    assert [r["id"] for r in db["t"].rows] == [1]
+    db.begin()
+    db["t"].insert({"id": 3}, pk="id")
+    db.commit()
+    db.close()
+    db2 = Database(path)
+    assert [r["id"] for r in db2["t"].rows] == [1, 3]
+    db2.close()
+
+
+def test_begin_inside_transaction_errors(fresh_db):
+    fresh_db.begin()
+    with pytest.raises(sqlite3.OperationalError):
+        fresh_db.begin()
+    fresh_db.rollback()
+
+
+def test_commit_and_rollback_without_transaction_are_noops(fresh_db):
+    fresh_db.commit()
+    fresh_db.rollback()
+    assert not fresh_db.conn.in_transaction
+
+
+def test_execute_write_commits_immediately(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["t"].insert({"id": 1}, pk="id")
+    db.execute("insert into t (id) values (2)")
+    # No implicit transaction is left open
+    assert not db.conn.in_transaction
+    # A completely separate connection sees the row straight away
+    other = sqlite3.connect(path)
+    assert other.execute("select count(*) from t").fetchone()[0] == 2
+    other.close()
+    db.close()
+
+
+def test_execute_write_respects_explicit_transaction(fresh_db):
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.begin()
+    fresh_db.execute("insert into t (id) values (2)")
+    # Still inside the explicit transaction - not committed
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1]
+
+
+def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db):
+    # A BEGIN hidden behind a leading comment must not be auto-committed
+    # out from under the caller
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.execute("-- start a transaction\nbegin")
+    assert fresh_db.conn.in_transaction
+    fresh_db.execute("insert into t (id) values (2)")
+    fresh_db.rollback()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1]
+
+
+def _sqlite_accepts_bom():
+    try:
+        sqlite3.connect(":memory:").execute("\ufeffselect 1")
+        return True
+    except sqlite3.OperationalError:
+        return False
+
+
+@pytest.mark.parametrize("begin_sql", ["; begin", "\ufeffbegin"])
+def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql):
+    # sqlite3 tolerates empty statements and a UTF-8 BOM before the first
+    # real token, so a BEGIN behind either must not be auto-committed
+    # out from under the caller
+    if begin_sql.startswith("\ufeff") and not _sqlite_accepts_bom():
+        pytest.skip("This SQLite version rejects a leading byte order mark")
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.execute(begin_sql)
+    assert fresh_db.conn.in_transaction
+    fresh_db.execute("insert into t (id) values (2)")
+    fresh_db.rollback()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1]
+
+
+def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir):
+    # A failed write must not leave the driver's implicit transaction open -
+    # that would silently disable auto-commit for every subsequent write
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["t"].insert({"id": 1}, pk="id")
+    with pytest.raises(sqlite3.IntegrityError):
+        db.execute("insert into t (id) values (1)")
+    assert not db.conn.in_transaction
+    # Subsequent writes commit as normal and survive closing the connection
+    db["other"].insert({"id": 2})
+    db.close()
+    db2 = Database(path)
+    assert db2["other"].exists()
+    db2.close()
+
+
+def test_execute_failed_write_preserves_explicit_transaction(fresh_db):
+    # A failed write inside an explicit transaction must not roll back
+    # the caller's earlier work - only the caller decides that
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    fresh_db.begin()
+    fresh_db.execute("insert into t (id) values (2)")
+    with pytest.raises(sqlite3.IntegrityError):
+        fresh_db.execute("insert into t (id) values (1)")
+    assert fresh_db.conn.in_transaction
+    fresh_db.commit()
+    assert [r["id"] for r in fresh_db["t"].rows] == [1, 2]
+
+
+def test_execute_failed_write_inside_atomic_preserves_block(fresh_db):
+    # A caught failure inside an atomic() block must leave the block's
+    # transaction open so its other work still commits
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    with fresh_db.atomic():
+        fresh_db.execute("insert into t (id) values (2)")
+        with pytest.raises(sqlite3.IntegrityError):
+            fresh_db.execute("insert into t (id) values (1)")
+    assert [r["id"] for r in fresh_db["t"].rows] == [1, 2]
+
+
+def test_query_returning_commits_after_iteration(tmpdir):
+    if sqlite3.sqlite_version_info < (3, 35, 0):
+        import pytest as _pytest
+
+        _pytest.skip("RETURNING requires SQLite 3.35.0 or higher")
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["t"].insert({"id": 1}, pk="id")
+    rows = list(db.query("insert into t (id) values (2) returning id"))
+    assert rows == [{"id": 2}]
+    assert not db.conn.in_transaction
+    other = sqlite3.connect(path)
+    assert other.execute("select count(*) from t").fetchone()[0] == 2
+    other.close()
+    db.close()
+
+
+TRIGGER_SQL = """
+create trigger no_bad before insert on t
+when new.v = 'bad'
+begin
+    select raise(rollback, 'trigger says no');
+end
+"""
+
+
+def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db):
+    # RAISE(ROLLBACK) rolls back the whole transaction and destroys every
+    # savepoint - atomic()'s cleanup must not mask the IntegrityError
+    # with "cannot rollback - no transaction is active"
+    fresh_db.execute("create table t (id integer primary key, v text)")
+    fresh_db.execute(TRIGGER_SQL)
+    with pytest.raises(sqlite3.IntegrityError, match="trigger says no"):
+        with fresh_db.atomic():
+            fresh_db.execute("insert into t (v) values ('bad')")
+    assert not fresh_db.conn.in_transaction
+
+
+def test_nested_atomic_preserves_error_from_transaction_destroying_trigger(
+    fresh_db,
+):
+    # The nested savepoint branch previously raised
+    # "no such savepoint" from ROLLBACK TO SAVEPOINT
+    fresh_db.execute("create table t (id integer primary key, v text)")
+    fresh_db.execute(TRIGGER_SQL)
+    with pytest.raises(sqlite3.IntegrityError, match="trigger says no"):
+        with fresh_db.atomic():
+            with fresh_db.atomic():
+                fresh_db.execute("insert into t (v) values ('bad')")
+    assert not fresh_db.conn.in_transaction
+
+
+def test_atomic_preserves_error_from_insert_or_rollback(fresh_db):
+    fresh_db["t"].insert({"id": 1}, pk="id")
+    with pytest.raises(sqlite3.IntegrityError):
+        with fresh_db.atomic():
+            fresh_db.execute("insert or rollback into t (id) values (1)")
+    assert not fresh_db.conn.in_transaction

tests/test_cli_bulk.py

@@ -0,0 +1,123 @@
+from click.testing import CliRunner
+from sqlite_utils import cli, Database
+import pathlib
+import pytest
+import subprocess
+import sys
+import time
+
+
+@pytest.fixture
+def test_db_and_path(tmpdir):
+    db_path = str(pathlib.Path(tmpdir) / "data.db")
+    db = Database(db_path)
+    db["example"].insert_all(
+        [
+            {"id": 1, "name": "One"},
+            {"id": 2, "name": "Two"},
+        ],
+        pk="id",
+    )
+    return db, db_path
+
+
+def test_cli_bulk(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "bulk",
+            db_path,
+            "insert into example (id, name) values (:id, myupper(:name))",
+            "-",
+            "--nl",
+            "--functions",
+            "myupper = lambda s: s.upper()",
+        ],
+        input='{"id": 3, "name": "Three"}\n{"id": 4, "name": "Four"}\n',
+    )
+    assert result.exit_code == 0, result.output
+    assert [
+        {"id": 1, "name": "One"},
+        {"id": 2, "name": "Two"},
+        {"id": 3, "name": "THREE"},
+        {"id": 4, "name": "FOUR"},
+    ] == list(db["example"].rows)
+
+
+def test_cli_bulk_multiple_functions(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "bulk",
+            db_path,
+            "insert into example (id, name) values (:id, myupper(mylower(:name)))",
+            "-",
+            "--nl",
+            "--functions",
+            "myupper = lambda s: s.upper()",
+            "--functions",
+            "mylower = lambda s: s.lower()",
+        ],
+        input='{"id": 3, "name": "ThReE"}\n{"id": 4, "name": "FoUr"}\n',
+    )
+    assert result.exit_code == 0, result.output
+    assert [
+        {"id": 1, "name": "One"},
+        {"id": 2, "name": "Two"},
+        {"id": 3, "name": "THREE"},
+        {"id": 4, "name": "FOUR"},
+    ] == list(db["example"].rows)
+
+
+def test_cli_bulk_batch_size(test_db_and_path):
+    db, db_path = test_db_and_path
+    proc = subprocess.Popen(
+        [
+            sys.executable,
+            "-m",
+            "sqlite_utils",
+            "bulk",
+            db_path,
+            "insert into example (id, name) values (:id, :name)",
+            "-",
+            "--nl",
+            "--batch-size",
+            "2",
+        ],
+        stdin=subprocess.PIPE,
+        stdout=sys.stdout,
+    )
+    # Writing one record should not commit
+    proc.stdin.write(b'{"id": 3, "name": "Three"}\n\n')
+    proc.stdin.flush()
+    time.sleep(1)
+    assert db["example"].count == 2
+
+    # Writing another should trigger a commit:
+    proc.stdin.write(b'{"id": 4, "name": "Four"}\n\n')
+    proc.stdin.flush()
+    time.sleep(1)
+    assert db["example"].count == 4
+
+    proc.stdin.close()
+    proc.wait()
+    assert proc.returncode == 0
+
+
+def test_cli_bulk_error(test_db_and_path):
+    _, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "bulk",
+            db_path,
+            "insert into example (id, name) value (:id, :name)",
+            "-",
+            "--nl",
+        ],
+        input='{"id": 3, "name": "Three"}',
+    )
+    assert result.exit_code == 1
+    assert result.output == 'Error: near "value": syntax error\n'

tests/test_cli_convert.py

@@ -0,0 +1,712 @@
+from click.testing import CliRunner
+from sqlite_utils import cli
+import sqlite_utils
+import json
+import textwrap
+import pathlib
+import pytest
+
+
+@pytest.fixture
+def test_db_and_path(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["example"].insert_all(
+        [
+            {"id": 1, "dt": "5th October 2019 12:04"},
+            {"id": 2, "dt": "6th October 2019 00:05:06"},
+            {"id": 3, "dt": ""},
+            {"id": 4, "dt": None},
+        ],
+        pk="id",
+    )
+    return db, db_path
+
+
+@pytest.fixture
+def fresh_db_and_path(tmpdir):
+    db_path = str(pathlib.Path(tmpdir) / "data.db")
+    db = sqlite_utils.Database(db_path)
+    return db, db_path
+
+
+@pytest.mark.parametrize(
+    "code",
+    [
+        "return value.replace('October', 'Spooktober')",
+        # Return is optional:
+        "value.replace('October', 'Spooktober')",
+        # Multiple lines are supported:
+        "v = value.replace('October', 'Spooktober')\nreturn v",
+        # Can also define a convert() function
+        "def convert(value): return value.replace('October', 'Spooktober')",
+        # ... with imports
+        "import re\n\ndef convert(value): return value.replace('October', 'Spooktober')",
+    ],
+)
+def test_convert_code(fresh_db_and_path, code):
+    db, db_path = fresh_db_and_path
+    db["t"].insert({"text": "October"})
+    result = CliRunner().invoke(
+        cli.cli, ["convert", db_path, "t", "text", code], catch_exceptions=False
+    )
+    assert result.exit_code == 0, result.output
+    value = list(db["t"].rows)[0]["text"]
+    assert value == "Spooktober"
+
+
+@pytest.mark.parametrize(
+    "bad_code",
+    (
+        "def foo(value)",
+        "$",
+    ),
+)
+def test_convert_code_errors(fresh_db_and_path, bad_code):
+    db, db_path = fresh_db_and_path
+    db["t"].insert({"text": "October"})
+    result = CliRunner().invoke(
+        cli.cli, ["convert", db_path, "t", "text", bad_code], catch_exceptions=False
+    )
+    assert result.exit_code == 1
+    assert result.output == "Error: Could not compile code\n"
+
+
+def test_convert_import(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "return re.sub('O..', 'OXX', value) if value else value",
+            "--import",
+            "re",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert [
+        {"id": 1, "dt": "5th OXXober 2019 12:04"},
+        {"id": 2, "dt": "6th OXXober 2019 00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ] == list(db["example"].rows)
+
+
+def test_convert_import_nested(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["example"].insert({"xml": '<item name="Cleo" />'})
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "xml",
+            'xml.etree.ElementTree.fromstring(value).attrib["name"]',
+            "--import",
+            "xml.etree.ElementTree",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert [
+        {"xml": "Cleo"},
+    ] == list(db["example"].rows)
+
+
+def test_convert_dryrun(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "return re.sub('O..', 'OXX', value)",
+            "--import",
+            "re",
+            "--dry-run",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == (
+        "5th October 2019 12:04\n"
+        " --- becomes:\n"
+        "5th OXXober 2019 12:04\n"
+        "\n"
+        "6th October 2019 00:05:06\n"
+        " --- becomes:\n"
+        "6th OXXober 2019 00:05:06\n"
+        "\n"
+        "\n"
+        " --- becomes:\n"
+        "\n"
+        "\n"
+        "None\n"
+        " --- becomes:\n"
+        "None\n\n"
+        "Would affect 4 rows"
+    )
+    # But it should not have actually modified the table data
+    assert list(db["example"].rows) == [
+        {"id": 1, "dt": "5th October 2019 12:04"},
+        {"id": 2, "dt": "6th October 2019 00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ]
+    # Test with a where clause too
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "return re.sub('O..', 'OXX', value)",
+            "--import",
+            "re",
+            "--dry-run",
+            "--where",
+            "id = :id",
+            "-p",
+            "id",
+            "4",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip().split("\n")[-1] == "Would affect 1 row"
+
+
+def test_convert_multi_dryrun(test_db_and_path):
+    db_path = test_db_and_path[1]
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "{'foo': 'bar', 'baz': 1}",
+            "--dry-run",
+            "--multi",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == (
+        "5th October 2019 12:04\n"
+        " --- becomes:\n"
+        '{"foo": "bar", "baz": 1}\n'
+        "\n"
+        "6th October 2019 00:05:06\n"
+        " --- becomes:\n"
+        '{"foo": "bar", "baz": 1}\n'
+        "\n"
+        "\n"
+        " --- becomes:\n"
+        "\n"
+        "\n"
+        "None\n"
+        " --- becomes:\n"
+        "None\n"
+        "\n"
+        "Would affect 4 rows"
+    )
+
+
+def test_convert_multi_dryrun_unicode_not_escaped(test_db_and_path):
+    db_path = test_db_and_path[1]
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "{'text': 'Japanese 日本語'}",
+            "--dry-run",
+            "--multi",
+        ],
+    )
+    assert result.exit_code == 0
+    # Preview should match what jsonify_if_needed() would actually store
+    assert '{"text": "Japanese 日本語"}' in result.output
+
+
+@pytest.mark.parametrize("drop", (True, False))
+def test_convert_output_column(test_db_and_path, drop):
+    db, db_path = test_db_and_path
+    args = [
+        "convert",
+        db_path,
+        "example",
+        "dt",
+        "value.replace('October', 'Spooktober') if value else value",
+        "--output",
+        "newcol",
+    ]
+    if drop:
+        args += ["--drop"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 0, result.output
+    expected = [
+        {
+            "id": 1,
+            "dt": "5th October 2019 12:04",
+            "newcol": "5th Spooktober 2019 12:04",
+        },
+        {
+            "id": 2,
+            "dt": "6th October 2019 00:05:06",
+            "newcol": "6th Spooktober 2019 00:05:06",
+        },
+        {"id": 3, "dt": "", "newcol": ""},
+        {"id": 4, "dt": None, "newcol": None},
+    ]
+    if drop:
+        for row in expected:
+            del row["dt"]
+    assert list(db["example"].rows) == expected
+
+
+@pytest.mark.parametrize(
+    "output_type,expected",
+    (
+        ("text", [(1, "1"), (2, "2"), (3, "3"), (4, "4")]),
+        ("float", [(1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0)]),
+        ("integer", [(1, 1), (2, 2), (3, 3), (4, 4)]),
+        (None, [(1, "1"), (2, "2"), (3, "3"), (4, "4")]),
+    ),
+)
+def test_convert_output_column_output_type(test_db_and_path, output_type, expected):
+    db, db_path = test_db_and_path
+    args = [
+        "convert",
+        db_path,
+        "example",
+        "id",
+        "value",
+        "--output",
+        "new_id",
+    ]
+    if output_type:
+        args += ["--output-type", output_type]
+    result = CliRunner().invoke(
+        cli.cli,
+        args,
+    )
+    assert result.exit_code == 0, result.output
+    assert expected == list(db.execute("select id, new_id from example"))
+
+
+@pytest.mark.parametrize(
+    "options,expected_error",
+    [
+        (
+            [
+                "dt",
+                "id",
+                "value.replace('October', 'Spooktober')",
+                "--output",
+                "newcol",
+            ],
+            "Cannot use --output with more than one column",
+        ),
+        (
+            [
+                "dt",
+                "value.replace('October', 'Spooktober')",
+                "--output",
+                "newcol",
+                "--output-type",
+                "invalid",
+            ],
+            "Error: Invalid value for '--output-type'",
+        ),
+        (
+            [
+                "value.replace('October', 'Spooktober')",
+            ],
+            "Missing argument 'COLUMNS...'",
+        ),
+    ],
+)
+def test_convert_output_error(test_db_and_path, options, expected_error):
+    db_path = test_db_and_path[1]
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+        ]
+        + options,
+    )
+    assert result.exit_code != 0
+    assert expected_error in result.output
+
+
+@pytest.mark.parametrize("drop", (True, False))
+def test_convert_multi(fresh_db_and_path, drop):
+    db, db_path = fresh_db_and_path
+    db["creatures"].insert_all(
+        [
+            {"id": 1, "name": "Simon"},
+            {"id": 2, "name": "Cleo"},
+        ],
+        pk="id",
+    )
+    args = [
+        "convert",
+        db_path,
+        "creatures",
+        "name",
+        "--multi",
+        '{"upper": value.upper(), "lower": value.lower()}',
+    ]
+    if drop:
+        args += ["--drop"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 0, result.output
+    expected = [
+        {"id": 1, "name": "Simon", "upper": "SIMON", "lower": "simon"},
+        {"id": 2, "name": "Cleo", "upper": "CLEO", "lower": "cleo"},
+    ]
+    if drop:
+        for row in expected:
+            del row["name"]
+    assert list(db["creatures"].rows) == expected
+
+
+def test_convert_multi_complex_column_types(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["rows"].insert_all(
+        [
+            {"id": 1},
+            {"id": 2},
+            {"id": 3},
+            {"id": 4},
+        ],
+        pk="id",
+    )
+    code = textwrap.dedent("""
+    if value == 1:
+        return {"is_str": "", "is_float": 1.2, "is_int": None}
+    elif value == 2:
+        return {"is_float": 1, "is_int": 12}
+    elif value == 3:
+        return {"is_bytes": b"blah"}
+    """)
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "rows",
+            "id",
+            "--multi",
+            code,
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["rows"].rows) == [
+        {"id": 1, "is_str": "", "is_float": 1.2, "is_int": None, "is_bytes": None},
+        {"id": 2, "is_str": None, "is_float": 1.0, "is_int": 12, "is_bytes": None},
+        {
+            "id": 3,
+            "is_str": None,
+            "is_float": None,
+            "is_int": None,
+            "is_bytes": b"blah",
+        },
+        {"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None},
+    ]
+    assert db["rows"].schema == (
+        'CREATE TABLE "rows" (\n'
+        '   "id" INTEGER PRIMARY KEY\n'
+        ', "is_str" TEXT, "is_float" REAL, "is_int" INTEGER, "is_bytes" BLOB)'
+    )
+
+
+@pytest.mark.parametrize("delimiter", [None, ";", "-"])
+def test_recipe_jsonsplit(tmpdir, delimiter):
+    db_path = str(pathlib.Path(tmpdir) / "data.db")
+    db = sqlite_utils.Database(db_path)
+    db["example"].insert_all(
+        [
+            {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])},
+            {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])},
+        ],
+        pk="id",
+    )
+    code = "r.jsonsplit(value)"
+    if delimiter:
+        code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter)
+    args = ["convert", db_path, "example", "tags", code]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 0, result.output
+    assert list(db["example"].rows) == [
+        {"id": 1, "tags": '["foo", "bar"]'},
+        {"id": 2, "tags": '["bar", "baz"]'},
+    ]
+
+
+@pytest.mark.parametrize(
+    "type,expected_array",
+    (
+        (None, ["1", "2", "3"]),
+        ("float", [1.0, 2.0, 3.0]),
+        ("int", [1, 2, 3]),
+    ),
+)
+def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array):
+    db, db_path = fresh_db_and_path
+    db["example"].insert_all(
+        [
+            {"id": 1, "records": "1,2,3"},
+        ],
+        pk="id",
+    )
+    code = "r.jsonsplit(value)"
+    if type:
+        code = "recipes.jsonsplit(value, type={})".format(type)
+    args = ["convert", db_path, "example", "records", code]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 0, result.output
+    assert json.loads(db["example"].get(1)["records"]) == expected_array
+
+
+@pytest.mark.parametrize("drop", (True, False))
+def test_recipe_jsonsplit_output(fresh_db_and_path, drop):
+    db, db_path = fresh_db_and_path
+    db["example"].insert_all(
+        [
+            {"id": 1, "records": "1,2,3"},
+        ],
+        pk="id",
+    )
+    code = "r.jsonsplit(value)"
+    args = ["convert", db_path, "example", "records", code, "--output", "tags"]
+    if drop:
+        args += ["--drop"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 0, result.output
+    expected = {
+        "id": 1,
+        "records": "1,2,3",
+        "tags": '["1", "2", "3"]',
+    }
+    if drop:
+        del expected["records"]
+    assert db["example"].get(1) == expected
+
+
+def test_cannot_use_drop_without_multi_or_output(fresh_db_and_path):
+    args = ["convert", fresh_db_and_path[1], "example", "records", "value", "--drop"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 1, result.output
+    assert "Error: --drop can only be used with --output or --multi" in result.output
+
+
+def test_cannot_use_multi_with_more_than_one_column(fresh_db_and_path):
+    args = [
+        "convert",
+        fresh_db_and_path[1],
+        "example",
+        "records",
+        "othercol",
+        "value",
+        "--multi",
+    ]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 1, result.output
+    assert "Error: Cannot use --multi with more than one column" in result.output
+
+
+def test_multi_with_bad_function(test_db_and_path):
+    args = [
+        "convert",
+        test_db_and_path[1],
+        "example",
+        "dt",
+        "value.upper()",
+        "--multi",
+    ]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 1, result.output
+    assert "When using --multi code must return a Python dictionary" in result.output
+
+
+def test_convert_where(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "str(value).upper()",
+            "--where",
+            "id = :id",
+            "-p",
+            "id",
+            "2",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["example"].rows) == [
+        {"id": 1, "dt": "5th October 2019 12:04"},
+        {"id": 2, "dt": "6TH OCTOBER 2019 00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ]
+
+
+def test_convert_where_multi(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["names"].insert_all(
+        [{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}], pk="id"
+    )
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "names",
+            "name",
+            '{"upper": value.upper()}',
+            "--where",
+            "id = :id",
+            "-p",
+            "id",
+            "2",
+            "--multi",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["names"].rows) == [
+        {"id": 1, "name": "Cleo", "upper": None},
+        {"id": 2, "name": "Bants", "upper": "BANTS"},
+    ]
+
+
+def test_convert_code_standard_input(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "names",
+            "name",
+            "-",
+        ],
+        input="value.upper()",
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["names"].rows) == [
+        {"id": 1, "name": "CLEO"},
+    ]
+
+
+def test_convert_hyphen_workaround(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["convert", db_path, "names", "name", '"-"'],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["names"].rows) == [
+        {"id": 1, "name": "-"},
+    ]
+
+
+def test_convert_initialization_pattern(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "names",
+            "name",
+            "-",
+        ],
+        input="import random\nrandom.seed(1)\ndef convert(value):    return random.randint(0, 100)",
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["names"].rows) == [
+        {"id": 1, "name": "17"},
+    ]
+
+
+def test_convert_handles_falsey_values(fresh_db_and_path):
+    # Falsey values like 0 should be converted (issue #527)
+    db, db_path = fresh_db_and_path
+    args = [
+        "convert",
+        db_path,
+        "t",
+        "x",
+        "-",
+    ]
+    db["t"].insert_all([{"x": 0}, {"x": 1}])
+    assert db["t"].get(1)["x"] == 0
+    assert db["t"].get(2)["x"] == 1
+    result = CliRunner().invoke(cli.cli, args, input="value + 1")
+    assert result.exit_code == 0, result.output
+    assert db["t"].get(1)["x"] == 1
+    assert db["t"].get(2)["x"] == 2
+
+
+@pytest.mark.parametrize(
+    "code",
+    [
+        # Direct callable reference (issue #686)
+        "r.parsedate",
+        "recipes.parsedate",
+        # Traditional call syntax still works
+        "r.parsedate(value)",
+        "recipes.parsedate(value)",
+    ],
+)
+def test_convert_callable_reference(test_db_and_path, code):
+    """Test that callable references like r.parsedate work without (value)"""
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli, ["convert", db_path, "example", "dt", code], catch_exceptions=False
+    )
+    assert result.exit_code == 0, result.output
+    rows = list(db["example"].rows)
+    assert rows[0]["dt"] == "2019-10-05"
+    assert rows[1]["dt"] == "2019-10-06"
+    assert rows[2]["dt"] == ""
+    assert rows[3]["dt"] is None
+
+
+def test_convert_callable_reference_with_import(fresh_db_and_path):
+    """Test callable reference from an imported module"""
+    db, db_path = fresh_db_and_path
+    db["example"].insert({"id": 1, "data": '{"name": "test"}'})
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "data",
+            "json.loads",
+            "--import",
+            "json",
+        ],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0, result.output
+    # json.loads returns a dict, which sqlite stores as JSON string
+    row = db["example"].get(1)
+    assert row["data"] == '{"name": "test"}'

tests/test_cli_insert.py

@@ -0,0 +1,890 @@
+from sqlite_utils import cli, Database
+from click.testing import CliRunner
+import json
+import pytest
+import subprocess
+import sys
+import time
+
+
+def test_insert_simple(tmpdir):
+    json_path = str(tmpdir / "dog.json")
+    db_path = str(tmpdir / "dogs.db")
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps({"name": "Cleo", "age": 4}))
+    result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path])
+    assert result.exit_code == 0
+    assert [{"age": 4, "name": "Cleo"}] == list(
+        Database(db_path).query("select * from dogs")
+    )
+    db = Database(db_path)
+    assert ["dogs"] == db.table_names()
+    assert [] == db["dogs"].indexes
+
+
+def test_insert_from_stdin(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", "-"],
+        input=json.dumps({"name": "Cleo", "age": 4}),
+    )
+    assert result.exit_code == 0
+    assert [{"age": 4, "name": "Cleo"}] == list(
+        Database(db_path).query("select * from dogs")
+    )
+
+
+def test_insert_invalid_json_error(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", "-"],
+        input="name,age\nCleo,4",
+    )
+    assert result.exit_code == 1
+    assert result.output == (
+        "Error: Invalid JSON - use --csv for CSV or --tsv for TSV files\n\n"
+        "JSON error: Expecting value: line 1 column 1 (char 0)\n"
+    )
+
+
+def test_insert_json_flatten(tmpdir):
+    db_path = str(tmpdir / "flat.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "items", "-", "--flatten"],
+        input=json.dumps({"nested": {"data": 4}}),
+    )
+    assert result.exit_code == 0
+    assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}]
+
+
+def test_insert_json_flatten_nl(tmpdir):
+    db_path = str(tmpdir / "flat.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "items", "-", "--flatten", "--nl"],
+        input="\n".join(
+            json.dumps(item)
+            for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}]
+        ),
+    )
+    assert result.exit_code == 0
+    assert list(Database(db_path).query("select * from items")) == [
+        {"nested_data": 4, "nested_other": None},
+        {"nested_data": None, "nested_other": 3},
+    ]
+
+
+@pytest.mark.parametrize(
+    "args,expected_pks",
+    (
+        (["--pk", "id"], ["id"]),
+        (["--pk", "id", "--pk", "name"], ["id", "name"]),
+    ),
+)
+def test_insert_with_primary_keys(db_path, tmpdir, args, expected_pks):
+    json_path = str(tmpdir / "dog.json")
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps({"id": 1, "name": "Cleo", "age": 4}))
+    result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path] + args)
+    assert result.exit_code == 0
+    assert [{"id": 1, "age": 4, "name": "Cleo"}] == list(
+        Database(db_path).query("select * from dogs")
+    )
+    db = Database(db_path)
+    assert db["dogs"].pks == expected_pks
+
+
+def test_insert_multiple_with_primary_key(db_path, tmpdir):
+    json_path = str(tmpdir / "dogs.json")
+    dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    assert dogs == list(db.query("select * from dogs order by id"))
+    assert ["id"] == db["dogs"].pks
+
+
+def test_insert_multiple_with_compound_primary_key(db_path, tmpdir):
+    json_path = str(tmpdir / "dogs.json")
+    dogs = [
+        {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3}
+        for i in range(1, 21)
+    ]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--pk", "breed"]
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    assert dogs == list(db.query("select * from dogs order by breed, id"))
+    assert {"breed", "id"} == set(db["dogs"].pks)
+    assert (
+        'CREATE TABLE "dogs" (\n'
+        '   "breed" TEXT,\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT,\n'
+        '   "age" INTEGER,\n'
+        '   PRIMARY KEY ("id", "breed")\n'
+        ")"
+    ) == db["dogs"].schema
+
+
+def test_insert_not_null_default(db_path, tmpdir):
+    json_path = str(tmpdir / "dogs.json")
+    dogs = [
+        {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10}
+        for i in range(1, 21)
+    ]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(dogs))
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", json_path, "--pk", "id"]
+        + ["--not-null", "name", "--not-null", "age"]
+        + ["--default", "score", "5", "--default", "age", "1"],
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    assert (
+        'CREATE TABLE "dogs" (\n'
+        '   "id" INTEGER PRIMARY KEY,\n'
+        '   "name" TEXT NOT NULL,\n'
+        "   \"age\" INTEGER NOT NULL DEFAULT '1',\n"
+        "   \"score\" INTEGER DEFAULT '5'\n)"
+    ) == db["dogs"].schema
+
+
+def test_insert_binary_base64(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "files", "-"],
+        input=r'{"content": {"$base64": true, "encoded": "aGVsbG8="}}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    actual = list(db.query("select content from files"))
+    assert actual == [{"content": b"hello"}]
+
+
+def test_insert_newline_delimited(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl"],
+        input='{"foo": "bar", "n": 1}\n\n{"foo": "baz", "n": 2}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert [
+        {"foo": "bar", "n": 1},
+        {"foo": "baz", "n": 2},
+    ] == list(db.query("select foo, n from from_json_nl"))
+
+
+def test_insert_ignore(db_path, tmpdir):
+    db = Database(db_path)
+    db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+    json_path = str(tmpdir / "dogs.json")
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps([{"id": 1, "name": "Bailey"}]))
+    # Should raise error without --ignore
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
+    )
+    assert result.exit_code != 0, result.output
+    # If we use --ignore it should run OK
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--ignore"]
+    )
+    assert result.exit_code == 0, result.output
+    # ... but it should actually have no effect
+    assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs"))
+
+
+@pytest.mark.parametrize(
+    "content,options",
+    [
+        ("foo\tbar\tbaz\n1\t2\tcat,dog", ["--tsv"]),
+        ('foo,bar,baz\n1,2,"cat,dog"', ["--csv"]),
+        ('foo;bar;baz\n1;2;"cat,dog"', ["--csv", "--delimiter", ";"]),
+        # --delimiter implies --csv:
+        ('foo;bar;baz\n1;2;"cat,dog"', ["--delimiter", ";"]),
+        ("foo,bar,baz\n1,2,|cat,dog|", ["--csv", "--quotechar", "|"]),
+        ("foo,bar,baz\n1,2,|cat,dog|", ["--quotechar", "|"]),
+    ],
+)
+def test_insert_csv_tsv(content, options, db_path, tmpdir):
+    db = Database(db_path)
+    file_path = str(tmpdir / "insert.csv-tsv")
+    with open(file_path, "w") as fp:
+        fp.write(content)
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", file_path] + options + ["--no-detect-types"],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows)
+
+
+@pytest.mark.parametrize("empty_null", (True, False))
+def test_insert_csv_empty_null(db_path, empty_null):
+    options = ["--csv", "--no-detect-types"]
+    if empty_null:
+        options.append("--empty-null")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", "-"] + options,
+        catch_exceptions=False,
+        input="foo,bar,baz\n1,,cat,dog",
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    assert [r for r in db["data"].rows] == [
+        {"foo": "1", "bar": None if empty_null else "", "baz": "cat"}
+    ]
+
+
+@pytest.mark.parametrize(
+    "input,args",
+    (
+        (
+            json.dumps(
+                [{"name": "One"}, {"name": "Two"}, {"name": "Three"}, {"name": "Four"}]
+            ),
+            [],
+        ),
+        ("name\nOne\nTwo\nThree\nFour\n", ["--csv"]),
+    ),
+)
+def test_insert_stop_after(tmpdir, input, args):
+    db_path = str(tmpdir / "data.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "rows", "-", "--stop-after", "2"] + args,
+        input=input,
+    )
+    assert result.exit_code == 0
+    assert [{"name": "One"}, {"name": "Two"}] == list(
+        Database(db_path).query("select * from rows")
+    )
+
+
+@pytest.mark.parametrize(
+    "options",
+    (
+        ["--tsv", "--nl"],
+        ["--tsv", "--csv"],
+        ["--csv", "--nl"],
+        ["--csv", "--nl", "--tsv"],
+    ),
+)
+def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir):
+    file_path = str(tmpdir / "insert.csv-tsv")
+    with open(file_path, "w") as fp:
+        fp.write("foo")
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "data", file_path] + options
+    )
+    assert result.exit_code != 0
+    assert "Error: Use just one of --nl, --csv or --tsv" == result.output.strip()
+
+
+def test_insert_replace(db_path, tmpdir):
+    test_insert_multiple_with_primary_key(db_path, tmpdir)
+    json_path = str(tmpdir / "insert-replace.json")
+    db = Database(db_path)
+    assert db["dogs"].count == 20
+    insert_replace_dogs = [
+        {"id": 1, "name": "Insert replaced 1", "age": 4},
+        {"id": 2, "name": "Insert replaced 2", "age": 4},
+        {"id": 21, "name": "Fresh insert 21", "age": 6},
+    ]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(insert_replace_dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"]
+    )
+    assert result.exit_code == 0, result.output
+    assert db["dogs"].count == 21
+    assert (
+        list(db.query("select * from dogs where id in (1, 2, 21) order by id"))
+        == insert_replace_dogs
+    )
+
+
+def test_insert_truncate(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl", "--batch-size=1"],
+        input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert [
+        {"foo": "bar", "n": 1},
+        {"foo": "baz", "n": 2},
+    ] == list(db.query("select foo, n from from_json_nl"))
+    # Truncate and insert new rows
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "from_json_nl",
+            "-",
+            "--nl",
+            "--truncate",
+            "--batch-size=1",
+        ],
+        input='{"foo": "bam", "n": 3}\n{"foo": "bat", "n": 4}',
+    )
+    assert result.exit_code == 0, result.output
+    assert [
+        {"foo": "bam", "n": 3},
+        {"foo": "bat", "n": 4},
+    ] == list(db.query("select foo, n from from_json_nl"))
+
+
+def test_insert_alter(db_path, tmpdir):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl"],
+        input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}',
+    )
+    assert result.exit_code == 0, result.output
+    # Should get an error with incorrect shaped additional data
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl"],
+        input='{"foo": "bar", "baz": 5}',
+    )
+    assert result.exit_code != 0, result.output
+    # If we run it again with --alter it should work correctly
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_json_nl", "-", "--nl", "--alter"],
+        input='{"foo": "bar", "baz": 5}',
+    )
+    assert result.exit_code == 0, result.output
+    # Soundness check the database itself
+    db = Database(db_path)
+    assert {"foo": str, "n": int, "baz": int} == db["from_json_nl"].columns_dict
+    assert [
+        {"foo": "bar", "n": 1, "baz": None},
+        {"foo": "baz", "n": 2, "baz": None},
+        {"foo": "bar", "baz": 5, "n": None},
+    ] == list(db.query("select foo, n, baz from from_json_nl"))
+
+
+def test_insert_analyze(db_path):
+    db = Database(db_path)
+    db["rows"].insert({"foo": "x", "n": 3})
+    db["rows"].create_index(["n"])
+    assert "sqlite_stat1" not in db.table_names()
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "rows", "-", "--nl", "--analyze"],
+        input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}',
+    )
+    assert result.exit_code == 0, result.output
+    assert "sqlite_stat1" in db.table_names()
+
+
+def test_insert_lines(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_lines", "-", "--lines"],
+        input='First line\nSecond line\n{"foo": "baz"}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert [
+        {"line": "First line"},
+        {"line": "Second line"},
+        {"line": '{"foo": "baz"}'},
+    ] == list(db.query("select line from from_lines"))
+
+
+def test_insert_text(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "from_text", "-", "--text"],
+        input='First line\nSecond line\n{"foo": "baz"}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert [{"text": 'First line\nSecond line\n{"foo": "baz"}'}] == list(
+        db.query("select text from from_text")
+    )
+
+
+@pytest.mark.parametrize(
+    "options,input",
+    (
+        ([], '[{"id": "1", "name": "Bob"}, {"id": "2", "name": "Cat"}]'),
+        (["--csv", "--no-detect-types"], "id,name\n1,Bob\n2,Cat"),
+        (["--nl"], '{"id": "1", "name": "Bob"}\n{"id": "2", "name": "Cat"}'),
+    ),
+)
+def test_insert_convert_json_csv_jsonnl(db_path, options, input):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "rows", "-", "--convert", '{**row, **{"extra": 1}}']
+        + options,
+        input=input,
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query("select id, name, extra from rows"))
+    assert rows == [
+        {"id": "1", "name": "Bob", "extra": 1},
+        {"id": "2", "name": "Cat", "extra": 1},
+    ]
+
+
+def test_insert_convert_text(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "text",
+            "-",
+            "--text",
+            "--convert",
+            '{"text": text.upper()}',
+        ],
+        input="This is text\nwill be upper now",
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query('select "text" from "text"'))
+    assert rows == [{"text": "THIS IS TEXT\nWILL BE UPPER NOW"}]
+
+
+def test_insert_convert_text_returning_iterator(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "text",
+            "-",
+            "--text",
+            "--convert",
+            '({"word": w} for w in text.split())',
+        ],
+        input="A bunch of words",
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query('select "word" from "text"'))
+    assert rows == [{"word": "A"}, {"word": "bunch"}, {"word": "of"}, {"word": "words"}]
+
+
+def test_insert_convert_lines(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "all",
+            "-",
+            "--lines",
+            "--convert",
+            '{"line": line.upper()}',
+        ],
+        input="This is text\nwill be upper now",
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query('select "line" from "all"'))
+    assert rows == [{"line": "THIS IS TEXT"}, {"line": "WILL BE UPPER NOW"}]
+
+
+def test_insert_convert_row_modifying_in_place(db_path):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "rows",
+            "-",
+            "--convert",
+            'row["is_chicken"] = True',
+        ],
+        input='{"name": "Azi"}',
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    rows = list(db.query("select name, is_chicken from rows"))
+    assert rows == [{"name": "Azi", "is_chicken": 1}]
+
+
+@pytest.mark.parametrize(
+    "options,expected_error",
+    (
+        (
+            ["--text", "--convert", "1"],
+            "Error: --convert must return dict or iterator\n",
+        ),
+        (["--convert", "1"], "Error: Rows must all be dictionaries, got: 1\n"),
+    ),
+)
+def test_insert_convert_error_messages(db_path, options, expected_error):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "rows",
+            "-",
+        ]
+        + options,
+        input='{"name": "Azi"}',
+    )
+    assert result.exit_code == 1
+    assert result.output == expected_error
+
+
+def test_insert_streaming_batch_size_1(db_path):
+    # https://github.com/simonw/sqlite-utils/issues/364
+    # Streaming with --batch-size 1 should commit on each record
+    # Can't use CliRunner().invoke() here bacuse we need to
+    # run assertions in between writing to process stdin
+    proc = subprocess.Popen(
+        [
+            sys.executable,
+            "-m",
+            "sqlite_utils",
+            "insert",
+            db_path,
+            "rows",
+            "-",
+            "--nl",
+            "--batch-size",
+            "1",
+        ],
+        stdin=subprocess.PIPE,
+        stdout=sys.stdout,
+    )
+    proc.stdin.write(b'{"name": "Azi"}\n')
+    proc.stdin.flush()
+
+    def try_until(expected):
+        tries = 0
+        while True:
+            rows = list(Database(db_path)["rows"].rows)
+            if rows == expected:
+                return
+            tries += 1
+            if tries > 10:
+                assert False, "Expected {}, got {}".format(expected, rows)
+            time.sleep(tries * 0.1)
+
+    try_until([{"name": "Azi"}])
+    proc.stdin.write(b'{"name": "Suna"}\n')
+    proc.stdin.flush()
+    try_until([{"name": "Azi"}, {"name": "Suna"}])
+    proc.stdin.close()
+    proc.wait()
+    assert proc.returncode == 0
+
+
+def test_insert_csv_headers_only(tmpdir):
+    """Test that CSV with only header row (no data) works with --detect-types (issue #702)"""
+    db_path = str(tmpdir / "test.db")
+    csv_path = str(tmpdir / "headers_only.csv")
+    with open(csv_path, "w") as fp:
+        fp.write("id,name,age\n")
+    # Should not crash with --detect-types (which is now the default)
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", csv_path, "--csv"],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    # Table should not exist since there were no data rows
+    db = Database(db_path)
+    assert not db["data"].exists()
+
+
+def test_insert_into_view_errors(tmpdir):
+    db_path = str(tmpdir / "test.db")
+    db = Database(db_path)
+    db["t"].insert({"id": 1})
+    db.create_view("v", "select * from t")
+    db.close()
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "v", "-"], input='{"id": 2}'
+    )
+    assert result.exit_code == 1
+    assert result.output.strip() == "Error: Table v is actually a view"
+
+
+def test_insert_csv_detect_types_leaves_existing_table_alone(db_path):
+    # Type detection is the default for CSV/TSV inserts, but it must only
+    # apply to tables created by this command - transforming a pre-existing
+    # table would rewrite its column types and corrupt data such as
+    # TEXT zip codes with leading zeros
+    db = Database(db_path)
+    db["places"].insert({"name": "Boston", "zip": "01234"})
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "places", "-", "--csv"],
+        catch_exceptions=False,
+        input="name,zip\nSF,94107",
+    )
+    assert result.exit_code == 0, result.output
+    assert db["places"].columns_dict["zip"] is str
+    assert list(db["places"].rows) == [
+        {"name": "Boston", "zip": "01234"},
+        {"name": "SF", "zip": "94107"},
+    ]
+
+
+def test_insert_csv_detect_types_new_table(db_path):
+    # A table created by the insert still gets detected types
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", "-", "--csv"],
+        catch_exceptions=False,
+        input="name,age,weight\nCleo,5,12.5",
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert db["data"].columns_dict == {"name": str, "age": int, "weight": float}
+
+
+@pytest.mark.parametrize(
+    "command,extra_args,input_text,expected_row",
+    (
+        (
+            "insert",
+            [],
+            "zipcode,score\n01234,9.5\n",
+            {"zipcode": "01234", "score": 9.5},
+        ),
+        (
+            "upsert",
+            ["--pk", "id"],
+            "id,zipcode,score\n1,01234,9.5\n",
+            {"id": 1, "zipcode": "01234", "score": 9.5},
+        ),
+    ),
+)
+def test_insert_upsert_csv_type_overrides_detected_types(
+    db_path, command, extra_args, input_text, expected_row
+):
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            command,
+            db_path,
+            "places",
+            "-",
+            "--csv",
+        ]
+        + extra_args
+        + [
+            "--type",
+            "zipcode",
+            "text",
+        ],
+        catch_exceptions=False,
+        input=input_text,
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    expected_columns = {"zipcode": str, "score": float}
+    if command == "upsert":
+        expected_columns = {"id": int, **expected_columns}
+    assert db["places"].columns_dict == expected_columns
+    assert list(db["places"].rows) == [expected_row]
+
+
+def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path):
+    db = Database(db_path)
+    db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["upsert", db_path, "places", "-", "--csv", "--pk", "id"],
+        catch_exceptions=False,
+        input="id,name,zip\n2,SF,94107",
+    )
+    assert result.exit_code == 0, result.output
+    assert db["places"].columns_dict["zip"] is str
+    assert db["places"].get(1)["zip"] == "01234"
+
+
+def test_insert_invalid_pk_clean_error(db_path):
+    # An invalid --pk against an existing table should be a clean CLI
+    # error, not a raw InvalidColumns traceback
+    db = Database(db_path)
+    db["t"].insert({"a": 1})
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "t", "-", "--pk", "badcol"],
+        input='{"a": 2}',
+    )
+    assert result.exit_code == 1
+    assert result.exception is None or isinstance(result.exception, SystemExit)
+    assert result.output.startswith("Error: Invalid primary key column")
+
+
+# --code tests, see https://github.com/simonw/sqlite-utils/issues/684
+CODE_ROWS_FUNCTION = """
+def rows():
+    yield {"id": 1, "name": "Cleo"}
+    yield {"id": 2, "name": "Suna"}
+"""
+
+CODE_ROWS_ITERABLE = """
+rows = [
+    {"id": 1, "name": "Cleo"},
+    {"id": 2, "name": "Suna"},
+]
+"""
+
+
+@pytest.mark.parametrize("code", (CODE_ROWS_FUNCTION, CODE_ROWS_ITERABLE))
+def test_insert_code(tmpdir, code):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", code, "--pk", "id"],
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert db["creatures"].pks == ["id"]
+    assert list(db["creatures"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Suna"},
+    ]
+
+
+def test_insert_code_from_file(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    code_path = str(tmpdir / "gen.py")
+    with open(code_path, "w") as fp:
+        fp.write(CODE_ROWS_FUNCTION)
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", code_path],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(Database(db_path)["creatures"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Suna"},
+    ]
+
+
+def test_upsert_code(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    db = Database(db_path)
+    db["creatures"].insert_all(
+        [{"id": 1, "name": "old"}, {"id": 2, "name": "Suna"}], pk="id"
+    )
+    result = CliRunner().invoke(
+        cli.cli,
+        ["upsert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--pk", "id"],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["creatures"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Suna"},
+    ]
+
+
+def test_insert_code_requires_file_or_code(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(cli.cli, ["insert", db_path, "creatures"])
+    assert result.exit_code == 1
+    assert "Provide either a FILE argument or --code" in result.output
+
+
+def test_insert_code_mutually_exclusive_with_file(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "-", "--code", CODE_ROWS_FUNCTION],
+        input="{}",
+    )
+    assert result.exit_code == 1
+    assert "--code cannot be used with a FILE argument" in result.output
+
+
+def test_insert_code_rejects_input_format_options(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--csv"],
+    )
+    assert result.exit_code == 1
+    assert "--code cannot be used with input format options" in result.output
+
+
+def test_insert_code_missing_rows(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", "x = 1"],
+    )
+    assert result.exit_code == 1
+    assert "must define a 'rows' function or iterable" in result.output
+
+
+def test_insert_code_single_dict(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "creatures",
+            "--code",
+            'rows = {"id": 1, "name": "Cleo"}',
+            "--pk",
+            "id",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(Database(db_path)["creatures"].rows) == [{"id": 1, "name": "Cleo"}]
+
+
+def test_insert_code_not_iterable(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", "rows = 5"],
+    )
+    assert result.exit_code == 1
+    assert "must define a 'rows' function or iterable" in result.output
+
+
+def test_insert_code_syntax_error(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", "def rows(:"],
+    )
+    assert result.exit_code == 1
+    assert "Error in --code" in result.output
+
+
+def test_insert_code_file_not_found(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "creatures", "--code", "missing.py"],
+    )
+    assert result.exit_code == 1
+    assert "File not found: missing.py" in result.output

tests/test_cli_memory.py

@@ -1,5 +1,5 @@
+import click
 import json
-
 import pytest
 from click.testing import CliRunner
 
@@ -24,7 +24,8 @@ def test_memory_csv(tmpdir, sql_from, use_stdin):
             sql_from = "stdin"
     else:
         csv_path = str(tmpdir / "test.csv")
-        open(csv_path, "w").write(content)
+        with open(csv_path, "w") as fp:
+            fp.write(content)
     result = CliRunner().invoke(
         cli.cli,
         ["memory", csv_path, "select * from {}".format(sql_from), "--nl"],
@@ -46,13 +47,14 @@ def test_memory_tsv(tmpdir, use_stdin):
     else:
         input = None
         path = str(tmpdir / "chickens.tsv")
-        open(path, "w").write(data)
+        with open(path, "w") as fp:
+            fp.write(data)
         path = path + ":tsv"
         sql_from = "chickens"
     result = CliRunner().invoke(
         cli.cli,
         ["memory", path, "select * from {}".format(sql_from)],
-        input=data,
+        input=input,
     )
     assert result.exit_code == 0, result.output
     assert json.loads(result.output.strip()) == [
@@ -71,7 +73,8 @@ def test_memory_json(tmpdir, use_stdin):
     else:
         input = None
         path = str(tmpdir / "chickens.json")
-        open(path, "w").write(data)
+        with open(path, "w") as fp:
+            fp.write(data)
         path = path + ":json"
         sql_from = "chickens"
     result = CliRunner().invoke(
@@ -96,13 +99,14 @@ def test_memory_json_nl(tmpdir, use_stdin):
     else:
         input = None
         path = str(tmpdir / "chickens.json")
-        open(path, "w").write(data)
+        with open(path, "w") as fp:
+            fp.write(data)
         path = path + ":nl"
         sql_from = "chickens"
     result = CliRunner().invoke(
         cli.cli,
         ["memory", path, "select * from {}".format(sql_from)],
-        input=data,
+        input=input,
     )
     assert result.exit_code == 0, result.output
     assert json.loads(result.output.strip()) == [
@@ -152,6 +156,24 @@ def test_memory_csv_encoding(tmpdir, use_stdin):
     }
 
 
+def test_memory_csv_headers_only(tmpdir):
+    csv_path = str(tmpdir / "headers_only.csv")
+    with open(csv_path, "w") as fp:
+        fp.write("id,name,age\n")
+
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", csv_path, "", "--schema"],
+        catch_exceptions=False,
+    )
+
+    assert result.exit_code == 0
+    assert result.output.strip() == (
+        'CREATE VIEW "t1" AS select * from "headers_only";\n'
+        'CREATE VIEW "t" AS select * from "headers_only";'
+    )
+
+
 @pytest.mark.parametrize("extra_args", ([], ["select 1"]))
 def test_memory_dump(extra_args):
     result = CliRunner().invoke(
@@ -160,18 +182,21 @@ def test_memory_dump(extra_args):
         input="id,name\n1,Cleo\n2,Bants",
     )
     assert result.exit_code == 0
-    assert result.output.strip() == (
+    expected = (
         "BEGIN TRANSACTION;\n"
-        'CREATE TABLE "stdin" (\n'
+        'CREATE TABLE IF NOT EXISTS "stdin" (\n'
-        "   [id] INTEGER,\n"
+        '   "id" INTEGER,\n'
-        "   [name] TEXT\n"
+        '   "name" TEXT\n'
         ");\n"
         "INSERT INTO \"stdin\" VALUES(1,'Cleo');\n"
         "INSERT INTO \"stdin\" VALUES(2,'Bants');\n"
-        "CREATE VIEW t1 AS select * from [stdin];\n"
+        'CREATE VIEW "t1" AS select * from "stdin";\n'
-        "CREATE VIEW t AS select * from [stdin];\n"
+        'CREATE VIEW "t" AS select * from "stdin";\n'
         "COMMIT;"
     )
+    # Using sqlite-dump it won't have IF NOT EXISTS
+    expected_alternative = expected.replace("IF NOT EXISTS ", "")
+    assert result.output.strip() in (expected, expected_alternative)
 
 
 @pytest.mark.parametrize("extra_args", ([], ["select 1"]))
@@ -184,11 +209,11 @@ def test_memory_schema(extra_args):
     assert result.exit_code == 0
     assert result.output.strip() == (
         'CREATE TABLE "stdin" (\n'
-        "   [id] INTEGER,\n"
+        '   "id" INTEGER,\n'
-        "   [name] TEXT\n"
+        '   "name" TEXT\n'
         ");\n"
-        "CREATE VIEW t1 AS select * from [stdin];\n"
+        'CREATE VIEW "t1" AS select * from "stdin";\n'
-        "CREATE VIEW t AS select * from [stdin];"
+        'CREATE VIEW "t" AS select * from "stdin";'
     )
 
 
@@ -220,3 +245,111 @@ def test_memory_no_detect_types(option):
         {"id": "1", "name": "Cleo", "weight": "45.5"},
         {"id": "2", "name": "Bants", "weight": "3.5"},
     ]
+
+
+def test_memory_flatten():
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-", "select * from stdin", "--flatten"],
+        input=json.dumps(
+            {
+                "httpRequest": {
+                    "latency": "0.112114537s",
+                    "requestMethod": "GET",
+                },
+                "insertId": "6111722f000b5b4c4d4071e2",
+            }
+        ),
+    )
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.output.strip()) == [
+        {
+            "httpRequest_latency": "0.112114537s",
+            "httpRequest_requestMethod": "GET",
+            "insertId": "6111722f000b5b4c4d4071e2",
+        }
+    ]
+
+
+def test_memory_analyze():
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-", "--analyze"],
+        input="id,name\n1,Cleo\n2,Bants",
+    )
+    assert result.exit_code == 0
+    assert result.output == (
+        "stdin.id: (1/2)\n\n"
+        "  Total rows: 2\n"
+        "  Null rows: 0\n"
+        "  Blank rows: 0\n\n"
+        "  Distinct values: 2\n\n"
+        "stdin.name: (2/2)\n\n"
+        "  Total rows: 2\n"
+        "  Null rows: 0\n"
+        "  Blank rows: 0\n\n"
+        "  Distinct values: 2\n\n"
+    )
+
+
+def test_memory_two_files_with_same_stem(tmpdir):
+    (tmpdir / "one").mkdir()
+    (tmpdir / "two").mkdir()
+    one = tmpdir / "one" / "data.csv"
+    two = tmpdir / "two" / "data.csv"
+    one.write_text("id,name\n1,Cleo\n2,Bants", encoding="utf-8")
+    two.write_text("id,name\n3,Blue\n4,Lila", encoding="utf-8")
+    result = CliRunner().invoke(cli.cli, ["memory", str(one), str(two), "", "--schema"])
+    assert result.exit_code == 0
+    assert result.output == (
+        'CREATE TABLE "data" (\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT\n'
+        ");\n"
+        'CREATE VIEW "t1" AS select * from "data";\n'
+        'CREATE VIEW "t" AS select * from "data";\n'
+        'CREATE TABLE "data_2" (\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT\n'
+        ");\n"
+        'CREATE VIEW "t2" AS select * from "data_2";\n'
+    )
+
+
+def test_memory_functions():
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "select hello()", "--functions", "hello = lambda: 'Hello'"],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == '[{"hello()": "Hello"}]'
+
+
+def test_memory_functions_multiple():
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "memory",
+            "select triple(2), quadruple(2)",
+            "--functions",
+            "def triple(x):\n    return x * 3",
+            "--functions",
+            "def quadruple(x):\n    return x * 4",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == '[{"triple(2)": 6, "quadruple(2)": 8}]'
+
+
+def test_memory_return_db(tmpdir):
+    # https://github.com/simonw/sqlite-utils/issues/643
+    from sqlite_utils.cli import cli
+
+    path = str(tmpdir / "dogs.csv")
+    with open(path, "w") as f:
+        f.write("id,name\n1,Cleo")
+
+    with click.Context(cli) as ctx:  # type: ignore[attr-defined]
+        db = ctx.invoke(cli.commands["memory"], paths=(path,), return_db=True)
+
+    assert db.table_names() == ["dogs"]

tests/test_cli_migrate.py

@@ -0,0 +1,507 @@
+import pathlib
+
+from click.testing import CliRunner
+import pytest
+import sqlite_utils
+import sqlite_utils.cli
+
+TWO_MIGRATIONS = """
+from sqlite_utils import Migrations
+
+m = Migrations("hello")
+
+@m()
+def foo(db):
+    db["foo"].insert({"hello": "world"})
+
+@m()
+def bar(db):
+    db["bar"].insert({"hello": "world"})
+"""
+
+
+@pytest.fixture
+def two_migrations(tmpdir):
+    path = pathlib.Path(tmpdir)
+    (path / "foo").mkdir()
+    migrations_py = path / "foo" / "migrations.py"
+    migrations_py.write_text(TWO_MIGRATIONS, "utf-8")
+    return path, migrations_py
+
+
+@pytest.fixture
+def two_sets_same_migration_name(tmpdir):
+    path = pathlib.Path(tmpdir)
+    migrations_py = path / "migrations.py"
+    migrations_py.write_text(
+        """
+from sqlite_utils import Migrations
+
+creatures = Migrations("creatures")
+
+@creatures()
+def create_table(db):
+    db["creatures"].insert({"name": "Cleo"})
+
+@creatures()
+def add_weight(db):
+    db["creature_weights"].insert({"weight": 4.2})
+
+sales = Migrations("sales")
+
+@sales()
+def create_table(db):
+    db["sales"].insert({"id": 1})
+
+@sales()
+def add_weight(db):
+    db["sales_weights"].insert({"weight": 10})
+""",
+        "utf-8",
+    )
+    return path, migrations_py
+
+
+@pytest.mark.parametrize("arg", ("TMPDIR", "TMPDIR/foo/migrations.py", "TMPDIR/foo/"))
+def test_basic(two_migrations, arg):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+
+    runner = CliRunner()
+
+    def _list():
+        list_result = runner.invoke(
+            sqlite_utils.cli.cli,
+            ["migrate", db_path, "--list", arg.replace("TMPDIR", str(path))],
+        )
+        assert list_result.exit_code == 0
+        return list_result.output
+
+    assert _list() == (
+        "Migrations for: hello\n\n"
+        "  Applied:\n\n"
+        "  Pending:\n"
+        "    foo\n"
+        "    bar\n\n"
+    )
+
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, arg.replace("TMPDIR", str(path))]
+    )
+    assert result.exit_code == 0, result.output
+
+    list_output = _list()
+    assert "Migrations for: hello\n\n  Applied:\n    " in list_output
+    prior_to_pending = list_output.split(" Pending")[0]
+    assert "  foo" in prior_to_pending
+    assert "  bar" in prior_to_pending
+    assert " Pending:\n    (none)" in list_output
+
+    db = sqlite_utils.Database(db_path)
+    assert db["foo"].exists()
+    assert db["bar"].exists()
+    assert db["_sqlite_migrations"].exists()
+    rows = list(db["_sqlite_migrations"].rows)
+    assert len(rows) == 2
+    assert rows[0]["name"] == "foo"
+    assert rows[1]["name"] == "bar"
+
+
+def test_list_same_migration_names_in_different_sets(capsys):
+    applied = sqlite_utils.Migrations("applied")
+
+    @applied(name="foo")
+    def applied_foo(db):
+        db["applied"].insert({"hello": "world"})
+
+    pending = sqlite_utils.Migrations("pending")
+
+    @pending(name="foo")
+    def pending_foo(db):
+        db["pending"].insert({"hello": "world"})
+
+    db = sqlite_utils.Database(memory=True)
+    applied.apply(db)
+
+    sqlite_utils.cli._display_migration_list(db, [applied, pending])
+
+    output = capsys.readouterr().out
+    assert (
+        "Migrations for: pending\n\n" "  Applied:\n\n" "  Pending:\n" "    foo\n\n"
+    ) in output
+
+
+def test_verbose(tmpdir):
+    path = pathlib.Path(tmpdir)
+    (path / "foo").mkdir()
+    migrations_py = path / "foo" / "migrations.py"
+    migrations_py.write_text(
+        """
+from sqlite_utils import Migrations
+
+m = Migrations("hello")
+
+@m()
+def foo(db):
+    db["dogs"].insert({"id": 1, "name": "Cleo"})
+    """,
+        "utf-8",
+    )
+    db_path = str(path / "test.db")
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py)]
+    )
+    assert result.exit_code == 0
+
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"]
+    )
+    assert result.exit_code == 0
+    expected = """
+Schema before:
+
+  CREATE TABLE "_sqlite_migrations" (
+     "id" INTEGER PRIMARY KEY,
+     "migration_set" TEXT,
+     "name" TEXT,
+     "applied_at" TEXT
+  );
+  CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name"
+      ON "_sqlite_migrations" ("migration_set", "name");
+  CREATE TABLE "dogs" (
+     "id" INTEGER,
+     "name" TEXT
+  );
+
+Schema after:
+
+  (unchanged)
+""".strip()
+    assert expected in result.output
+
+    new_migration = """
+@m()
+def bar(db):
+    db["dogs"].add_column("age", int)
+    db["dogs"].add_column("weight", float)
+    db["dogs"].transform()
+"""
+    migrations_py.write_text(migrations_py.read_text("utf-8") + new_migration)
+
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"]
+    )
+    assert result.exit_code == 0
+    expected_diff = """
+Schema diff:
+
+     ON "_sqlite_migrations" ("migration_set", "name");
+ CREATE TABLE "dogs" (
+    "id" INTEGER,
+-   "name" TEXT
++   "name" TEXT,
++   "age" INTEGER,
++   "weight" REAL
+ );
+""".strip()
+    assert expected_diff in result.output
+
+
+def test_stop_before(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(path / "foo" / "migrations.py"),
+            "--stop-before",
+            "bar",
+        ],
+    )
+    assert result.exit_code == 0
+    db = sqlite_utils.Database(db_path)
+    assert db["foo"].exists()
+    assert not db["bar"].exists()
+
+
+def test_stop_before_multiple_sets_unqualified(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    (path / "foo" / "migrations2.py").write_text(
+        """
+from sqlite_utils import Migrations
+
+m = Migrations("hello2")
+
+@m()
+def foo(db):
+    db["foo"].insert({"hello": "world"})
+    """,
+        "utf-8",
+    )
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(path / "foo" / "migrations.py"),
+            str(path / "foo" / "migrations2.py"),
+            "--stop-before",
+            "foo",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    db = sqlite_utils.Database(db_path)
+    assert db.table_names() == ["_sqlite_migrations"]
+    assert list(db["_sqlite_migrations"].rows) == []
+
+
+def test_stop_before_qualified_only_affects_named_set(two_sets_same_migration_name):
+    path, migrations_py = two_sets_same_migration_name
+    db_path = str(path / "test.db")
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(migrations_py),
+            "--stop-before",
+            "creatures:add_weight",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    db = sqlite_utils.Database(db_path)
+    assert db["creatures"].exists()
+    assert not db["creature_weights"].exists()
+    assert db["sales"].exists()
+    assert db["sales_weights"].exists()
+
+
+def test_stop_before_multiple_qualified(two_sets_same_migration_name):
+    path, migrations_py = two_sets_same_migration_name
+    db_path = str(path / "test.db")
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(migrations_py),
+            "--stop-before",
+            "creatures:add_weight",
+            "--stop-before",
+            "sales:add_weight",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    db = sqlite_utils.Database(db_path)
+    assert db["creatures"].exists()
+    assert not db["creature_weights"].exists()
+    assert db["sales"].exists()
+    assert not db["sales_weights"].exists()
+
+
+LEGACY_MIGRATIONS = """
+import datetime
+
+class _Migration:
+    def __init__(self, name, fn):
+        self.name = name
+        self.fn = fn
+
+class _Applied:
+    def __init__(self, name, applied_at):
+        self.name = name
+        self.applied_at = applied_at
+
+class LegacyMigrations:
+    # Mimics the sqlite-migrate 0.x Migrations class, in particular
+    # apply(db, stop_before=None) taking a single string
+    migrations_table = "_sqlite_migrations"
+
+    def __init__(self, name):
+        self.name = name
+        self._migrations = []
+
+    def __call__(self, fn):
+        self._migrations.append(_Migration(fn.__name__, fn))
+        return fn
+
+    def ensure_migrations_table(self, db):
+        db[self.migrations_table].create(
+            {"migration_set": str, "name": str, "applied_at": str},
+            pk=("migration_set", "name"),
+            if_not_exists=True,
+        )
+
+    def applied(self, db):
+        self.ensure_migrations_table(db)
+        return [
+            _Applied(row["name"], row["applied_at"])
+            for row in db[self.migrations_table].rows_where(
+                "migration_set = ?", [self.name]
+            )
+        ]
+
+    def pending(self, db):
+        applied = {m.name for m in self.applied(db)}
+        return [m for m in self._migrations if m.name not in applied]
+
+    def apply(self, db, stop_before=None):
+        for migration in self.pending(db):
+            if migration.name == stop_before:
+                return
+            migration.fn(db)
+            db[self.migrations_table].insert(
+                {
+                    "migration_set": self.name,
+                    "name": migration.name,
+                    "applied_at": str(
+                        datetime.datetime.now(datetime.timezone.utc)
+                    ),
+                }
+            )
+
+legacy = LegacyMigrations("legacy_set")
+
+@legacy
+def first(db):
+    db["first"].insert({"hello": "world"})
+
+@legacy
+def second(db):
+    db["second"].insert({"hello": "world"})
+"""
+
+
+def test_stop_before_unknown_name_errors(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli,
+        ["migrate", db_path, str(path), "--stop-before", "fooo"],
+    )
+    assert result.exit_code == 1
+    assert "--stop-before did not match any migrations: fooo" in result.output
+    # Nothing should have been applied
+    db = sqlite_utils.Database(db_path)
+    assert "foo" not in db.table_names()
+    assert "bar" not in db.table_names()
+
+
+def test_stop_before_with_legacy_migrations_class(tmpdir):
+    path = pathlib.Path(tmpdir)
+    (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8")
+    db_path = str(path / "test.db")
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli,
+        ["migrate", db_path, str(path), "--stop-before", "second"],
+    )
+    assert result.exit_code == 0, result.output
+    db = sqlite_utils.Database(db_path)
+    assert "first" in db.table_names()
+    assert "second" not in db.table_names()
+
+
+def test_stop_before_multiple_values_for_legacy_set_errors(tmpdir):
+    path = pathlib.Path(tmpdir)
+    (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8")
+    db_path = str(path / "test.db")
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli,
+        [
+            "migrate",
+            db_path,
+            str(path),
+            "--stop-before",
+            "legacy_set:first",
+            "--stop-before",
+            "legacy_set:second",
+        ],
+    )
+    assert result.exit_code == 1
+    assert "single --stop-before" in result.output
+
+
+def test_list_does_not_create_database_file(two_migrations):
+    path, _ = two_migrations
+    db_path = path / "test.db"
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", str(db_path), str(path), "--list"]
+    )
+    assert result.exit_code == 0, result.output
+    assert "Pending:\n    foo\n    bar" in result.output
+    # Listing migrations must not create the database file
+    assert not db_path.exists()
+
+
+def test_list_does_not_upgrade_legacy_migrations_table(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    db = sqlite_utils.Database(db_path)
+    db["_sqlite_migrations"].create(
+        {"migration_set": str, "name": str, "applied_at": str},
+        pk=("migration_set", "name"),
+    )
+    db["_sqlite_migrations"].insert(
+        {"migration_set": "hello", "name": "foo", "applied_at": "x"}
+    )
+    db.close()
+    runner = CliRunner()
+    result = runner.invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"]
+    )
+    assert result.exit_code == 0, result.output
+    assert "foo - x" in result.output
+    # --list must not perform the one-way legacy schema upgrade
+    db2 = sqlite_utils.Database(db_path)
+    assert db2["_sqlite_migrations"].pks == ["migration_set", "name"]
+    db2.close()
+
+
+def test_stop_before_applied_migration_errors(two_migrations):
+    path, _ = two_migrations
+    db_path = str(path / "test.db")
+    migrations_path = str(path / "foo" / "migrations.py")
+    # Apply everything first
+    first = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        ["migrate", db_path, migrations_path, "--stop-before", "bar"],
+    )
+    assert first.exit_code == 0
+    # foo is now applied - stopping before it is an error, and bar
+    # must not be applied as a side effect
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli,
+        ["migrate", db_path, migrations_path, "--stop-before", "foo"],
+    )
+    assert result.exit_code != 0
+    assert "already been applied" in result.output
+    db = sqlite_utils.Database(db_path)
+    assert not db["bar"].exists()
+
+
+def test_list_with_legacy_class_is_read_only(tmpdir):
+    # Legacy sqlite-migrate classes create the _sqlite_migrations table
+    # from their pending()/applied() methods - --list must roll that
+    # back so it stays a read-only operation as documented
+    path = pathlib.Path(tmpdir)
+    (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8")
+    db_path = str(path / "test.db")
+    db = sqlite_utils.Database(db_path)
+    db["existing"].insert({"id": 1})
+    db.close()
+    result = CliRunner().invoke(
+        sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"]
+    )
+    assert result.exit_code == 0, result.output
+    assert "first" in result.output
+    db2 = sqlite_utils.Database(db_path)
+    assert "_sqlite_migrations" not in db2.table_names()
+    db2.close()

tests/test_column_casing.py

@@ -0,0 +1,233 @@
+"""
+SQLite treats column names as case-insensitive. These tests exercise the
+places where sqlite-utils performs Python-side lookups of column names
+provided by the caller, which should match the schema case-insensitively.
+
+https://github.com/simonw/sqlite-utils/issues/760
+"""
+
+import pytest
+
+from sqlite_utils import Database
+from sqlite_utils.db import ForeignKey
+
+
+def test_insert_populates_last_pk_case_insensitively(fresh_db):
+    books = fresh_db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.insert({"Id": 1, "Title": "One"}, pk="id")
+    assert books.last_pk == 1
+
+
+def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db):
+    books = fresh_db["books"]
+    books.create({"Author": str, "Position": int, "Title": str})
+    books.insert(
+        {"Author": "Sue", "Position": 1, "Title": "One"}, pk=("author", "position")
+    )
+    assert books.last_pk == ("Sue", 1)
+
+
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_pk_case_differs_from_schema(use_old_upsert):
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    books = db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.insert({"Id": 1, "Title": "One"})
+    books.upsert({"id": 1, "title": "Won"}, pk="id")
+    assert list(books.rows) == [{"Id": 1, "Title": "Won"}]
+    assert books.last_pk == 1
+
+
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_record_key_case_differs_from_pk(use_old_upsert):
+    # all_columns comes from the record keys, pk= from the caller
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    books = db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.upsert({"ID": 1, "Title": "One"}, pk="id")
+    assert list(books.rows) == [{"Id": 1, "Title": "One"}]
+    assert books.last_pk == 1
+
+
+def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db):
+    # pk is inferred from the existing schema as "Id", records use "id"
+    books = fresh_db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.upsert({"id": 1, "title": "One"})
+    assert list(books.rows) == [{"Id": 1, "Title": "One"}]
+    assert books.last_pk == 1
+
+
+def test_upsert_list_mode_pk_case_insensitive(fresh_db):
+    books = fresh_db["books"]
+    books.create({"Id": int, "Title": str}, pk="Id")
+    books.upsert_all([["id", "title"], [1, "One"]], pk="Id")
+    assert list(books.rows) == [{"Id": 1, "Title": "One"}]
+    assert books.last_pk == 1
+
+
+def test_lookup_pk_case_insensitive(fresh_db):
+    fresh_db["species"].create({"ID": int, "Name": str}, pk="ID")
+    fresh_db["species"].insert({"ID": 5, "Name": "Palm"})
+    fresh_db["species"].create_index(["Name"], unique=True)
+    assert fresh_db["species"].lookup({"Name": "Palm"}, pk="id") == 5
+
+
+def test_lookup_does_not_create_redundant_index(fresh_db):
+    fresh_db["species"].create({"id": int, "Name": str}, pk="id")
+    fresh_db["species"].create_index(["Name"], unique=True)
+    fresh_db["species"].lookup({"name": "Palm"})
+    assert len(fresh_db["species"].indexes) == 1
+
+
+def test_create_table_transform_same_columns_different_case(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": int})
+    fresh_db["t"].insert({"Name": "Cleo", "Age": 5})
+    fresh_db.create_table("t", {"name": str, "age": int}, transform=True)
+    # Schema casing is preserved - SQLite considers these the same columns
+    assert fresh_db["t"].columns_dict == {"Name": str, "Age": int}
+    assert list(fresh_db["t"].rows) == [{"Name": "Cleo", "Age": 5}]
+
+
+def test_create_table_transform_case_insensitive_with_changes(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": int})
+    fresh_db.create_table("t", {"name": str, "age": str, "size": int}, transform=True)
+    # age changed type, size added, Name untouched
+    assert fresh_db["t"].columns_dict == {"Name": str, "Age": str, "size": int}
+
+
+def test_transform_types_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": str})
+    fresh_db["t"].transform(types={"age": int})
+    assert fresh_db["t"].columns_dict == {"Name": str, "Age": int}
+
+
+def test_transform_rename_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Name": str})
+    fresh_db["t"].transform(rename={"name": "title"})
+    assert fresh_db["t"].columns_dict == {"title": str}
+
+
+def test_transform_drop_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": int})
+    fresh_db["t"].transform(drop=["name"])
+    assert fresh_db["t"].columns_dict == {"Age": int}
+
+
+def test_transform_not_null_and_defaults_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Name": str, "Age": int})
+    fresh_db["t"].transform(not_null={"name"}, defaults={"age": 3})
+    columns = {c.name: c for c in fresh_db["t"].columns}
+    assert columns["Name"].notnull
+    assert fresh_db["t"].default_values == {"Age": 3}
+
+
+def test_transform_pk_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Id": int, "Name": str})
+    fresh_db["t"].transform(pk="id")
+    assert fresh_db["t"].pks == ["Id"]
+    assert fresh_db["t"].columns_dict == {"Id": int, "Name": str}
+
+
+def test_transform_drop_foreign_keys_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create(
+        {"id": int, "Parent_ID": int},
+        pk="id",
+        foreign_keys=[("Parent_ID", "parent", "Id")],
+    )
+    fresh_db["child"].transform(drop_foreign_keys=["parent_id"])
+    assert fresh_db["child"].foreign_keys == []
+
+
+def test_add_foreign_key_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id")
+    fresh_db["child"].add_foreign_key("parent_id", "parent", "id")
+    fks = fresh_db["child"].foreign_keys
+    assert len(fks) == 1
+    # The foreign key should use the schema casing of the columns
+    assert fks[0].column == "Parent_ID"
+    assert fks[0].other_column == "Id"
+
+
+def test_add_foreign_keys_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id")
+    fresh_db.add_foreign_keys([("child", "parent_id", "parent", "id")])
+    fks = fresh_db["child"].foreign_keys
+    assert len(fks) == 1
+    assert fks[0].column == "Parent_ID"
+    assert fks[0].other_column == "Id"
+
+
+def test_add_foreign_key_detects_existing_case_insensitively(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create(
+        {"id": int, "Parent_ID": int},
+        pk="id",
+        foreign_keys=[("Parent_ID", "parent", "Id")],
+    )
+    # ignore=True should treat this as already existing, not add a duplicate
+    fresh_db["child"].add_foreign_key("parent_id", "parent", "id", ignore=True)
+    assert len(fresh_db["child"].foreign_keys) == 1
+
+
+def test_add_column_fk_col_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create({"id": int}, pk="id")
+    fresh_db["child"].add_column("parent_id", int, fk="parent", fk_col="id")
+    fks = fresh_db["child"].foreign_keys
+    assert len(fks) == 1
+    assert fks[0].other_column == "Id"
+
+
+def test_extract_case_insensitive(fresh_db):
+    fresh_db["trees"].insert({"id": 1, "Species": "Palm"}, pk="id")
+    fresh_db["trees"].extract("species")
+    assert fresh_db["trees"].columns_dict == {"id": int, "Species_id": int}
+    assert list(fresh_db["Species"].rows) == [{"id": 1, "Species": "Palm"}]
+
+
+def test_convert_multi_case_insensitive(fresh_db):
+    fresh_db["t"].insert({"id": 1, "Name": "Cleo"}, pk="id")
+    fresh_db["t"].convert("name", lambda v: {"upper": v.upper()}, multi=True)
+    assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "upper": "CLEO"}]
+
+
+def test_convert_output_case_insensitive(fresh_db):
+    fresh_db["t"].insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id")
+    fresh_db["t"].convert("name", lambda v: v.upper(), output="upper")
+    assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "Upper": "CLEO"}]
+
+
+def test_create_table_sql_pk_case_insensitive(fresh_db):
+    fresh_db["t"].create({"Id": int, "Name": str}, pk="id")
+    # Should not have created an extra lowercase "id" column
+    assert fresh_db["t"].columns_dict == {"Id": int, "Name": str}
+    assert fresh_db["t"].pks == ["Id"]
+
+
+def test_create_table_not_null_and_defaults_case_insensitive(fresh_db):
+    fresh_db["t"].create(
+        {"Name": str, "Age": int}, not_null={"name"}, defaults={"age": 1}
+    )
+    columns = {c.name: c for c in fresh_db["t"].columns}
+    assert columns["Name"].notnull
+    assert fresh_db["t"].default_values == {"Age": 1}
+
+
+def test_create_table_foreign_keys_case_insensitive(fresh_db):
+    fresh_db["parent"].create({"Id": int}, pk="Id")
+    fresh_db["child"].create(
+        {"id": int, "Parent_ID": int},
+        pk="id",
+        foreign_keys=[("parent_id", "parent", "id")],
+    )
+    fks = fresh_db["child"].foreign_keys
+    assert fks == [
+        ForeignKey(
+            table="child", column="Parent_ID", other_table="parent", other_column="Id"
+        )
+    ]

tests/test_constructor.py

@@ -1,5 +1,8 @@
 from sqlite_utils import Database
+from sqlite_utils.db import TransactionError
+from sqlite_utils.utils import sqlite3
 import pytest
+import sys
 
 
 def test_recursive_triggers():
@@ -10,3 +13,101 @@ def test_recursive_triggers():
 def test_recursive_triggers_off():
     db = Database(memory=True, recursive_triggers=False)
     assert not db.execute("PRAGMA recursive_triggers").fetchone()[0]
+
+
+def test_memory_name():
+    db1 = Database(memory_name="shared")
+    db2 = Database(memory_name="shared")
+    db1["dogs"].insert({"name": "Cleo"})
+    assert list(db2["dogs"].rows) == [{"name": "Cleo"}]
+
+
+def test_sqlite_version():
+    db = Database(memory=True)
+    version = db.sqlite_version
+    assert isinstance(version, tuple)
+    as_string = ".".join(map(str, version))
+    actual = next(db.query("select sqlite_version() as v"))["v"]
+    assert actual == as_string
+
+
+def test_database_context_manager(tmpdir):
+    path = str(tmpdir / "test.db")
+    with Database(path) as db:
+        db["t"].insert({"id": 1})
+        # Raw writes commit automatically too
+        db.execute("insert into t (id) values (2)")
+        # An explicitly opened transaction left uncommitted on purpose:
+        db.begin()
+        db.execute("insert into t (id) values (3)")
+    # The connection is closed...
+    with pytest.raises(sqlite3.ProgrammingError):
+        db.execute("select 1")
+    # ... and the open explicit transaction was rolled back, not committed
+    db2 = Database(path)
+    assert [r["id"] for r in db2["t"].rows] == [1, 2]
+    db2.close()
+
+
+@pytest.mark.parametrize("memory", [True, False])
+def test_database_close(tmpdir, memory):
+    if memory:
+        db = Database(memory=True)
+    else:
+        db = Database(str(tmpdir / "test.db"))
+    assert db.execute("select 1 + 1").fetchone()[0] == 2
+    db.close()
+    with pytest.raises(sqlite3.ProgrammingError):
+        db.execute("select 1 + 1")
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 12),
+    reason="sqlite3.connect(autocommit=) requires Python 3.12",
+)
+@pytest.mark.parametrize("autocommit", [True, False])
+def test_autocommit_connections_are_rejected(tmpdir, autocommit):
+    # These connection modes break commit()/rollback() in ways that
+    # silently lose data, so the constructor refuses them
+    conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=autocommit)
+    with pytest.raises(TransactionError):
+        Database(conn)
+    conn.close()
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 12),
+    reason="sqlite3.LEGACY_TRANSACTION_CONTROL requires Python 3.12",
+)
+def test_legacy_transaction_control_connection_is_accepted(tmpdir):
+    conn = sqlite3.connect(
+        str(tmpdir / "test.db"), autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL
+    )
+    db = Database(conn)
+    db["t"].insert({"id": 1}, pk="id")
+    assert [r["id"] for r in db["t"].rows] == [1]
+    db.close()
+
+
+def test_memory_attribute_for_memory_true():
+    db = Database(memory=True)
+    assert db.memory is True
+    assert db.memory_name is None
+
+
+def test_memory_attribute_for_memory_name():
+    db = Database(memory_name="shared_attr")
+    assert db.memory is True
+    assert db.memory_name == "shared_attr"
+
+
+def test_memory_attribute_for_memory_string_path():
+    db = Database(":memory:")
+    assert db.memory is True
+    assert db.memory_name is None
+
+
+def test_memory_attribute_for_file_path(tmpdir):
+    db = Database(str(tmpdir / "file.db"))
+    assert db.memory is False
+    assert db.memory_name is None

tests/test_conversions.py

@@ -1,6 +1,3 @@
-import pytest
-
-
 def test_insert_conversion(fresh_db):
     table = fresh_db["table"]
     table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"})

tests/test_convert.py

@@ -0,0 +1,159 @@
+from sqlite_utils.db import BadMultiValues
+import pytest
+
+
+@pytest.mark.parametrize(
+    "columns,fn,expected",
+    (
+        (
+            "title",
+            lambda value: value.upper(),
+            {"title": "MIXED CASE", "abstract": "Abstract"},
+        ),
+        (
+            ["title", "abstract"],
+            lambda value: value.upper(),
+            {"title": "MIXED CASE", "abstract": "ABSTRACT"},
+        ),
+        (
+            "title",
+            lambda value: {"upper": value.upper(), "lower": value.lower()},
+            {
+                "title": '{"upper": "MIXED CASE", "lower": "mixed case"}',
+                "abstract": "Abstract",
+            },
+        ),
+    ),
+)
+def test_convert(fresh_db, columns, fn, expected):
+    table = fresh_db["table"]
+    table.insert({"title": "Mixed Case", "abstract": "Abstract"})
+    table.convert(columns, fn)
+    assert list(table.rows) == [expected]
+
+
+@pytest.mark.parametrize(
+    "where,where_args", (("id > 1", None), ("id > :id", {"id": 1}), ("id > ?", [1]))
+)
+def test_convert_where(fresh_db, where, where_args):
+    table = fresh_db["table"]
+    table.insert_all(
+        [
+            {"id": 1, "title": "One"},
+            {"id": 2, "title": "Two"},
+        ],
+        pk="id",
+    )
+    table.convert(
+        "title", lambda value: value.upper(), where=where, where_args=where_args
+    )
+    assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}]
+
+
+def test_convert_handles_falsey_values(fresh_db):
+    # Falsey values like 0 should be converted (issue #527)
+    table = fresh_db["table"]
+    table.insert_all([{"x": 0}, {"x": 1}])
+    assert table.get(1)["x"] == 0
+    assert table.get(2)["x"] == 1
+    table.convert("x", lambda x: x + 1)
+    assert table.get(1)["x"] == 1
+    assert table.get(2)["x"] == 2
+
+
+@pytest.mark.parametrize(
+    "drop,expected",
+    (
+        (False, {"title": "Mixed Case", "other": "MIXED CASE"}),
+        (True, {"other": "MIXED CASE"}),
+    ),
+)
+def test_convert_output(fresh_db, drop, expected):
+    table = fresh_db["table"]
+    table.insert({"title": "Mixed Case"})
+    table.convert("title", lambda v: v.upper(), output="other", drop=drop)
+    assert list(table.rows) == [expected]
+
+
+def test_convert_output_multiple_column_error(fresh_db):
+    table = fresh_db["table"]
+    with pytest.raises(ValueError) as excinfo:
+        table.convert(["title", "other"], lambda v: v, output="out")
+        assert "output= can only be used with a single column" in str(excinfo.value)
+
+
+@pytest.mark.parametrize(
+    "type,expected",
+    (
+        (int, {"other": 123}),
+        (float, {"other": 123.0}),
+    ),
+)
+def test_convert_output_type(fresh_db, type, expected):
+    table = fresh_db["table"]
+    table.insert({"number": "123"})
+    table.convert("number", lambda v: v, output="other", output_type=type, drop=True)
+    assert list(table.rows) == [expected]
+
+
+def test_convert_multi(fresh_db):
+    table = fresh_db["table"]
+    table.insert({"title": "Mixed Case"})
+    table.convert(
+        "title",
+        lambda v: {
+            "upper": v.upper(),
+            "lower": v.lower(),
+            "both": {
+                "upper": v.upper(),
+                "lower": v.lower(),
+            },
+        },
+        multi=True,
+    )
+    assert list(table.rows) == [
+        {
+            "title": "Mixed Case",
+            "upper": "MIXED CASE",
+            "lower": "mixed case",
+            "both": '{"upper": "MIXED CASE", "lower": "mixed case"}',
+        }
+    ]
+
+
+def test_convert_multi_where(fresh_db):
+    table = fresh_db["table"]
+    table.insert_all(
+        [
+            {"id": 1, "title": "One"},
+            {"id": 2, "title": "Two"},
+        ],
+        pk="id",
+    )
+    table.convert(
+        "title",
+        lambda v: {"upper": v.upper(), "lower": v.lower()},
+        multi=True,
+        where="id > ?",
+        where_args=[1],
+    )
+    assert list(table.rows) == [
+        {"id": 1, "lower": None, "title": "One", "upper": None},
+        {"id": 2, "lower": "two", "title": "Two", "upper": "TWO"},
+    ]
+
+
+def test_convert_multi_exception(fresh_db):
+    table = fresh_db["table"]
+    table.insert({"title": "Mixed Case"})
+    with pytest.raises(BadMultiValues):
+        table.convert("title", lambda v: v.upper(), multi=True)
+
+
+def test_convert_repeated(fresh_db):
+    table = fresh_db["table"]
+    col = "num"
+    table.insert({col: 1})
+    table.convert(col, lambda x: x * 2)
+    table.convert(col, lambda _x: 0)
+    assert table.get(1) == {col: 0}

tests/test_create_view.py

@@ -15,7 +15,7 @@ def test_create_view_error(fresh_db):
 
 
 def test_create_view_only_arrow_one_param(fresh_db):
-    with pytest.raises(AssertionError):
+    with pytest.raises(ValueError):
         fresh_db.create_view("bar", "select 1 + 2", ignore=True, replace=True)
 
 

tests/test_default_value.py

@@ -0,0 +1,59 @@
+import pytest
+
+EXAMPLES = [
+    ("TEXT DEFAULT 'foo'", "'foo'", "'foo'"),
+    ("TEXT DEFAULT 'foo)'", "'foo)'", "'foo)'"),
+    ("INTEGER DEFAULT '1'", "'1'", "'1'"),
+    ("INTEGER DEFAULT 1", "1", "'1'"),
+    ("INTEGER DEFAULT (1)", "1", "'1'"),
+    # Expressions
+    (
+        "TEXT DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))",
+        "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')",
+        "(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))",
+    ),
+    # Special values
+    ("TEXT DEFAULT CURRENT_TIME", "CURRENT_TIME", "CURRENT_TIME"),
+    ("TEXT DEFAULT CURRENT_DATE", "CURRENT_DATE", "CURRENT_DATE"),
+    ("TEXT DEFAULT CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"),
+    ("TEXT DEFAULT current_timestamp", "current_timestamp", "current_timestamp"),
+    ("TEXT DEFAULT (CURRENT_TIMESTAMP)", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"),
+    # Strings
+    ("TEXT DEFAULT 'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'"),
+    ('TEXT DEFAULT "CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"'),
+    # Boolean and null keyword literals must stay unquoted
+    ("INTEGER DEFAULT TRUE", "TRUE", "TRUE"),
+    ("INTEGER DEFAULT FALSE", "FALSE", "FALSE"),
+    ("INTEGER DEFAULT true", "true", "true"),
+    ("TEXT DEFAULT NULL", "NULL", "NULL"),
+]
+
+
+@pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES)
+def test_quote_default_value(fresh_db, column_def, initial_value, expected_value):
+    fresh_db.execute("create table foo (col {})".format(column_def))
+    assert initial_value == fresh_db["foo"].columns[0].default_value
+    assert expected_value == fresh_db.quote_default_value(
+        fresh_db["foo"].columns[0].default_value
+    )
+
+
+def test_insert_empty_record_uses_default_values(fresh_db):
+    fresh_db.execute("""
+        CREATE TABLE has_defaults (
+            id INTEGER PRIMARY KEY,
+            name TEXT,
+            timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
+            is_active INTEGER NOT NULL DEFAULT 1
+        )
+        """)
+
+    table = fresh_db["has_defaults"]
+    table.insert({})
+
+    rows = list(table.rows)
+    assert len(rows) == 1
+    assert rows[0]["id"] == 1
+    assert rows[0]["name"] is None
+    assert rows[0]["timestamp"] is not None
+    assert rows[0]["is_active"] == 1

tests/test_delete.py

@@ -1,3 +1,6 @@
+import sqlite_utils
+
+
 def test_delete_rowid_table(fresh_db):
     table = fresh_db["table"]
     table.insert({"foo": 1}).last_pk
@@ -18,15 +21,44 @@ def test_delete_where(fresh_db):
     table = fresh_db["table"]
     for i in range(1, 11):
         table.insert({"id": i}, pk="id")
-    assert 10 == table.count
+    assert table.count == 10
     table.delete_where("id > ?", [5])
-    assert 5 == table.count
+    assert table.count == 5
 
 
 def test_delete_where_all(fresh_db):
     table = fresh_db["table"]
     for i in range(1, 11):
         table.insert({"id": i}, pk="id")
-    assert 10 == table.count
+    assert table.count == 10
     table.delete_where()
-    assert 0 == table.count
+    assert table.count == 0
+
+
+def test_delete_where_commits(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = sqlite_utils.Database(path)
+    db["table"].insert_all([{"id": i} for i in range(5)], pk="id")
+    db["table"].delete_where("id > ?", [2])
+    # The connection must not be left inside an open transaction,
+    # otherwise subsequent atomic() blocks never commit either
+    assert not db.conn.in_transaction
+    db["table"].insert({"id": 100})
+    db.close()
+    db2 = sqlite_utils.Database(path)
+    assert [r["id"] for r in db2["table"].rows] == [0, 1, 2, 100]
+    db2.close()
+
+
+def test_delete_where_analyze(fresh_db):
+    table = fresh_db["table"]
+    table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id")
+    table.create_index(["i"], analyze=True)
+    assert "sqlite_stat1" in fresh_db.table_names()
+    assert list(fresh_db["sqlite_stat1"].rows) == [
+        {"tbl": "table", "idx": "idx_table_i", "stat": "10 1"}
+    ]
+    table.delete_where("id > ?", [5], analyze=True)
+    assert list(fresh_db["sqlite_stat1"].rows) == [
+        {"tbl": "table", "idx": "idx_table_i", "stat": "6 1"}
+    ]

tests/test_docs.py

@@ -1,15 +1,19 @@
-from sqlite_utils import cli
+from click.testing import CliRunner
+from sqlite_utils import cli, recipes
 from pathlib import Path
 import pytest
 import re
 
 docs_path = Path(__file__).parent.parent / "docs"
-commands_re = re.compile(r"(?:\$ |    )sqlite-utils (\S+) ")
+commands_re = re.compile(r"(?:\$ |    )sqlite-utils (\S+)")
+recipes_re = re.compile(r"r\.(\w+)\(")
 
 
 @pytest.fixture(scope="session")
 def documented_commands():
-    rst = (docs_path / "cli.rst").read_text()
+    rst = ""
+    for doc in ("cli.rst", "plugins.rst"):
+        rst += (docs_path / doc).read_text()
     return {
         command
         for command in commands_re.findall(rst)
@@ -17,11 +21,42 @@ def documented_commands():
     }
 
 
+@pytest.fixture(scope="session")
+def documented_recipes():
+    rst = (docs_path / "cli.rst").read_text()
+    return set(recipes_re.findall(rst))
+
+
 @pytest.mark.parametrize("command", cli.cli.commands.keys())
 def test_commands_are_documented(documented_commands, command):
     assert command in documented_commands
 
 
 @pytest.mark.parametrize("command", cli.cli.commands.values())
-def test_commands_have_docstrings(command):
+def test_commands_have_help(command):
-    assert command.__doc__, "{} is missing a docstring".format(command)
+    assert command.help, "{} is missing its help".format(command)
+
+
+def test_convert_help():
+    result = CliRunner().invoke(cli.cli, ["convert", "--help"])
+    assert result.exit_code == 0
+    for expected in (
+        "r.jsonsplit(value:",
+        "r.parsedate(value:",
+        "r.parsedatetime(value:",
+    ):
+        assert expected in result.output
+
+
+@pytest.mark.parametrize(
+    "recipe",
+    [
+        n
+        for n in dir(recipes)
+        if not n.startswith("_")
+        and n not in ("json", "parser", "Callable", "Optional")
+        and callable(getattr(recipes, n))
+    ],
+)
+def test_recipes_are_documented(documented_recipes, recipe):
+    assert recipe in documented_recipes

tests/test_duplicate.py

@@ -0,0 +1,41 @@
+from sqlite_utils.db import NoTable
+import datetime
+import pytest
+
+
+def test_duplicate(fresh_db):
+    # Create table using native Sqlite statement:
+    fresh_db.execute("""CREATE TABLE "table1" (
+    "text_col" TEXT,
+    "real_col" REAL,
+    "int_col" INTEGER,
+    "bool_col" INTEGER,
+    "datetime_col" TEXT)""")
+    # Insert one row of mock data:
+    dt = datetime.datetime.now()
+    data = {
+        "text_col": "Cleo",
+        "real_col": 3.14,
+        "int_col": -255,
+        "bool_col": True,
+        "datetime_col": str(dt),
+    }
+    table1 = fresh_db["table1"]
+    row_id = table1.insert(data).last_rowid
+    # Duplicate table:
+    table2 = table1.duplicate("table2")
+    # Ensure data integrity:
+    assert data == table2.get(row_id)
+    # Ensure schema integrity:
+    assert [
+        {"name": "text_col", "type": "TEXT"},
+        {"name": "real_col", "type": "REAL"},
+        {"name": "int_col", "type": "INT"},
+        {"name": "bool_col", "type": "INT"},
+        {"name": "datetime_col", "type": "TEXT"},
+    ] == [{"name": col.name, "type": col.type} for col in table2.columns]
+
+
+def test_duplicate_fails_if_table_does_not_exist(fresh_db):
+    with pytest.raises(NoTable):
+        fresh_db["not_a_table"].duplicate("duplicated")

tests/test_enable_counts.py

@@ -14,8 +14,31 @@ def test_enable_counts_specific_table(fresh_db):
     # Now enable counts
     foo.enable_counts()
     assert foo.triggers_dict == {
-        "foo_counts_insert": "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\nBEGIN\n    INSERT OR REPLACE INTO [_counts]\n    VALUES (\n        'foo',\n        COALESCE(\n            (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n        0\n        ) + 1\n    );\nEND",
+        "foo_counts_insert": (
-        "foo_counts_delete": "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\nBEGIN\n    INSERT OR REPLACE INTO [_counts]\n    VALUES (\n        'foo',\n        COALESCE(\n            (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n        0\n        ) - 1\n    );\nEND",
+            'CREATE TRIGGER "foo_counts_insert" AFTER INSERT ON "foo"\n'
+            "BEGIN\n"
+            '    INSERT OR REPLACE INTO "_counts"\n'
+            "    VALUES (\n        'foo',\n"
+            "        COALESCE(\n"
+            '            (SELECT count FROM "_counts" WHERE "table" = \'foo\'),\n'
+            "        0\n"
+            "        ) + 1\n"
+            "    );\n"
+            "END"
+        ),
+        "foo_counts_delete": (
+            'CREATE TRIGGER "foo_counts_delete" AFTER DELETE ON "foo"\n'
+            "BEGIN\n"
+            '    INSERT OR REPLACE INTO "_counts"\n'
+            "    VALUES (\n"
+            "        'foo',\n"
+            "        COALESCE(\n"
+            '            (SELECT count FROM "_counts" WHERE "table" = \'foo\'),\n'
+            "        0\n"
+            "        ) - 1\n"
+            "    );\n"
+            "END"
+        ),
     }
     assert fresh_db.table_names() == ["foo", "_counts"]
     assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}]
@@ -106,19 +129,19 @@ def test_uses_counts_after_enable_counts(counts_db_path):
     db = Database(counts_db_path)
     logged = []
     with db.tracer(lambda sql, parameters: logged.append((sql, parameters))):
-        assert db["foo"].count == 1
+        assert db.table("foo").count == 1
         assert logged == [
             ("select name from sqlite_master where type = 'view'", None),
-            ("select count(*) from [foo]", None),
+            ('select count(*) from "foo"', []),
         ]
         logged.clear()
         assert not db.use_counts_table
         db.enable_counts()
         assert db.use_counts_table
-        assert db["foo"].count == 1
+        assert db.table("foo").count == 1
     assert logged == [
         (
-            "CREATE TABLE IF NOT EXISTS [_counts](\n   [table] TEXT PRIMARY KEY,\n   count INTEGER DEFAULT 0\n);",
+            'CREATE TABLE IF NOT EXISTS "_counts"(\n   "table" TEXT PRIMARY KEY,\n   count INTEGER DEFAULT 0\n);',
             None,
         ),
         ("select name from sqlite_master where type = 'table'", None),
@@ -134,7 +157,7 @@ def test_uses_counts_after_enable_counts(counts_db_path):
         ("SELECT quote(:value)", {"value": "baz"}),
         ("select sql from sqlite_master where name = ?", ("_counts",)),
         ("select name from sqlite_master where type = 'view'", None),
-        ("select [table], count from _counts where [table] in (?)", ["foo"]),
+        ('select "table", count from _counts where "table" in (?)', ["foo"]),
     ]
 
 

tests/test_extract.py

@@ -1,4 +1,4 @@
-from sqlite_utils.db import Index, InvalidColumns
+from sqlite_utils.db import InvalidColumns
 import itertools
 import pytest
 
@@ -24,19 +24,16 @@ def test_extract_single_column(fresh_db, table, fk_column):
     fresh_db["tree"].extract("species", table=table, fk_column=fk_column)
     assert fresh_db["tree"].schema == (
         'CREATE TABLE "tree" (\n'
-        "   [id] INTEGER PRIMARY KEY,\n"
+        '   "id" INTEGER PRIMARY KEY,\n'
-        "   [name] TEXT,\n"
+        '   "name" TEXT,\n'
-        "   [{}] INTEGER,\n".format(expected_fk)
+        '   "{}" INTEGER REFERENCES "{}"("id"),\n'.format(expected_fk, expected_table)
-        + "   [end] INTEGER,\n"
+        + '   "end" INTEGER\n'
-        + "   FOREIGN KEY([{}]) REFERENCES [{}]([id])\n".format(
-            expected_fk, expected_table
-        )
         + ")"
     )
     assert fresh_db[expected_table].schema == (
-        "CREATE TABLE [{}] (\n".format(expected_table)
+        'CREATE TABLE "{}" (\n'.format(expected_table)
-        + "   [id] INTEGER PRIMARY KEY,\n"
+        + '   "id" INTEGER PRIMARY KEY,\n'
-        "   [species] TEXT\n"
+        '   "species" TEXT\n'
         ")"
     )
     assert list(fresh_db[expected_table].rows) == [
@@ -74,17 +71,16 @@ def test_extract_multiple_columns_with_rename(fresh_db):
     )
     assert fresh_db["tree"].schema == (
         'CREATE TABLE "tree" (\n'
-        "   [id] INTEGER PRIMARY KEY,\n"
+        '   "id" INTEGER PRIMARY KEY,\n'
-        "   [name] TEXT,\n"
+        '   "name" TEXT,\n'
-        "   [common_name_latin_name_id] INTEGER,\n"
+        '   "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n'
-        "   FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n"
         ")"
     )
     assert fresh_db["common_name_latin_name"].schema == (
-        "CREATE TABLE [common_name_latin_name] (\n"
+        'CREATE TABLE "common_name_latin_name" (\n'
-        "   [id] INTEGER PRIMARY KEY,\n"
+        '   "id" INTEGER PRIMARY KEY,\n'
-        "   [name] TEXT,\n"
+        '   "name" TEXT,\n'
-        "   [latin_name] TEXT\n"
+        '   "latin_name" TEXT\n'
         ")"
     )
     assert list(fresh_db["common_name_latin_name"].rows) == [
@@ -126,14 +122,11 @@ def test_extract_rowid_table(fresh_db):
     fresh_db["tree"].extract(["common_name", "latin_name"])
     assert fresh_db["tree"].schema == (
         'CREATE TABLE "tree" (\n'
-        "   [name] TEXT,\n"
+        '   "name" TEXT,\n'
-        "   [common_name_latin_name_id] INTEGER,\n"
+        '   "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n'
-        "   FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n"
         ")"
     )
-    assert (
+    assert fresh_db.execute("""
-        fresh_db.execute(
-            """
         select
             tree.name,
             common_name_latin_name.common_name,
@@ -141,10 +134,7 @@ def test_extract_rowid_table(fresh_db):
         from tree
             join common_name_latin_name
             on tree.common_name_latin_name_id = common_name_latin_name.id
-    """
+    """).fetchall() == [("Tree 1", "Palm", "Arecaceae")]
-        ).fetchall()
-        == [("Tree 1", "Palm", "Arecaceae")]
-    )
 
 
 def test_reuse_lookup_table(fresh_db):
@@ -157,17 +147,15 @@ def test_reuse_lookup_table(fresh_db):
     fresh_db["individuals"].extract("species", rename={"species": "name"})
     assert fresh_db["sightings"].schema == (
         'CREATE TABLE "sightings" (\n'
-        "   [id] INTEGER PRIMARY KEY,\n"
+        '   "id" INTEGER PRIMARY KEY,\n'
-        "   [species_id] INTEGER,\n"
+        '   "species_id" INTEGER REFERENCES "species"("id")\n'
-        "   FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
         ")"
     )
     assert fresh_db["individuals"].schema == (
         'CREATE TABLE "individuals" (\n'
-        "   [id] INTEGER PRIMARY KEY,\n"
+        '   "id" INTEGER PRIMARY KEY,\n'
-        "   [name] TEXT,\n"
+        '   "name" TEXT,\n'
-        "   [species_id] INTEGER,\n"
+        '   "species_id" INTEGER REFERENCES "species"("id")\n'
-        "   FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
         ")"
     )
     assert list(fresh_db["species"].rows) == [
@@ -186,3 +174,131 @@ def test_extract_error_on_incompatible_existing_lookup_table(fresh_db):
     fresh_db["species2"].insert({"id": 1, "common_name": 3.5})
     with pytest.raises(InvalidColumns):
         fresh_db["tree"].extract("common_name", table="species2")
+
+
+def test_extract_works_with_null_values(fresh_db):
+    fresh_db["listens"].insert_all(
+        [
+            {"id": 1, "track_title": "foo", "album_title": "bar"},
+            {"id": 2, "track_title": "baz", "album_title": None},
+        ],
+        pk="id",
+    )
+    fresh_db["listens"].extract(
+        columns=["album_title"], table="albums", fk_column="album_id"
+    )
+    assert list(fresh_db["listens"].rows) == [
+        {"id": 1, "track_title": "foo", "album_id": 1},
+        {"id": 2, "track_title": "baz", "album_id": None},
+    ]
+    assert list(fresh_db["albums"].rows) == [
+        {"id": 1, "album_title": "bar"},
+    ]
+
+
+def test_extract_null_values_single_column(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/186
+    fresh_db["species"].insert({"id": 1, "species": "Wolf"}, pk="id")
+    fresh_db["individuals"].insert_all(
+        [
+            {"id": 10, "name": "Terriana", "species": "Fox"},
+            {"id": 11, "name": "Spenidorm", "species": None},
+            {"id": 12, "name": "Grantheim", "species": "Wolf"},
+            {"id": 13, "name": "Turnutopia", "species": None},
+            {"id": 14, "name": "Wargal", "species": "Wolf"},
+        ],
+        pk="id",
+    )
+    fresh_db["individuals"].extract("species")
+    # No null row should have been added to species
+    assert list(fresh_db["species"].rows) == [
+        {"id": 1, "species": "Wolf"},
+        {"id": 2, "species": "Fox"},
+    ]
+    assert list(fresh_db["individuals"].rows) == [
+        {"id": 10, "name": "Terriana", "species_id": 2},
+        {"id": 11, "name": "Spenidorm", "species_id": None},
+        {"id": 12, "name": "Grantheim", "species_id": 1},
+        {"id": 13, "name": "Turnutopia", "species_id": None},
+        {"id": 14, "name": "Wargal", "species_id": 1},
+    ]
+
+
+def test_extract_null_values_multiple_columns(fresh_db):
+    # A row should be extracted if at least one column is not null -
+    # only rows where ALL extracted columns are null are left alone
+    fresh_db["circulation"].insert_all(
+        [
+            {"id": 1, "title": "title one", "creator": "creator one", "year": 2018},
+            {"id": 2, "title": "title two", "creator": None, "year": 2019},
+            {"id": 3, "title": None, "creator": None, "year": 2020},
+            {"id": 4, "title": None, "creator": None, "year": 2021},
+        ],
+        pk="id",
+    )
+    fresh_db["circulation"].extract(
+        ["title", "creator"], table="books", fk_column="book_id"
+    )
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "title one", "creator": "creator one"},
+        {"id": 2, "title": "title two", "creator": None},
+    ]
+    assert list(fresh_db["circulation"].rows) == [
+        {"id": 1, "book_id": 1, "year": 2018},
+        {"id": 2, "book_id": 2, "year": 2019},
+        {"id": 3, "book_id": None, "year": 2020},
+        {"id": 4, "book_id": None, "year": 2021},
+    ]
+
+
+def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db):
+    # Even if the lookup table already contains an all-null row, rows where
+    # every extracted column is null should keep a null foreign key
+    fresh_db["species"].insert({"id": 1, "species": None}, pk="id")
+    fresh_db["individuals"].insert_all(
+        [
+            {"id": 10, "name": "Terriana", "species": "Fox"},
+            {"id": 11, "name": "Spenidorm", "species": None},
+        ],
+        pk="id",
+    )
+    fresh_db["individuals"].extract("species")
+    assert list(fresh_db["species"].rows) == [
+        {"id": 1, "species": None},
+        {"id": 2, "species": "Fox"},
+    ]
+    assert list(fresh_db["individuals"].rows) == [
+        {"id": 10, "name": "Terriana", "species_id": 2},
+        {"id": 11, "name": "Spenidorm", "species_id": None},
+    ]
+
+
+def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db):
+    # Unique indexes treat NULLs as distinct, so INSERT OR IGNORE alone
+    # cannot dedupe NULL-containing rows against the existing lookup
+    # table - extracting a second table into the same lookup previously
+    # inserted duplicate rows that nothing pointed to
+    fresh_db["t1"].insert_all(
+        [
+            {"id": 1, "species": None, "common": "X"},
+            {"id": 2, "species": "Oak", "common": "Oak"},
+        ],
+        pk="id",
+    )
+    fresh_db["t2"].insert_all([{"id": 1, "species": None, "common": "X"}], pk="id")
+    fresh_db["t1"].extract(["species", "common"], table="lk")
+    fresh_db["t2"].extract(["species", "common"], table="lk")
+    assert fresh_db["lk"].count == 2
+    # Both tables point at the same lookup row
+    t1_fk = fresh_db.execute("select lk_id from t1 where id = 1").fetchone()[0]
+    t2_fk = fresh_db.execute("select lk_id from t2 where id = 1").fetchone()[0]
+    assert t1_fk == t2_fk
+
+
+def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db):
+    # Non-NULL rows were already deduped by the unique index - keep it so
+    fresh_db["t1"].insert_all([{"id": 1, "species": "Oak"}], pk="id")
+    fresh_db["t2"].insert_all([{"id": 1, "species": "Oak"}], pk="id")
+    fresh_db["t1"].extract(["species"], table="lk")
+    fresh_db["t2"].extract(["species"], table="lk")
+    assert fresh_db["lk"].count == 1

tests/test_extracts.py

@@ -1,4 +1,4 @@
-from sqlite_utils.db import Index, ForeignKey
+from sqlite_utils.db import Index
 import pytest
 
 
@@ -25,18 +25,18 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory):
             {"id": 2, "species_id": "Oak"},
             {"id": 3, "species_id": "Palm"},
         ],
-        **insert_kwargs
+        **insert_kwargs,
     )
     # Should now have two tables: Trees and Species
     assert {expected_table, "Trees"} == set(fresh_db.table_names())
     assert (
-        "CREATE TABLE [{}] (\n   [id] INTEGER PRIMARY KEY,\n   [value] TEXT\n)".format(
+        'CREATE TABLE "{}" (\n   "id" INTEGER PRIMARY KEY,\n   "value" TEXT\n)'.format(
             expected_table
         )
         == fresh_db[expected_table].schema
     )
     assert (
-        "CREATE TABLE [Trees] (\n   [id] INTEGER,\n   [species_id] INTEGER REFERENCES [{}]([id])\n)".format(
+        'CREATE TABLE "Trees" (\n   "id" INTEGER,\n   "species_id" INTEGER REFERENCES "{}"("id")\n)'.format(
             expected_table
         )
         == fresh_db["Trees"].schema
@@ -67,3 +67,51 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory):
         {"id": 2, "species_id": 1},
         {"id": 3, "species_id": 2},
     ] == list(fresh_db["Trees"].rows)
+
+
+def test_extracts_null_values(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/186
+    # Null values should stay null, not be extracted into the lookup table
+    fresh_db["Trees"].insert_all(
+        [
+            {"id": 1, "species_id": "Oak"},
+            {"id": 2, "species_id": None},
+            {"id": 3, "species_id": "Palm"},
+            {"id": 4, "species_id": None},
+        ],
+        extracts={"species_id": "Species"},
+    )
+    assert list(fresh_db["Species"].rows) == [
+        {"id": 1, "value": "Oak"},
+        {"id": 2, "value": "Palm"},
+    ]
+    assert list(fresh_db["Trees"].rows) == [
+        {"id": 1, "species_id": 1},
+        {"id": 2, "species_id": None},
+        {"id": 3, "species_id": 2},
+        {"id": 4, "species_id": None},
+    ]
+
+
+def test_extracts_null_values_list_mode(fresh_db):
+    # Same as test_extracts_null_values but for list-based records
+    fresh_db["Trees"].insert_all(
+        [
+            ["id", "species_id"],
+            [1, "Oak"],
+            [2, None],
+            [3, "Palm"],
+            [4, None],
+        ],
+        extracts={"species_id": "Species"},
+    )
+    assert list(fresh_db["Species"].rows) == [
+        {"id": 1, "value": "Oak"},
+        {"id": 2, "value": "Palm"},
+    ]
+    assert list(fresh_db["Trees"].rows) == [
+        {"id": 1, "species_id": 1},
+        {"id": 2, "species_id": None},
+        {"id": 3, "species_id": 2},
+        {"id": 4, "species_id": None},
+    ]

tests/test_foreign_keys.py

@@ -0,0 +1,691 @@
+"""Tests for compound (multi-column) foreign keys - issue #594."""
+
+import pytest
+from sqlite_utils import Database
+from sqlite_utils.db import AlterError, ForeignKey
+from sqlite_utils.utils import sqlite3
+
+COMPOUND_SCHEMA = """
+CREATE TABLE departments (
+    campus_name TEXT NOT NULL,
+    dept_code TEXT NOT NULL,
+    dept_name TEXT,
+    PRIMARY KEY (campus_name, dept_code)
+);
+CREATE TABLE courses (
+    course_code TEXT PRIMARY KEY,
+    course_name TEXT,
+    campus_name TEXT NOT NULL,
+    dept_code TEXT NOT NULL,
+    FOREIGN KEY (campus_name, dept_code)
+    REFERENCES departments(campus_name, dept_code)
+);
+"""
+
+
+@pytest.fixture
+def compound_db():
+    db = Database(memory=True)
+    db.executescript(COMPOUND_SCHEMA)
+    return db
+
+
+def test_compound_foreign_key(compound_db):
+    fks = compound_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.table == "courses"
+    assert fk.other_table == "departments"
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_columns == ("campus_name", "dept_code")
+    # Scalar column/other_column can't sensibly hold a compound key
+    assert fk.column is None
+    assert fk.other_column is None
+
+
+def test_single_foreign_key_gets_columns_fields(fresh_db):
+    fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
+    fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1})
+    fresh_db["books"].add_foreign_key("author_id", "authors", "id")
+    fk = fresh_db["books"].foreign_keys[0]
+    assert fk.is_compound is False
+    assert fk.column == "author_id"
+    assert fk.other_column == "id"
+    assert fk.columns == ("author_id",)
+    assert fk.other_columns == ("id",)
+
+
+def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db):
+    # Clean break in 4.0: ForeignKey is a dataclass, not a namedtuple, so the
+    # old tuple unpacking and indexing patterns now fail hard.
+    fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
+    fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1})
+    fresh_db["books"].add_foreign_key("author_id", "authors", "id")
+    fk = fresh_db["books"].foreign_keys[0]
+    with pytest.raises(TypeError):
+        table, column, other_table, other_column = fk
+    with pytest.raises(TypeError):
+        fk[0]
+
+
+def test_foreign_keys_are_sortable(fresh_db):
+    fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
+    fresh_db["categories"].insert({"id": 1, "name": "Wildlife"}, pk="id")
+    fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1, "category_id": 1})
+    fresh_db.add_foreign_keys(
+        [
+            ("books", "author_id", "authors", "id"),
+            ("books", "category_id", "categories", "id"),
+        ]
+    )
+    fks = sorted(fresh_db["books"].foreign_keys)
+    assert fks[0].column == "author_id"
+    assert fks[1].column == "category_id"
+
+
+def test_mixed_compound_and_single_foreign_keys_are_sortable():
+    # compound FKs have column=None, which must not break sorting
+    # against single-column FKs (None < str raises TypeError)
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE departments (
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        PRIMARY KEY (campus_name, dept_code)
+    );
+    CREATE TABLE accreditations (id INTEGER PRIMARY KEY);
+    CREATE TABLE courses (
+        course_code TEXT PRIMARY KEY,
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        accreditation_id INTEGER REFERENCES accreditations(id),
+        FOREIGN KEY (campus_name, dept_code)
+        REFERENCES departments(campus_name, dept_code)
+    );
+    """)
+    fks = db["courses"].foreign_keys
+    assert len(fks) == 2
+    assert {fk.is_compound for fk in fks} == {True, False}
+    fks_sorted = sorted(fks)
+    assert fks_sorted[0].other_table == "accreditations"
+    assert fks_sorted[1].other_table == "departments"
+
+
+@pytest.fixture
+def departments_db():
+    db = Database(memory=True)
+    db.create_table(
+        "departments",
+        {"campus_name": str, "dept_code": str, "dept_name": str},
+        pk=("campus_name", "dept_code"),
+    )
+    return db
+
+
+EXPECTED_COURSES_SCHEMA = (
+    'CREATE TABLE "courses" (\n'
+    '   "course_code" TEXT PRIMARY KEY,\n'
+    '   "campus_name" TEXT,\n'
+    '   "dept_code" TEXT,\n'
+    '   FOREIGN KEY ("campus_name", "dept_code") '
+    'REFERENCES "departments"("campus_name", "dept_code")\n'
+    ")"
+)
+
+
+@pytest.mark.parametrize(
+    "foreign_keys",
+    (
+        [
+            ForeignKey(
+                table="courses",
+                column=None,
+                other_table="departments",
+                other_column=None,
+                columns=("campus_name", "dept_code"),
+                other_columns=("campus_name", "dept_code"),
+                is_compound=True,
+            )
+        ],
+        [(("campus_name", "dept_code"), "departments", ("campus_name", "dept_code"))],
+        # Two-item form guesses the other table's primary key:
+        [(("campus_name", "dept_code"), "departments")],
+        # Lists work too, though tuples are the documented form:
+        [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])],
+    ),
+)
+def test_create_table_with_compound_foreign_key(departments_db, foreign_keys):
+    departments_db.create_table(
+        "courses",
+        {"course_code": str, "campus_name": str, "dept_code": str},
+        pk="course_code",
+        foreign_keys=foreign_keys,
+    )
+    assert departments_db["courses"].schema == EXPECTED_COURSES_SCHEMA
+    fks = departments_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_table == "departments"
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_create_table_compound_foreign_key_enforced(departments_db):
+    departments_db.execute("PRAGMA foreign_keys = ON")
+    departments_db.create_table(
+        "courses",
+        {"course_code": str, "campus_name": str, "dept_code": str},
+        pk="course_code",
+        foreign_keys=[(("campus_name", "dept_code"), "departments")],
+    )
+    departments_db["departments"].insert(
+        {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"}
+    )
+    departments_db["courses"].insert(
+        {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"}
+    )
+    with pytest.raises(sqlite3.IntegrityError):
+        departments_db.execute(
+            "insert into courses (course_code, campus_name, dept_code) "
+            "values ('X1', 'Nowhere', 'NOPE')"
+        )
+
+
+def test_create_table_compound_foreign_key_missing_other_column(departments_db):
+    with pytest.raises(AlterError):
+        departments_db.create_table(
+            "courses",
+            {"course_code": str, "campus_name": str, "dept_code": str},
+            pk="course_code",
+            foreign_keys=[
+                (("campus_name", "dept_code"), "departments", ("campus_name", "nope"))
+            ],
+        )
+
+
+def test_transform_preserves_compound_foreign_key(compound_db):
+    compound_db["courses"].transform(rename={"course_name": "title"})
+    fks = compound_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_table == "departments"
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_transform_rename_member_column_updates_compound_foreign_key(compound_db):
+    compound_db["courses"].transform(rename={"campus_name": "campus"})
+    fks = compound_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus", "dept_code")
+    # Referenced columns in the other table are unchanged
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_transform_drop_member_column_drops_compound_foreign_key(compound_db):
+    # Matches single-column behavior: dropping the column silently
+    # drops the foreign key that used it
+    compound_db["courses"].transform(drop={"dept_code"})
+    assert compound_db["courses"].foreign_keys == []
+    assert "FOREIGN KEY" not in compound_db["courses"].schema
+
+
+@pytest.mark.parametrize(
+    "drop_foreign_keys",
+    (
+        # A bare column name matches any foreign key it participates in:
+        ["campus_name"],
+        # A tuple must match the full compound key:
+        [("campus_name", "dept_code")],
+    ),
+)
+def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys):
+    compound_db["courses"].transform(drop_foreign_keys=drop_foreign_keys)
+    assert compound_db["courses"].foreign_keys == []
+    # The columns themselves survive
+    assert {"campus_name", "dept_code"} <= set(
+        compound_db["courses"].columns_dict.keys()
+    )
+
+
+@pytest.fixture
+def courses_db(departments_db):
+    departments_db.create_table(
+        "courses",
+        {"course_code": str, "campus_name": str, "dept_code": str},
+        pk="course_code",
+    )
+    return departments_db
+
+
+def test_add_compound_foreign_key(courses_db):
+    t = courses_db["courses"].add_foreign_key(
+        ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")
+    )
+    # Returns self
+    assert t.name == "courses"
+    fks = courses_db["courses"].foreign_keys
+    assert len(fks) == 1
+    fk = fks[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_table == "departments"
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_add_compound_foreign_key_guesses_other_columns(courses_db):
+    # Lists work here too, though tuples are the documented form
+    courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments")
+    fk = courses_db["courses"].foreign_keys[0]
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_add_compound_foreign_key_error_if_already_exists(courses_db):
+    courses_db["courses"].add_foreign_key(("campus_name", "dept_code"), "departments")
+    with pytest.raises(AlterError) as ex:
+        courses_db["courses"].add_foreign_key(
+            ("campus_name", "dept_code"), "departments"
+        )
+    assert "already exists" in ex.value.args[0]
+    # ignore=True should not raise
+    courses_db["courses"].add_foreign_key(
+        ("campus_name", "dept_code"), "departments", ignore=True
+    )
+
+
+def test_add_compound_foreign_key_error_if_column_missing(courses_db):
+    with pytest.raises(AlterError):
+        courses_db["courses"].add_foreign_key(("campus_name", "nope"), "departments")
+
+
+def test_db_add_foreign_keys_compound(courses_db):
+    courses_db.add_foreign_keys(
+        [
+            (
+                "courses",
+                ("campus_name", "dept_code"),
+                "departments",
+                ("campus_name", "dept_code"),
+            )
+        ]
+    )
+    fk = courses_db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.columns == ("campus_name", "dept_code")
+
+
+def test_index_foreign_keys_compound_creates_composite_index(compound_db):
+    compound_db.index_foreign_keys()
+    index_columns = [i.columns for i in compound_db["courses"].indexes]
+    assert ["campus_name", "dept_code"] in index_columns
+    # No separate single-column indexes for the members
+    assert ["campus_name"] not in index_columns
+    assert ["dept_code"] not in index_columns
+
+
+def test_foreign_key_captures_on_delete_and_on_update():
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE authors (id INTEGER PRIMARY KEY);
+    CREATE TABLE books (
+        id INTEGER PRIMARY KEY,
+        author_id INTEGER REFERENCES authors(id)
+            ON DELETE CASCADE ON UPDATE RESTRICT
+    );
+    """)
+    fk = db["books"].foreign_keys[0]
+    assert fk.on_delete == "CASCADE"
+    assert fk.on_update == "RESTRICT"
+
+
+def test_foreign_key_on_delete_defaults_to_no_action(fresh_db):
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
+    fresh_db["books"].add_foreign_key("author_id", "authors", "id")
+    fk = fresh_db["books"].foreign_keys[0]
+    assert fk.on_delete == "NO ACTION"
+    assert fk.on_update == "NO ACTION"
+
+
+def test_create_table_foreign_key_with_on_delete(fresh_db):
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db.create_table(
+        "books",
+        {"id": int, "author_id": int},
+        pk="id",
+        foreign_keys=[
+            ForeignKey(
+                table="books",
+                column="author_id",
+                other_table="authors",
+                other_column="id",
+                on_delete="CASCADE",
+            )
+        ],
+    )
+    assert "ON DELETE CASCADE" in fresh_db["books"].schema
+    assert fresh_db["books"].foreign_keys[0].on_delete == "CASCADE"
+
+
+def test_transform_preserves_on_delete_cascade():
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE authors (id INTEGER PRIMARY KEY);
+    CREATE TABLE books (
+        id INTEGER PRIMARY KEY,
+        title TEXT,
+        author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE
+    );
+    """)
+    db["books"].transform(rename={"title": "book_title"})
+    fk = db["books"].foreign_keys[0]
+    assert fk.on_delete == "CASCADE"
+    assert fk.on_update == "NO ACTION"
+    assert "ON DELETE CASCADE" in db["books"].schema
+
+
+def test_transform_preserves_compound_foreign_key_on_delete():
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE departments (
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        PRIMARY KEY (campus_name, dept_code)
+    );
+    CREATE TABLE courses (
+        course_code TEXT PRIMARY KEY,
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        FOREIGN KEY (campus_name, dept_code)
+        REFERENCES departments(campus_name, dept_code) ON DELETE CASCADE
+    );
+    """)
+    db["courses"].transform(rename={"course_code": "code"})
+    fk = db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.on_delete == "CASCADE"
+    assert "ON DELETE CASCADE" in db["courses"].schema
+
+
+def test_implicit_primary_key_reference_is_resolved():
+    # REFERENCES authors (no column) has "to" of None in the pragma -
+    # it should be resolved to the primary key of the other table
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE authors (author_id INTEGER PRIMARY KEY);
+    CREATE TABLE books (
+        id INTEGER PRIMARY KEY,
+        author_id INTEGER REFERENCES authors
+    );
+    """)
+    fk = db["books"].foreign_keys[0]
+    assert fk.is_compound is False
+    assert fk.other_column == "author_id"
+    assert fk.other_columns == ("author_id",)
+
+
+def test_implicit_compound_primary_key_reference_is_resolved():
+    db = Database(memory=True)
+    db.executescript("""
+    CREATE TABLE departments (
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        PRIMARY KEY (campus_name, dept_code)
+    );
+    CREATE TABLE courses (
+        course_code TEXT PRIMARY KEY,
+        campus_name TEXT NOT NULL,
+        dept_code TEXT NOT NULL,
+        FOREIGN KEY (campus_name, dept_code) REFERENCES departments
+    );
+    """)
+    fk = db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_foreign_key_normalizes_list_columns_to_tuples():
+    # Compound columns passed as lists are normalized to tuples, so they
+    # compare equal to introspected ForeignKeys
+    fk = ForeignKey(
+        table="courses",
+        column=None,
+        other_table="departments",
+        other_column=None,
+        columns=["campus_name", "dept_code"],
+        other_columns=["campus_name", "dept_code"],
+        is_compound=True,
+    )
+    assert fk.columns == ("campus_name", "dept_code")
+    assert fk.other_columns == ("campus_name", "dept_code")
+
+
+def test_add_foreign_keys_preserves_actions(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/594 review finding:
+    # ForeignKey objects passed to db.add_foreign_keys() were flattened
+    # to plain tuples, losing on_delete/on_update
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
+    fresh_db.add_foreign_keys(
+        [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")]
+    )
+    fk = fresh_db["books"].foreign_keys[0]
+    assert fk.on_delete == "CASCADE"
+    assert "ON DELETE CASCADE" in fresh_db["books"].schema
+
+
+def test_add_foreign_keys_preserves_actions_compound(courses_db):
+    courses_db.add_foreign_keys(
+        [
+            ForeignKey(
+                table="courses",
+                column=None,
+                other_table="departments",
+                other_column=None,
+                columns=("campus_name", "dept_code"),
+                other_columns=("campus_name", "dept_code"),
+                is_compound=True,
+                on_delete="CASCADE",
+            )
+        ]
+    )
+    fk = courses_db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.on_delete == "CASCADE"
+    assert "ON DELETE CASCADE" in courses_db["courses"].schema
+
+
+def test_add_foreign_key_on_delete_on_update(fresh_db):
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
+    fresh_db["books"].add_foreign_key(
+        "author_id", "authors", "id", on_delete="CASCADE", on_update="RESTRICT"
+    )
+    fk = fresh_db["books"].foreign_keys[0]
+    assert fk.on_delete == "CASCADE"
+    assert fk.on_update == "RESTRICT"
+    assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db["books"].schema
+    # The cascade should actually fire
+    fresh_db.execute("PRAGMA foreign_keys = ON")
+    fresh_db.execute("delete from authors where id = 1")
+    assert fresh_db["books"].count == 0
+
+
+def test_add_compound_foreign_key_on_delete(courses_db):
+    courses_db["courses"].add_foreign_key(
+        ("campus_name", "dept_code"), "departments", on_delete="SET NULL"
+    )
+    fk = courses_db["courses"].foreign_keys[0]
+    assert fk.is_compound is True
+    assert fk.on_delete == "SET NULL"
+    assert "ON DELETE SET NULL" in courses_db["courses"].schema
+
+
+def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db):
+    # The other table's PRIMARY KEY declares its columns in a different
+    # order to the table's column order. SQLite resolves the implicit
+    # "REFERENCES other" using PRIMARY KEY declaration order, so the
+    # introspected other_columns must too
+    fresh_db.execute("create table other (b text, a text, primary key (a, b))")
+    fresh_db.execute(
+        "create table child (x text, y text, foreign key (x, y) references other)"
+    )
+    fk = fresh_db["child"].foreign_keys[0]
+    assert fk.other_columns == ("a", "b")
+
+
+def test_transform_implicit_compound_foreign_key_stays_valid(fresh_db):
+    # transform() rewrites the implicit FK with explicit columns - they
+    # must be in PRIMARY KEY declaration order or valid data fails the
+    # foreign key check with an IntegrityError
+    fresh_db.execute("create table other (b text, a text, primary key (a, b))")
+    fresh_db.execute(
+        "create table child (x text, y text, foreign key (x, y) references other)"
+    )
+    fresh_db.execute("PRAGMA foreign_keys = ON")
+    fresh_db["other"].insert({"a": "A", "b": "B"})
+    fresh_db["child"].insert({"x": "A", "y": "B"})
+    fresh_db["child"].transform(types={"x": str})
+    assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b")
+    # The constraint still points the right way around
+    fresh_db["child"].insert({"x": "A", "y": "B"})
+    with pytest.raises(sqlite3.IntegrityError):
+        fresh_db["child"].insert({"x": "B", "y": "A"})
+
+
+def test_create_compound_foreign_key_guesses_pk_declaration_order(fresh_db):
+    fresh_db.execute("create table other (b text, a text, primary key (a, b))")
+    fresh_db["other"].insert({"a": "A", "b": "B"})
+    fresh_db["child"].create(
+        {"id": int, "x": str, "y": str},
+        pk="id",
+        foreign_keys=[(("x", "y"), "other")],
+    )
+    assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b")
+    fresh_db.execute("PRAGMA foreign_keys = ON")
+    fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"})
+    with pytest.raises(sqlite3.IntegrityError):
+        fresh_db["child"].insert({"id": 2, "x": "B", "y": "A"})
+
+
+def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db):
+    fresh_db.execute("create table other (b text, a text, primary key (a, b))")
+    fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id")
+    fresh_db["child"].add_foreign_key(("x", "y"), "other")
+    assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b")
+
+
+def test_foreign_keys_are_hashable(fresh_db):
+    # set() over foreign_keys worked with the 3.x namedtuple and must
+    # keep working with the dataclass
+    fresh_db["p"].insert({"id": 1}, pk="id")
+    fresh_db["c"].insert(
+        {"id": 1, "pid": 1}, pk="id", foreign_keys=[("pid", "p", "id")]
+    )
+    fks = set(fresh_db["c"].foreign_keys)
+    assert len(fks) == 1
+    assert ForeignKey("c", "pid", "p", "id") in fks
+    # Usable as dict keys too
+    assert {fk: True for fk in fks}
+
+
+def test_foreign_key_is_immutable():
+    import dataclasses
+
+    fk = ForeignKey("c", "pid", "p", "id")
+    with pytest.raises(dataclasses.FrozenInstanceError):
+        fk.table = "other"
+
+
+def test_foreign_key_equality_and_hash_include_actions():
+    # Two foreign keys differing only in ON DELETE behavior are different
+    # constraints - they compare unequal and hash separately
+    plain = ForeignKey("c", "pid", "p", "id")
+    cascade = ForeignKey("c", "pid", "p", "id", on_delete="CASCADE")
+    assert plain != cascade
+    assert len({plain, cascade}) == 2
+    assert plain == ForeignKey("c", "pid", "p", "id")
+
+
+def test_create_table_mixed_foreign_keys_list(fresh_db):
+    # 3.x accepted a mix of ForeignKey objects, tuples and bare column
+    # strings in foreign_keys= (ForeignKey was a namedtuple, so it passed
+    # the tuple check) - keep accepting the mix
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["publishers"].insert({"id": 1}, pk="id")
+    fresh_db["books"].create(
+        {"id": int, "author_id": int, "publisher_id": int},
+        pk="id",
+        foreign_keys=[
+            ForeignKey("books", "author_id", "authors", "id"),
+            ("publisher_id", "publishers", "id"),
+        ],
+    )
+    fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys}
+    assert fks == {"author_id": "authors", "publisher_id": "publishers"}
+
+
+def test_create_table_mixed_foreign_keys_with_string(fresh_db):
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["publishers"].insert({"id": 1}, pk="id")
+    fresh_db["books"].create(
+        {"id": int, "author_id": int, "publisher_id": int},
+        pk="id",
+        foreign_keys=[
+            "author_id",  # bare column, table and column guessed
+            ("publisher_id", "publishers", "id"),
+        ],
+    )
+    fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys}
+    assert fks == {"author_id": "authors", "publisher_id": "publishers"}
+
+
+def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db):
+    # Requesting an existing foreign key with different ON DELETE/ON UPDATE
+    # actions was silently skipped, dropping the requested change
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert(
+        {"id": 1, "author_id": 1},
+        pk="id",
+        foreign_keys=[("author_id", "authors", "id")],
+    )
+    with pytest.raises(AlterError) as ex:
+        fresh_db.add_foreign_keys(
+            [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")]
+        )
+    assert "ON DELETE" in str(ex.value)
+    assert fresh_db["books"].foreign_keys[0].on_delete == "NO ACTION"
+
+
+def test_add_foreign_keys_identical_existing_is_noop(fresh_db):
+    # An exact match, including actions, is silently skipped so repeated
+    # calls stay idempotent
+    fresh_db["authors"].insert({"id": 1}, pk="id")
+    fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id")
+    fresh_db["books"].add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")
+    fresh_db.add_foreign_keys(
+        [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")]
+    )
+    fks = fresh_db["books"].foreign_keys
+    assert len(fks) == 1
+    assert fks[0].on_delete == "CASCADE"
+
+
+def test_add_foreign_keys_compound_column_count_mismatch_errors(fresh_db):
+    # Previously the extra other-column was silently discarded, creating
+    # a single-column foreign key to just ("id")
+    fresh_db["departments"].insert(
+        {"campus": "north", "code": "cs"}, pk=("campus", "code")
+    )
+    fresh_db["courses"].insert({"id": 1, "campus": "north"}, pk="id")
+    with pytest.raises(ValueError) as ex:
+        fresh_db.add_foreign_keys(
+            [("courses", ("campus",), "departments", ("campus", "code"))]
+        )
+    assert "same number of columns" in str(ex.value)
+    assert fresh_db["courses"].foreign_keys == []

tests/test_fts.py

@@ -1,6 +1,7 @@
 import pytest
 from sqlite_utils import Database
 from sqlite_utils.utils import sqlite3
+from unittest.mock import ANY
 
 search_records = [
     {
@@ -82,6 +83,20 @@ def test_enable_fts_escape_table_names(fresh_db):
     assert [] == list(table.search("bar"))
 
 
+def test_search_duplicate_columns_are_deduped(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/624
+    table = fresh_db["t"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"], fts_version="FTS4")
+    rows = list(table.search("tanuki", columns=["text", "text"]))
+    assert rows == [
+        {
+            "text": "tanuki are running tricksters",
+            "text_2": "tanuki are running tricksters",
+        }
+    ]
+
+
 def test_search_limit_offset(fresh_db):
     table = fresh_db["t"]
     table.insert_all(search_records)
@@ -94,6 +109,64 @@ def test_search_limit_offset(fresh_db):
     )
 
 
+@pytest.mark.parametrize("fts_version", ("FTS4", "FTS5"))
+def test_search_where(fresh_db, fts_version):
+    table = fresh_db["t"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"], fts_version=fts_version)
+    results = list(
+        table.search("are", where="country = :country", where_args={"country": "Japan"})
+    )
+    assert results == [
+        {
+            "rowid": 1,
+            "text": "tanuki are running tricksters",
+            "country": "Japan",
+            "not_searchable": "foo",
+        }
+    ]
+
+
+def test_search_where_args_disallows_query(fresh_db):
+    table = fresh_db["t"]
+    with pytest.raises(ValueError) as ex:
+        list(
+            table.search(
+                "x", where="author = :query", where_args={"query": "not allowed"}
+            )
+        )
+    assert (
+        ex.value.args[0]
+        == "'query' is a reserved key and cannot be passed to where_args for .search()"
+    )
+
+
+def test_search_include_rank(fresh_db):
+    table = fresh_db["t"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"], fts_version="FTS5")
+    results = list(table.search("are", include_rank=True))
+    assert results == [
+        {
+            "rowid": 1,
+            "text": "tanuki are running tricksters",
+            "country": "Japan",
+            "not_searchable": "foo",
+            "rank": ANY,
+        },
+        {
+            "rowid": 2,
+            "text": "racoons are biting trash pandas",
+            "country": "USA",
+            "not_searchable": "bar",
+            "rank": ANY,
+        },
+    ]
+    assert isinstance(results[0]["rank"], float)
+    assert isinstance(results[1]["rank"], float)
+    assert results[0]["rank"] < results[1]["rank"]
+
+
 def test_enable_fts_table_names_containing_spaces(fresh_db):
     table = fresh_db["test"]
     table.insert({"column with spaces": "in its name"})
@@ -148,32 +221,32 @@ def test_populate_fts_escape_table_names(fresh_db):
     ] == list(table.search("usa"))
 
 
-def test_fts_tokenize(fresh_db):
+@pytest.mark.parametrize("fts_version", ("4", "5"))
-    for fts_version in ("4", "5"):
+def test_fts_tokenize(fresh_db, fts_version):
-        table_name = "searchable_{}".format(fts_version)
+    table_name = "searchable_{}".format(fts_version)
-        table = fresh_db[table_name]
+    table = fresh_db[table_name]
-        table.insert_all(search_records)
+    table.insert_all(search_records)
-        # Test without porter stemming
+    # Test without porter stemming
-        table.enable_fts(
+    table.enable_fts(
-            ["text", "country"],
+        ["text", "country"],
-            fts_version="FTS{}".format(fts_version),
+        fts_version="FTS{}".format(fts_version),
-        )
+    )
-        assert [] == list(table.search("bite"))
+    assert [] == list(table.search("bite"))
-        # Test WITH stemming
+    # Test WITH stemming
-        table.disable_fts()
+    table.disable_fts()
-        table.enable_fts(
+    table.enable_fts(
-            ["text", "country"],
+        ["text", "country"],
-            fts_version="FTS{}".format(fts_version),
+        fts_version="FTS{}".format(fts_version),
-            tokenize="porter",
+        tokenize="porter",
-        )
+    )
-        rows = list(table.search("bite", order_by="rowid"))
+    rows = list(table.search("bite", order_by="rowid"))
-        assert len(rows) == 1
+    assert len(rows) == 1
-        assert {
+    assert {
-            "rowid": 2,
+        "rowid": 2,
-            "text": "racoons are biting trash pandas",
+        "text": "racoons are biting trash pandas",
-            "country": "USA",
+        "country": "USA",
-            "not_searchable": "bar",
+        "not_searchable": "bar",
-        }.items() <= rows[0].items()
+    }.items() <= rows[0].items()
 
 
 def test_optimize_fts(fresh_db):
@@ -254,13 +327,12 @@ def test_disable_fts(fresh_db, create_triggers):
     assert ["searchable"] == fresh_db.table_names()
 
 
-@pytest.mark.parametrize("table_to_fix", ["searchable", "searchable_fts"])
+def test_rebuild_fts(fresh_db):
-def test_rebuild_fts(fresh_db, table_to_fix):
     table = fresh_db["searchable"]
     table.insert(search_records[0])
     table.enable_fts(["text", "country"])
     # Run a search
-    rows = list(table.search("tanuki"))
+    rows = list(table.search("are"))
     assert len(rows) == 1
     assert {
         "rowid": 1,
@@ -268,21 +340,32 @@ def test_rebuild_fts(fresh_db, table_to_fix):
         "country": "Japan",
         "not_searchable": "foo",
     }.items() <= rows[0].items()
-    # Delete from searchable_fts_data
+    # Insert another record
-    fresh_db["searchable_fts_data"].delete_where()
+    table.insert(search_records[1])
-    # This should have broken the index
+    # This should NOT show up in searches
-    with pytest.raises(sqlite3.DatabaseError):
+    assert len(list(table.search("are"))) == 1
-        list(table.search("tanuki"))
     # Running rebuild_fts() should fix it
-    fresh_db[table_to_fix].rebuild_fts()
+    table.rebuild_fts()
-    rows2 = list(table.search("tanuki"))
+    rows2 = list(table.search("are"))
-    assert len(rows2) == 1
+    assert len(rows2) == 2
-    assert {
+
-        "rowid": 1,
+
-        "text": "tanuki are running tricksters",
+@pytest.mark.parametrize("method", ["optimize", "rebuild_fts"])
-        "country": "Japan",
+def test_optimize_and_rebuild_fts_commit(tmpdir, method):
-        "not_searchable": "foo",
+    path = str(tmpdir / "test.db")
-    }.items() <= rows2[0].items()
+    db = Database(path)
+    table = db["searchable"]
+    table.insert(search_records[0])
+    table.enable_fts(["text", "country"])
+    getattr(table, method)()
+    # The connection must not be left inside an open transaction,
+    # otherwise this and all subsequent writes are lost on close
+    assert not db.conn.in_transaction
+    table.insert(search_records[1])
+    db.close()
+    db2 = Database(path)
+    assert db2["searchable"].count == 2
+    db2.close()
 
 
 @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"])
@@ -369,12 +452,35 @@ def test_enable_fts_replace_does_nothing_if_args_the_same():
     assert all(q[0].startswith("select ") for q in queries)
 
 
-def test_enable_fts_error_message_on_views():
+def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table():
+    db = Database(memory=True)
+    db["books"].insert(
+        {
+            "id": 1,
+            "title": "Habits of Australian Marsupials",
+            "author": "Marlee Hawkins",
+        },
+        pk="id",
+    )
+    db.executescript("""
+        CREATE VIRTUAL TABLE [books_fts] USING FTS5 (
+            [title],
+            content=[books]
+        );
+    """)
+
+    db["books"].enable_fts(["title", "author"], replace=True)
+
+    assert db["books_fts"].columns_dict.keys() == {"title", "author"}
+    assert 'content="books"' in db["books_fts"].schema
+
+
+def test_view_has_no_enable_fts():
     db = Database(memory=True)
     db.create_view("hello", "select 1 + 1")
-    with pytest.raises(NotImplementedError) as e:
+    # Views deliberately do not have an enable_fts() method
-        db["hello"].enable_fts()
+    with pytest.raises(AttributeError):
-        assert e.value.args[0] == "enable_fts() is supported on tables but not on views"
+        db["hello"].enable_fts()  # type: ignore[union-attr]
 
 
 @pytest.mark.parametrize(
@@ -384,85 +490,107 @@ def test_enable_fts_error_message_on_views():
             {},
             "FTS5",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
                 "        *\n"
-                "    from [books]\n"
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].*\n"
+                '    "original".*\n'
                 "from\n"
-                "    [original]\n"
+                '    "original"\n'
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
-                "    [books_fts].rank"
+                '    "books_fts".rank'
             ),
         ),
         (
             {"columns": ["title"], "order_by": "rowid", "limit": 10},
             "FTS5",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
-                "        [title]\n"
+                '        "title"\n'
-                "    from [books]\n"
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].[title]\n"
+                '    "original"."title"\n'
                 "from\n"
-                "    [original]\n"
+                '    "original"\n'
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
                 "    rowid\n"
                 "limit 10"
             ),
         ),
+        (
+            {"where": "author = :author"},
+            "FTS5",
+            (
+                'with "original" as (\n'
+                "    select\n"
+                "        rowid,\n"
+                "        *\n"
+                '    from "books"\n'
+                "    where author = :author\n"
+                ")\n"
+                "select\n"
+                '    "original".*\n'
+                "from\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
+                "where\n"
+                '    "books_fts" match :query\n'
+                "order by\n"
+                '    "books_fts".rank'
+            ),
+        ),
         (
             {"columns": ["title"]},
             "FTS4",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
-                "        [title]\n"
+                '        "title"\n'
-                "    from [books]\n"
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].[title]\n"
+                '    "original"."title"\n'
                 "from\n"
-                "    [original]\n"
+                '    "original"\n'
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
-                "    rank_bm25(matchinfo([books_fts], 'pcnalx'))"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))"
             ),
         ),
         (
             {"offset": 1, "limit": 1},
             "FTS4",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
                 "        *\n"
-                "    from [books]\n"
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].*\n"
+                '    "original".*\n'
                 "from\n"
-                "    [original]\n"
+                '    "original"\n'
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
-                "    rank_bm25(matchinfo([books_fts], 'pcnalx'))\n"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))\n"
                 "limit 1 offset 1"
             ),
         ),
@@ -470,24 +598,90 @@ def test_enable_fts_error_message_on_views():
             {"limit": 2},
             "FTS4",
             (
-                "with original as (\n"
+                'with "original" as (\n'
                 "    select\n"
                 "        rowid,\n"
                 "        *\n"
-                "    from [books]\n"
+                '    from "books"\n'
                 ")\n"
                 "select\n"
-                "    [original].*\n"
+                '    "original".*\n'
                 "from\n"
-                "    [original]\n"
+                '    "original"\n'
-                "    join [books_fts] on [original].rowid = [books_fts].rowid\n"
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
                 "where\n"
-                "    [books_fts] match :query\n"
+                '    "books_fts" match :query\n'
                 "order by\n"
-                "    rank_bm25(matchinfo([books_fts], 'pcnalx'))\n"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))\n"
                 "limit 2"
             ),
         ),
+        (
+            {"where": "author = :author"},
+            "FTS4",
+            (
+                'with "original" as (\n'
+                "    select\n"
+                "        rowid,\n"
+                "        *\n"
+                '    from "books"\n'
+                "    where author = :author\n"
+                ")\n"
+                "select\n"
+                '    "original".*\n'
+                "from\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
+                "where\n"
+                '    "books_fts" match :query\n'
+                "order by\n"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))"
+            ),
+        ),
+        (
+            {"include_rank": True},
+            "FTS5",
+            (
+                'with "original" as (\n'
+                "    select\n"
+                "        rowid,\n"
+                "        *\n"
+                '    from "books"\n'
+                ")\n"
+                "select\n"
+                '    "original".*,\n'
+                '    "books_fts".rank rank\n'
+                "from\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
+                "where\n"
+                '    "books_fts" match :query\n'
+                "order by\n"
+                '    "books_fts".rank'
+            ),
+        ),
+        (
+            {"include_rank": True},
+            "FTS4",
+            (
+                'with "original" as (\n'
+                "    select\n"
+                "        rowid,\n"
+                "        *\n"
+                '    from "books"\n'
+                ")\n"
+                "select\n"
+                '    "original".*,\n'
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx')) rank\n"
+                "from\n"
+                '    "original"\n'
+                '    join "books_fts" on "original".rowid = "books_fts".rowid\n'
+                "where\n"
+                '    "books_fts" match :query\n'
+                "order by\n"
+                "    rank_bm25(matchinfo(\"books_fts\", 'pcnalx'))"
+            ),
+        ),
     ],
 )
 def test_search_sql(kwargs, fts, expected):
@@ -501,3 +695,54 @@ def test_search_sql(kwargs, fts, expected):
     db["books"].enable_fts(["title", "author"], fts_version=fts)
     sql = db["books"].search_sql(**kwargs)
     assert sql == expected
+
+
+@pytest.mark.parametrize(
+    "input,expected",
+    (
+        ("dog", '"dog"'),
+        ("cat,", '"cat,"'),
+        ("cat's", '"cat\'s"'),
+        ("dog.", '"dog."'),
+        ("cat dog", '"cat" "dog"'),
+        # If a phrase is already double quoted, leave it so
+        ('"cat dog"', '"cat dog"'),
+        ('"cat dog" fish', '"cat dog" "fish"'),
+        # Sensibly handle unbalanced double quotes
+        ('cat"', '"cat"'),
+        ('"cat dog" "fish', '"cat dog" "fish"'),
+    ),
+)
+def test_quote_fts_query(fresh_db, input, expected):
+    table = fresh_db["searchable"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"])
+    quoted = fresh_db.quote_fts(input)
+    assert quoted == expected
+    # Executing query does not crash.
+    list(table.search(quoted))
+
+
+def test_search_quote(fresh_db):
+    table = fresh_db["searchable"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"])
+    query = "cat's"
+    with pytest.raises(sqlite3.OperationalError):
+        list(table.search(query))
+    # No exception with quote=True
+    list(table.search(query, quote=True))
+
+
+def test_enable_fts_cli_on_view_errors(tmpdir):
+    db_path = str(tmpdir / "test.db")
+    db = Database(db_path)
+    db["t"].insert({"text": "hello"})
+    db.create_view("v", "select * from t")
+    db.close()
+    from click.testing import CliRunner
+    from sqlite_utils import cli as cli_module
+
+    result = CliRunner().invoke(cli_module.cli, ["enable-fts", db_path, "v", "text"])
+    assert result.exit_code == 1
+    assert result.output.strip() == "Error: Table v is actually a view"

tests/test_gis.py

@@ -0,0 +1,236 @@
+import json
+import pytest
+
+from click.testing import CliRunner
+from sqlite_utils.cli import cli
+from sqlite_utils.db import Database
+from sqlite_utils.utils import find_spatialite, sqlite3
+
+pytestmark = [
+    pytest.mark.skipif(
+        not find_spatialite(), reason="Could not find SpatiaLite extension"
+    ),
+    pytest.mark.skipif(
+        not hasattr(sqlite3.Connection, "enable_load_extension"),
+        reason="sqlite3.Connection missing enable_load_extension",
+    ),
+]
+
+
+# python API tests
+def test_find_spatialite():
+    spatialite = find_spatialite()
+    assert spatialite is None or isinstance(spatialite, str)
+
+
+def test_init_spatialite():
+    db = Database(memory=True)
+    spatialite = find_spatialite()
+    db.init_spatialite(spatialite)
+    assert "spatial_ref_sys" in db.table_names()
+
+
+def test_add_geometry_column():
+    db = Database(memory=True)
+    spatialite = find_spatialite()
+    db.init_spatialite(spatialite)
+
+    # create a table first
+    table = db.create_table("locations", {"id": str, "properties": str})
+    table.add_geometry_column(
+        column_name="geometry",
+        geometry_type="Point",
+        srid=4326,
+        coord_dimension="XY",
+    )
+
+    assert db["geometry_columns"].get(["locations", "geometry"]) == {
+        "f_table_name": "locations",
+        "f_geometry_column": "geometry",
+        "geometry_type": 1,  # point
+        "coord_dimension": 2,
+        "srid": 4326,
+        "spatial_index_enabled": 0,
+    }
+
+
+def test_create_spatial_index():
+    db = Database(memory=True)
+    spatialite = find_spatialite()
+    assert db.init_spatialite(spatialite)
+
+    # create a table, add a geometry column with default values
+    table = db.create_table("locations", {"id": str, "properties": str})
+    assert table.add_geometry_column("geometry", "Point")
+
+    # index it
+    assert table.create_spatial_index("geometry")
+
+    assert "idx_locations_geometry" in db.table_names()
+
+
+def test_double_create_spatial_index():
+    db = Database(memory=True)
+    spatialite = find_spatialite()
+    db.init_spatialite(spatialite)
+
+    # create a table, add a geometry column with default values
+    table = db.create_table("locations", {"id": str, "properties": str})
+    table.add_geometry_column("geometry", "Point")
+
+    # index it, return True
+    assert table.create_spatial_index("geometry")
+
+    assert "idx_locations_geometry" in db.table_names()
+
+    # call it again, return False
+    assert not table.create_spatial_index("geometry")
+
+
+# cli tests
+@pytest.mark.parametrize("use_spatialite_shortcut", [True, False])
+def test_query_load_extension(use_spatialite_shortcut):
+    # Without --load-extension:
+    result = CliRunner().invoke(cli, [":memory:", "select spatialite_version()"])
+    assert result.exit_code == 1
+    assert "no such function: spatialite_version" in result.output
+    # With --load-extension:
+    if use_spatialite_shortcut:
+        load_extension = "spatialite"
+    else:
+        load_extension = find_spatialite()
+    result = CliRunner().invoke(
+        cli,
+        [
+            ":memory:",
+            "select spatialite_version()",
+            "--load-extension={}".format(load_extension),
+        ],
+    )
+    assert result.exit_code == 0, result.stdout
+    assert ["spatialite_version()"] == list(json.loads(result.output)[0].keys())
+
+
+def test_cli_create_spatialite(tmpdir):
+    # sqlite-utils create test.db --init-spatialite
+    db_path = tmpdir / "created.db"
+    result = CliRunner().invoke(
+        cli, ["create-database", str(db_path), "--init-spatialite"]
+    )
+
+    assert result.exit_code == 0
+    assert db_path.exists()
+    assert db_path.read_binary()[:16] == b"SQLite format 3\x00"
+
+    db = Database(str(db_path))
+    assert "spatial_ref_sys" in db.table_names()
+
+
+def test_cli_add_geometry_column(tmpdir):
+    # create a rowid table with one column
+    db_path = tmpdir / "spatial.db"
+    db = Database(str(db_path))
+    db.init_spatialite()
+
+    table = db["locations"].create({"name": str})
+
+    result = CliRunner().invoke(
+        cli,
+        [
+            "add-geometry-column",
+            str(db_path),
+            table.name,
+            "geometry",
+            "--type",
+            "POINT",
+        ],
+    )
+
+    assert result.exit_code == 0
+
+    assert db["geometry_columns"].get(["locations", "geometry"]) == {
+        "f_table_name": "locations",
+        "f_geometry_column": "geometry",
+        "geometry_type": 1,  # point
+        "coord_dimension": 2,
+        "srid": 4326,
+        "spatial_index_enabled": 0,
+    }
+
+
+def test_cli_add_geometry_column_options(tmpdir):
+    # create a rowid table with one column
+    db_path = tmpdir / "spatial.db"
+    db = Database(str(db_path))
+    db.init_spatialite()
+    table = db["locations"].create({"name": str})
+
+    result = CliRunner().invoke(
+        cli,
+        [
+            "add-geometry-column",
+            str(db_path),
+            table.name,
+            "geometry",
+            "-t",
+            "POLYGON",
+            "--srid",
+            "3857",  # https://epsg.io/3857
+            "--not-null",
+        ],
+    )
+
+    assert result.exit_code == 0
+
+    assert db["geometry_columns"].get(["locations", "geometry"]) == {
+        "f_table_name": "locations",
+        "f_geometry_column": "geometry",
+        "geometry_type": 3,  # polygon
+        "coord_dimension": 2,
+        "srid": 3857,
+        "spatial_index_enabled": 0,
+    }
+
+    column = table.columns[1]
+    assert column.notnull
+
+
+def test_cli_add_geometry_column_invalid_type(tmpdir):
+    # create a rowid table with one column
+    db_path = tmpdir / "spatial.db"
+    db = Database(str(db_path))
+    db.init_spatialite()
+
+    table = db["locations"].create({"name": str})
+
+    result = CliRunner().invoke(
+        cli,
+        [
+            "add-geometry-column",
+            str(db_path),
+            table.name,
+            "geometry",
+            "--type",
+            "NOT-A-TYPE",
+        ],
+    )
+
+    assert 2 == result.exit_code
+
+
+def test_cli_create_spatial_index(tmpdir):
+    # create a rowid table with one column
+    db_path = tmpdir / "spatial.db"
+    db = Database(str(db_path))
+    db.init_spatialite()
+
+    table = db["locations"].create({"name": str})
+    table.add_geometry_column("geometry", "POINT")
+
+    result = CliRunner().invoke(
+        cli, ["create-spatial-index", str(db_path), table.name, "geometry"]
+    )
+
+    assert result.exit_code == 0
+
+    assert "idx_locations_geometry" in db.table_names()

tests/test_hypothesis.py

@@ -2,6 +2,7 @@ from hypothesis import given
 import hypothesis.strategies as st
 import sqlite_utils
 
+
 # SQLite integers are -(2^63) to 2^63 - 1
 @given(st.integers(-9223372036854775808, 9223372036854775807))
 def test_roundtrip_integers(integer):

tests/test_insert_files.py

@@ -2,9 +2,19 @@ from sqlite_utils import cli, Database
 from click.testing import CliRunner
 import os
 import pathlib
+import pytest
+import sys
 
 
-def test_insert_files():
+@pytest.mark.parametrize("silent", (False, True))
+@pytest.mark.parametrize(
+    "pk_args,expected_pks",
+    (
+        (["--pk", "path"], ["path"]),
+        (["--pk", "path", "--pk", "name"], ["path", "name"]),
+    ),
+)
+def test_insert_files(silent, pk_args, expected_pks):
     runner = CliRunner()
     with runner.isolated_filesystem():
         tmpdir = pathlib.Path(".")
@@ -12,7 +22,7 @@ def test_insert_files():
         (tmpdir / "one.txt").write_text("This is file one", "utf-8")
         (tmpdir / "two.txt").write_text("Two is shorter", "utf-8")
         (tmpdir / "nested").mkdir()
-        (tmpdir / "nested" / "three.txt").write_text("Three is nested", "utf-8")
+        (tmpdir / "nested" / "three.zz.txt").write_text("Three is nested", "utf-8")
         coltypes = (
             "name",
             "path",
@@ -21,6 +31,7 @@ def test_insert_files():
             "md5",
             "mode",
             "content",
+            "content_text",
             "mtime",
             "ctime",
             "mtime_int",
@@ -28,13 +39,18 @@ def test_insert_files():
             "mtime_iso",
             "ctime_iso",
             "size",
+            "suffix",
+            "stem",
         )
         cols = []
         for coltype in coltypes:
             cols += ["-c", "{}:{}".format(coltype, coltype)]
         result = runner.invoke(
             cli.cli,
-            ["insert-files", db_path, "files", str(tmpdir)] + cols + ["--pk", "path"],
+            ["insert-files", db_path, "files", str(tmpdir)]
+            + cols
+            + pk_args
+            + (["--silent"] if silent else []),
             catch_exceptions=False,
         )
         assert result.exit_code == 0, result.stdout
@@ -43,31 +59,40 @@ def test_insert_files():
         one, two, three = (
             rows_by_path["one.txt"],
             rows_by_path["two.txt"],
-            rows_by_path[os.path.join("nested", "three.txt")],
+            rows_by_path[os.path.join("nested", "three.zz.txt")],
         )
         assert {
             "content": b"This is file one",
+            "content_text": "This is file one",
             "md5": "556dfb57fce9ca301f914e2273adf354",
             "name": "one.txt",
             "path": "one.txt",
             "sha256": "e34138f26b5f7368f298b4e736fea0aad87ddec69fbd04dc183b20f4d844bad5",
             "size": 16,
+            "stem": "one",
+            "suffix": ".txt",
         }.items() <= one.items()
         assert {
             "content": b"Two is shorter",
+            "content_text": "Two is shorter",
             "md5": "f86f067b083af1911043eb215e74ac70",
             "name": "two.txt",
             "path": "two.txt",
             "sha256": "9368988ed16d4a2da0af9db9b686d385b942cb3ffd4e013f43aed2ec041183d9",
             "size": 14,
+            "stem": "two",
+            "suffix": ".txt",
         }.items() <= two.items()
         assert {
             "content": b"Three is nested",
+            "content_text": "Three is nested",
             "md5": "12580f341781f5a5b589164d3cd39523",
-            "name": "three.txt",
+            "name": "three.zz.txt",
-            "path": os.path.join("nested", "three.txt"),
+            "path": os.path.join("nested", "three.zz.txt"),
             "sha256": "6dd45aaaaa6b9f96af19363a92c8fca5d34791d3c35c44eb19468a6a862cc8cd",
             "size": 15,
+            "stem": "three.zz",
+            "suffix": ".txt",
         }.items() <= three.items()
         # Assert the other int/str/float columns exist and are of the right types
         expected_types = {
@@ -79,24 +104,68 @@ def test_insert_files():
             "mtime_iso": str,
             "mode": int,
             "fullpath": str,
+            "content": bytes,
+            "content_text": str,
+            "stem": str,
+            "suffix": str,
         }
         for colname, expected_type in expected_types.items():
             for row in (one, two, three):
                 assert isinstance(row[colname], expected_type)
+        assert set(db["files"].pks) == set(expected_pks)
 
 
-def test_insert_files_stdin():
+@pytest.mark.parametrize(
+    "use_text,encoding,input,expected",
+    (
+        (False, None, "hello world", b"hello world"),
+        (True, None, "hello world", "hello world"),
+        (False, None, b"S\xe3o Paulo", b"S\xe3o Paulo"),
+        (True, "latin-1", b"S\xe3o Paulo", "S\xe3o Paulo"),
+    ),
+)
+def test_insert_files_stdin(use_text, encoding, input, expected):
     runner = CliRunner()
     with runner.isolated_filesystem():
         tmpdir = pathlib.Path(".")
         db_path = str(tmpdir / "files.db")
+        args = ["insert-files", db_path, "files", "-", "--name", "stdin-name"]
+        if use_text:
+            args += ["--text"]
+        if encoding is not None:
+            args += ["--encoding", encoding]
         result = runner.invoke(
             cli.cli,
-            ["insert-files", db_path, "files", "-", "--name", "stdin-name"],
+            args,
             catch_exceptions=False,
-            input="hello world",
+            input=input,
         )
         assert result.exit_code == 0, result.stdout
         db = Database(db_path)
         row = list(db["files"].rows)[0]
-        assert {"path": "stdin-name", "content": b"hello world", "size": 11} == row
+        key = "content"
+        if use_text:
+            key = "content_text"
+        assert {"path": "stdin-name", key: expected}.items() <= row.items()
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="Windows has a different way of handling default encodings",
+)
+def test_insert_files_bad_text_encoding_error():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        tmpdir = pathlib.Path(".")
+        latin = tmpdir / "latin.txt"
+        latin.write_bytes(b"S\xe3o Paulo")
+        db_path = str(tmpdir / "files.db")
+        result = runner.invoke(
+            cli.cli,
+            ["insert-files", db_path, "files", str(latin), "--text"],
+            catch_exceptions=False,
+        )
+        assert result.exit_code == 1, result.output
+        assert result.output.strip().startswith(
+            "Error: Could not read file '{}' as text".format(str(latin.resolve()))
+        )

tests/test_introspect.py

@@ -2,6 +2,14 @@ from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn
 import pytest
 
 
+def _check_supports_strict():
+    """Check if SQLite supports strict tables without leaking the database."""
+    db = Database(memory=True)
+    result = db.supports_strict
+    db.close()
+    return result
+
+
 def test_table_names(existing_db):
     assert ["foo"] == existing_db.table_names()
 
@@ -33,26 +41,50 @@ def test_detect_fts(existing_db):
     assert "woo_fts" == existing_db["woo_fts"].detect_fts()
     assert "woo2_fts" == existing_db["woo2"].detect_fts()
     assert "woo2_fts" == existing_db["woo2_fts"].detect_fts()
-    assert None == existing_db["foo"].detect_fts()
+    assert existing_db["foo"].detect_fts() is None
+
+
+@pytest.mark.parametrize("reverse_order", (True, False))
+def test_detect_fts_similar_tables(fresh_db, reverse_order):
+    # https://github.com/simonw/sqlite-utils/issues/434
+    table1, table2 = ("demo", "demo2")
+    if reverse_order:
+        table1, table2 = table2, table1
+
+    fresh_db[table1].insert({"title": "Hello"}).enable_fts(
+        ["title"], fts_version="FTS4"
+    )
+    fresh_db[table2].insert({"title": "Hello"}).enable_fts(
+        ["title"], fts_version="FTS4"
+    )
+    assert fresh_db[table1].detect_fts() == "{}_fts".format(table1)
+    assert fresh_db[table2].detect_fts() == "{}_fts".format(table2)
 
 
 def test_tables(existing_db):
-    assert 1 == len(existing_db.tables)
+    assert len(existing_db.tables) == 1
-    assert "foo" == existing_db.tables[0].name
+    assert existing_db.tables[0].name == "foo"
 
 
 def test_views(fresh_db):
     fresh_db.create_view("foo_view", "select 1")
-    assert 1 == len(fresh_db.views)
+    assert len(fresh_db.views) == 1
     view = fresh_db.views[0]
     assert isinstance(view, View)
-    assert "foo_view" == view.name
+    assert view.name == "foo_view"
-    assert "<View foo_view (1)>" == repr(view)
+    assert repr(view) == "<View foo_view (1)>"
-    assert {"1": str} == view.columns_dict
+    assert view.columns_dict == {"1": str}
 
 
 def test_count(existing_db):
-    assert 3 == existing_db["foo"].count
+    assert existing_db["foo"].count == 3
+    assert existing_db["foo"].count_where() == 3
+    assert existing_db["foo"].execute_count() == 3
+
+
+def test_count_where(existing_db):
+    assert existing_db["foo"].count_where("text != ?", ["two"]) == 2
+    assert existing_db["foo"].count_where("text != :t", {"t": "two"}) == 2
 
 
 def test_columns(existing_db):
@@ -77,13 +109,11 @@ def test_table_repr(fresh_db):
 
 
 def test_indexes(fresh_db):
-    fresh_db.executescript(
+    fresh_db.executescript("""
-        """
         create table Gosh (c1 text, c2 text, c3 text);
         create index Gosh_c1 on Gosh(c1);
         create index Gosh_c2c3 on Gosh(c2, c3);
-    """
+    """)
-    )
     assert [
         Index(
             seq=0,
@@ -98,13 +128,11 @@ def test_indexes(fresh_db):
 
 
 def test_xindexes(fresh_db):
-    fresh_db.executescript(
+    fresh_db.executescript("""
-        """
         create table Gosh (c1 text, c2 text, c3 text);
         create index Gosh_c1 on Gosh(c1);
         create index Gosh_c2c3 on Gosh(c2, c3 desc);
-    """
+    """)
-    )
     assert fresh_db["Gosh"].xindexes == [
         XIndex(
             name="Gosh_c2c3",
@@ -175,9 +203,21 @@ def test_triggers_and_triggers_dict(fresh_db):
         (t.name, t.table) for t in fresh_db["authors"].triggers
     }
     expected_triggers = {
-        "authors_ai": "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n  INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND",
+        "authors_ai": (
-        "authors_ad": "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n  INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\nEND",
+            'CREATE TRIGGER "authors_ai" AFTER INSERT ON "authors" BEGIN\n'
-        "authors_au": "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n  INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n  INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND",
+            '  INSERT INTO "authors_fts" (rowid, "name", "famous_works") VALUES (new.rowid, new."name", new."famous_works");\n'
+            "END"
+        ),
+        "authors_ad": (
+            'CREATE TRIGGER "authors_ad" AFTER DELETE ON "authors" BEGIN\n'
+            '  INSERT INTO "authors_fts" ("authors_fts", rowid, "name", "famous_works") VALUES(\'delete\', old.rowid, old."name", old."famous_works");\n'
+            "END"
+        ),
+        "authors_au": (
+            'CREATE TRIGGER "authors_au" AFTER UPDATE ON "authors" BEGIN\n'
+            '  INSERT INTO "authors_fts" ("authors_fts", rowid, "name", "famous_works") VALUES(\'delete\', old.rowid, old."name", old."famous_works");\n'
+            '  INSERT INTO "authors_fts" (rowid, "name", "famous_works") VALUES (new.rowid, new."name", new."famous_works");\nEND'
+        ),
     }
     assert authors.triggers_dict == expected_triggers
     assert fresh_db["other"].triggers == []
@@ -233,15 +273,66 @@ def test_has_counts_triggers(fresh_db):
         ),
     ],
 )
-def test_virtual_table_using(sql, expected_name, expected_using):
+def test_virtual_table_using(fresh_db, sql, expected_name, expected_using):
-    db = Database(memory=True)
+    fresh_db.execute(sql)
-    db.execute(sql)
+    assert fresh_db[expected_name].virtual_table_using == expected_using
-    assert db[expected_name].virtual_table_using == expected_using
 
 
-def test_use_rowid():
+def test_use_rowid(fresh_db):
-    db = Database(memory=True)
+    fresh_db["rowid_table"].insert({"name": "Cleo"})
-    db["rowid_table"].insert({"name": "Cleo"})
+    fresh_db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id")
-    db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id")
+    assert fresh_db["rowid_table"].use_rowid
-    assert db["rowid_table"].use_rowid
+    assert not fresh_db["regular_table"].use_rowid
-    assert not db["regular_table"].use_rowid
+
+
+@pytest.mark.skipif(
+    not _check_supports_strict(),
+    reason="Needs SQLite version that supports strict",
+)
+@pytest.mark.parametrize(
+    "create_table,expected_strict",
+    (
+        ("create table t (id integer) strict", True),
+        ("create table t (id integer) STRICT", True),
+        ("create table t (id integer primary key) StriCt, WITHOUT ROWID", True),
+        ("create table t (id integer primary key) WITHOUT ROWID", False),
+        ("create table t (id integer)", False),
+    ),
+)
+def test_table_strict(fresh_db, create_table, expected_strict):
+    fresh_db.execute(create_table)
+    table = fresh_db["t"]
+    assert table.strict == expected_strict
+
+
+@pytest.mark.parametrize(
+    "value",
+    (
+        1,
+        1.3,
+        "foo",
+        True,
+        b"binary",
+    ),
+)
+def test_table_default_values(fresh_db, value):
+    fresh_db["default_values"].insert(
+        {"nodefault": 1, "value": value}, defaults={"value": value}
+    )
+    default_values = fresh_db["default_values"].default_values
+    assert default_values == {"value": value}
+
+
+def test_pks_use_primary_key_declaration_order(fresh_db):
+    # PRIMARY KEY (a, b) declared against columns stored in order (b, a) -
+    # pks must follow the declaration order, which is what SQLite uses to
+    # resolve implicit foreign key references and compound pk lookups
+    fresh_db.execute("create table t (b text, a text, primary key (a, b))")
+    assert fresh_db["t"].pks == ["a", "b"]
+
+
+def test_transform_preserves_compound_pk_declaration_order(fresh_db):
+    fresh_db.execute("create table t (a text, b text, c text, primary key (b, a))")
+    fresh_db["t"].transform(drop={"c"})
+    assert fresh_db["t"].pks == ["b", "a"]
+    assert 'PRIMARY KEY ("b", "a")' in fresh_db["t"].schema

tests/test_list_mode.py

@@ -0,0 +1,288 @@
+"""
+Tests for list-based iteration in insert_all and upsert_all
+"""
+
+import pytest
+from sqlite_utils import Database
+
+
+def test_insert_all_list_mode_basic():
+    """Test basic insert_all with list-based iteration"""
+    db = Database(memory=True)
+
+    def data_generator():
+        # First yield column names
+        yield ["id", "name", "age"]
+        # Then yield data rows
+        yield [1, "Alice", 30]
+        yield [2, "Bob", 25]
+        yield [3, "Charlie", 35]
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows)
+    assert len(rows) == 3
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": 25}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35}
+
+
+def test_insert_all_list_mode_with_pk():
+    """Test insert_all with list mode and primary key"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield ["id", "name", "score"]
+        yield [1, "Alice", 95]
+        yield [2, "Bob", 87]
+
+    db["scores"].insert_all(data_generator(), pk="id")
+
+    assert db["scores"].pks == ["id"]
+    rows = list(db["scores"].rows)
+    assert len(rows) == 2
+
+
+def test_upsert_all_list_mode():
+    """Test upsert_all with list-based iteration"""
+    db = Database(memory=True)
+
+    # Initial insert
+    def initial_data():
+        yield ["id", "name", "value"]
+        yield [1, "Alice", 100]
+        yield [2, "Bob", 200]
+
+    db["data"].insert_all(initial_data(), pk="id")
+
+    # Upsert with some updates and new records
+    def upsert_data():
+        yield ["id", "name", "value"]
+        yield [1, "Alice", 150]  # Update existing
+        yield [3, "Charlie", 300]  # Insert new
+
+    db["data"].upsert_all(upsert_data(), pk="id")
+
+    rows = list(db["data"].rows_where(order_by="id"))
+    assert len(rows) == 3
+    assert rows[0] == {"id": 1, "name": "Alice", "value": 150}
+    assert rows[1] == {"id": 2, "name": "Bob", "value": 200}
+    assert rows[2] == {"id": 3, "name": "Charlie", "value": 300}
+
+
+def test_list_mode_with_various_types():
+    """Test list mode with different data types"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield ["id", "name", "score", "active"]
+        yield [1, "Alice", 95.5, True]
+        yield [2, "Bob", 87.3, False]
+        yield [3, "Charlie", None, True]
+
+    db["mixed"].insert_all(data_generator())
+
+    rows = list(db["mixed"].rows)
+    assert len(rows) == 3
+    assert rows[0]["score"] == 95.5
+    assert rows[1]["active"] == 0  # SQLite stores boolean as int
+    assert rows[2]["score"] is None
+
+
+def test_list_mode_error_non_string_columns():
+    """Test that non-string column names raise an error"""
+    db = Database(memory=True)
+
+    def bad_data():
+        yield [1, 2, 3]  # Non-string column names
+        yield ["a", "b", "c"]
+
+    with pytest.raises(ValueError, match="must be a list of column name strings"):
+        db["bad"].insert_all(bad_data())
+
+
+def test_list_mode_error_mixed_types():
+    """Test that mixing list and dict raises an error"""
+    db = Database(memory=True)
+
+    def bad_data():
+        yield ["id", "name"]
+        yield {"id": 1, "name": "Alice"}  # Should be a list, not dict
+
+    with pytest.raises(ValueError, match="must also be lists"):
+        db["bad"].insert_all(bad_data())
+
+
+def test_list_mode_empty_after_headers():
+    """Test that only headers without data works gracefully"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield ["id", "name", "age"]
+        # No data rows
+
+    result = db["people"].insert_all(data_generator())
+    assert result is not None
+    assert not db["people"].exists()
+
+
+def test_list_mode_batch_processing():
+    """Test list mode with large dataset requiring batching"""
+    db = Database(memory=True)
+
+    def large_data():
+        yield ["id", "value"]
+        for i in range(1000):
+            yield [i, f"value_{i}"]
+
+    db["large"].insert_all(large_data(), batch_size=100)
+
+    count = db.execute("SELECT COUNT(*) as c FROM large").fetchone()[0]
+    assert count == 1000
+
+
+def test_list_mode_shorter_rows():
+    """Test that rows shorter than column list get NULL values"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield ["id", "name", "age", "city"]
+        yield [1, "Alice", 30, "NYC"]
+        yield [2, "Bob"]  # Missing age and city
+        yield [3, "Charlie", 35]  # Missing city
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows_where(order_by="id"))
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None}
+
+
+def test_backwards_compatibility_dict_mode():
+    """Ensure dict mode still works (backward compatibility)"""
+    db = Database(memory=True)
+
+    # Traditional dict-based insert
+    data = [
+        {"id": 1, "name": "Alice", "age": 30},
+        {"id": 2, "name": "Bob", "age": 25},
+    ]
+
+    db["people"].insert_all(data)
+
+    rows = list(db["people"].rows)
+    assert len(rows) == 2
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30}
+
+
+def test_insert_all_tuple_mode_basic():
+    """Test basic insert_all with tuple-based iteration"""
+    db = Database(memory=True)
+
+    def data_generator():
+        # First yield column names as tuple
+        yield ("id", "name", "age")
+        # Then yield data rows as tuples
+        yield (1, "Alice", 30)
+        yield (2, "Bob", 25)
+        yield (3, "Charlie", 35)
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows)
+    assert len(rows) == 3
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": 25}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35}
+
+
+def test_insert_all_mixed_list_tuple():
+    """Test insert_all with mixed lists and tuples for data rows"""
+    db = Database(memory=True)
+
+    def data_generator():
+        # Column names as list
+        yield ["id", "name", "age"]
+        # Mix of list and tuple data rows
+        yield [1, "Alice", 30]
+        yield (2, "Bob", 25)
+        yield [3, "Charlie", 35]
+        yield (4, "Diana", 40)
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows)
+    assert len(rows) == 4
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": 25}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35}
+    assert rows[3] == {"id": 4, "name": "Diana", "age": 40}
+
+
+def test_upsert_all_tuple_mode():
+    """Test upsert_all with tuple-based iteration"""
+    db = Database(memory=True)
+
+    # Initial insert with tuples
+    def initial_data():
+        yield ("id", "name", "value")
+        yield (1, "Alice", 100)
+        yield (2, "Bob", 200)
+
+    db["data"].insert_all(initial_data(), pk="id")
+
+    # Upsert with tuples
+    def upsert_data():
+        yield ("id", "name", "value")
+        yield (1, "Alice", 150)  # Update existing
+        yield (3, "Charlie", 300)  # Insert new
+
+    db["data"].upsert_all(upsert_data(), pk="id")
+
+    rows = list(db["data"].rows_where(order_by="id"))
+    assert len(rows) == 3
+    assert rows[0] == {"id": 1, "name": "Alice", "value": 150}
+    assert rows[1] == {"id": 2, "name": "Bob", "value": 200}
+    assert rows[2] == {"id": 3, "name": "Charlie", "value": 300}
+
+
+def test_tuple_mode_shorter_rows():
+    """Test that tuple rows shorter than column list get NULL values"""
+    db = Database(memory=True)
+
+    def data_generator():
+        yield "id", "name", "age", "city"
+        yield 1, "Alice", 30, "NYC"
+        yield 2, "Bob"  # Missing age and city
+        yield 3, "Charlie", 35  # Missing city
+
+    db["people"].insert_all(data_generator())
+
+    rows = list(db["people"].rows_where(order_by="id"))
+    assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"}
+    assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None}
+    assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None}
+
+
+def test_list_mode_single_record_upsert_last_pk():
+    """Test that last_pk is populated correctly for single-record upserts in list mode"""
+    db = Database(memory=True)
+
+    # Create table first
+    db["data"].insert({"id": 1, "name": "Alice", "value": 100}, pk="id")
+
+    # Now upsert a single record using list mode
+    def upsert_data():
+        yield ["id", "name", "value"]
+        yield [1, "Alice", 150]  # Update existing
+
+    table = db["data"]
+    table.upsert_all(upsert_data(), pk="id")
+
+    # Verify the data was updated
+    rows = list(db["data"].rows)
+    assert rows == [{"id": 1, "name": "Alice", "value": 150}]
+
+    # Verify last_pk is populated correctly
+    assert table.last_pk == 1

tests/test_lookup.py

@@ -66,3 +66,118 @@ def test_lookup_fails_if_constraint_cannot_be_added(fresh_db):
     # This will fail because the name column is not unique
     with pytest.raises(Exception, match="UNIQUE constraint failed"):
         species.lookup({"name": "Palm"})
+
+
+def test_lookup_with_extra_values(fresh_db):
+    species = fresh_db["species"]
+    id = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2020-01-01"})
+    assert species.get(id) == {
+        "id": 1,
+        "name": "Palm",
+        "type": "Tree",
+        "first_seen": "2020-01-01",
+    }
+    # A subsequent lookup() should ignore the second dictionary
+    id2 = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2021-02-02"})
+    assert id2 == id
+    assert species.get(id2) == {
+        "id": 1,
+        "name": "Palm",
+        "type": "Tree",
+        "first_seen": "2020-01-01",
+    }
+
+
+def test_lookup_with_extra_insert_parameters(fresh_db):
+    other_table = fresh_db["other_table"]
+    other_table.insert({"id": 1, "name": "Name"}, pk="id")
+    species = fresh_db["species"]
+    id = species.lookup(
+        {"name": "Palm", "type": "Tree"},
+        {
+            "first_seen": "2020-01-01",
+            "make_not_null": 1,
+            "fk_to_other": 1,
+            "default_is_dog": "cat",
+            "extract_this": "This is extracted",
+            "convert_to_upper": "upper",
+            "make_this_integer": "2",
+            "this_at_front": 1,
+        },
+        pk="renamed_id",
+        foreign_keys=(("fk_to_other", "other_table", "id"),),
+        column_order=("this_at_front",),
+        not_null={"make_not_null"},
+        defaults={"default_is_dog": "dog"},
+        extracts=["extract_this"],
+        conversions={"convert_to_upper": "upper(?)"},
+        columns={"make_this_integer": int},
+    )
+    assert species.schema == (
+        'CREATE TABLE "species" (\n'
+        '   "renamed_id" INTEGER PRIMARY KEY,\n'
+        '   "this_at_front" INTEGER,\n'
+        '   "name" TEXT,\n'
+        '   "type" TEXT,\n'
+        '   "first_seen" TEXT,\n'
+        '   "make_not_null" INTEGER NOT NULL,\n'
+        '   "fk_to_other" INTEGER REFERENCES "other_table"("id"),\n'
+        "   \"default_is_dog\" TEXT DEFAULT 'dog',\n"
+        '   "extract_this" INTEGER REFERENCES "extract_this"("id"),\n'
+        '   "convert_to_upper" TEXT,\n'
+        '   "make_this_integer" INTEGER\n'
+        ")"
+    )
+    assert species.get(id) == {
+        "renamed_id": id,
+        "this_at_front": 1,
+        "name": "Palm",
+        "type": "Tree",
+        "first_seen": "2020-01-01",
+        "make_not_null": 1,
+        "fk_to_other": 1,
+        "default_is_dog": "cat",
+        "extract_this": 1,
+        "convert_to_upper": "UPPER",
+        "make_this_integer": 2,
+    }
+    assert species.indexes == [
+        Index(
+            seq=0,
+            name="idx_species_name_type",
+            unique=1,
+            origin="c",
+            partial=0,
+            columns=["name", "type"],
+        )
+    ]
+
+
+@pytest.mark.parametrize("strict", (False, True))
+def test_lookup_new_table_strict(fresh_db, strict):
+    fresh_db["species"].lookup({"name": "Palm"}, strict=strict)
+    assert fresh_db["species"].strict == strict or not fresh_db.supports_strict
+
+
+def test_lookup_null_value_idempotent(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/186
+    # Repeated lookups of a null value should return the same row,
+    # not insert a duplicate row each time
+    species = fresh_db["species"]
+    first_id = species.lookup({"name": None})
+    second_id = species.lookup({"name": None})
+    assert first_id == second_id
+    assert list(species.rows) == [{"id": first_id, "name": None}]
+
+
+def test_lookup_compound_key_with_null_idempotent(fresh_db):
+    species = fresh_db["species"]
+    palm_id = species.lookup({"name": "Palm", "type": None})
+    oak_id = species.lookup({"name": "Oak", "type": "Tree"})
+    assert palm_id == species.lookup({"name": "Palm", "type": None})
+    assert oak_id == species.lookup({"name": "Oak", "type": "Tree"})
+    assert palm_id != oak_id
+    assert list(species.rows) == [
+        {"id": palm_id, "name": "Palm", "type": None},
+        {"id": oak_id, "name": "Oak", "type": "Tree"},
+    ]

tests/test_m2m.py

@@ -109,9 +109,9 @@ def test_m2m_with_table_objects(fresh_db):
     )
     expected_tables = {"dogs", "humans", "dogs_humans"}
     assert expected_tables == set(fresh_db.table_names())
-    assert 1 == dogs.count
+    assert dogs.count == 1
-    assert 2 == humans.count
+    assert humans.count == 2
-    assert 2 == fresh_db["dogs_humans"].count
+    assert fresh_db["dogs_humans"].count == 2
 
 
 def test_m2m_lookup(fresh_db):
@@ -139,9 +139,9 @@ def test_m2m_lookup(fresh_db):
 
 def test_m2m_requires_either_records_or_lookup(fresh_db):
     people = fresh_db.table("people", pk="id").insert({"name": "Wahyu"})
-    with pytest.raises(AssertionError):
+    with pytest.raises(ValueError):
         people.m2m("tags")
-    with pytest.raises(AssertionError):
+    with pytest.raises(ValueError):
         people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"})
 
 

tests/test_migrations.py

@@ -0,0 +1,246 @@
+import pytest
+import sqlite_utils
+from sqlite_utils import Migrations
+
+
+@pytest.fixture
+def migrations():
+    migrations = Migrations("test")
+
+    @migrations()
+    def m001(db):
+        db["dogs"].insert({"name": "Cleo"})
+
+    @migrations()
+    def m002(db):
+        db["cats"].create({"name": str})
+        db.execute("insert into dogs (name) values ('Pancakes')")
+
+    return migrations
+
+
+@pytest.fixture
+def migrations_not_ordered_alphabetically():
+    # Names order alphabetically in the wrong direction but this
+    # should still be applied correctly.
+    migrations = Migrations("test")
+
+    @migrations()
+    def m002(db):
+        db["dogs"].insert({"name": "Cleo"})
+
+    @migrations()
+    def m001(db):
+        db["cats"].create({"name": str})
+        db.execute("insert into dogs (name) values ('Pancakes')")
+
+    return migrations
+
+
+@pytest.fixture
+def migrations2():
+    migrations = Migrations("test2")
+
+    @migrations()
+    def m001(db):
+        db["dogs2"].insert({"name": "Cleo"})
+
+    return migrations
+
+
+def test_basic(migrations):
+    db = sqlite_utils.Database(memory=True)
+    assert db.table_names() == []
+    migrations.apply(db)
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"}
+
+
+def test_stop_before(migrations):
+    db = sqlite_utils.Database(memory=True)
+    assert db.table_names() == []
+    migrations.apply(db, stop_before="m002")
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs"}
+    migrations.apply(db)
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"}
+
+
+def test_two_migration_sets(migrations, migrations2):
+    db = sqlite_utils.Database(memory=True)
+    assert db.table_names() == []
+    migrations.apply(db)
+    migrations2.apply(db)
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats", "dogs2"}
+
+
+def test_order_does_not_matter(migrations, migrations_not_ordered_alphabetically):
+    db1 = sqlite_utils.Database(memory=True)
+    db2 = sqlite_utils.Database(memory=True)
+    migrations.apply(db1)
+    migrations_not_ordered_alphabetically.apply(db2)
+    assert db1.schema == db2.schema
+
+
+def test_applied_at_is_a_string(migrations):
+    db = sqlite_utils.Database(memory=True)
+    migrations.apply(db)
+    applied = migrations.applied(db)
+    assert len(applied) == 2
+    for migration in applied:
+        # applied_at is the TEXT timestamp straight from the
+        # _sqlite_migrations table, e.g. "2026-07-04 12:00:00.000000+00:00"
+        assert isinstance(migration.applied_at, str)
+        assert migration.applied_at.endswith("+00:00")
+
+
+def test_failing_migration_rolls_back(migrations):
+    @migrations()
+    def m003(db):
+        db["birds"].create({"name": str})
+        db.execute("insert into dogs (name) values ('Dozer')")
+        raise ValueError("boom")
+
+    db = sqlite_utils.Database(memory=True)
+    with pytest.raises(ValueError):
+        migrations.apply(db)
+    # m001 and m002 committed before the failure and stay applied
+    assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"}
+    assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"]
+    assert [m.name for m in migrations.applied(db)] == ["m001", "m002"]
+    # Everything m003 did was rolled back and it is still pending
+    assert [m.name for m in migrations.pending(db)] == ["m003"]
+
+
+def test_rerun_after_failure_applies_each_migration_once():
+    state = {"fail": True}
+    migrations = Migrations("test")
+
+    @migrations()
+    def m001(db):
+        db["dogs"].insert({"name": "Cleo"})
+
+    @migrations()
+    def m002(db):
+        db["dogs"].insert({"name": "Pancakes"})
+        if state["fail"]:
+            raise ValueError("boom")
+
+    db = sqlite_utils.Database(memory=True)
+    with pytest.raises(ValueError):
+        migrations.apply(db)
+    state["fail"] = False
+    migrations.apply(db)
+    # m001 must not have been re-applied, m002 applied exactly once
+    assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"]
+    assert [m.name for m in migrations.applied(db)] == ["m001", "m002"]
+
+
+def test_non_transactional_migration_allows_vacuum(tmpdir):
+    path = str(tmpdir / "test.db")
+    db = sqlite_utils.Database(path)
+    migrations = Migrations("test")
+
+    @migrations()
+    def m001(db):
+        db["dogs"].insert({"name": "Cleo"})
+
+    @migrations(transactional=False)
+    def m002(db):
+        db.execute("VACUUM")
+
+    migrations.apply(db)
+    assert [m.name for m in migrations.applied(db)] == ["m001", "m002"]
+    db.close()
+
+
+def test_apply_composes_inside_outer_transaction(migrations):
+    db = sqlite_utils.Database(memory=True)
+    with pytest.raises(ZeroDivisionError):
+        with db.atomic():
+            migrations.apply(db)
+            raise ZeroDivisionError
+    # The outer transaction rolled back, taking the migrations with it
+    assert db.table_names() == []
+
+
+@pytest.mark.parametrize(
+    "create_table,pk",
+    (
+        (
+            {
+                "migration_set": str,
+                "name": str,
+                "applied_at": str,
+            },
+            "name",
+        ),
+        (
+            {
+                "migration_set": str,
+                "name": str,
+                "applied_at": str,
+            },
+            ("migration_set", "name"),
+        ),
+    ),
+)
+def test_upgrades_sqlite_migrations(migrations, create_table, pk):
+    db = sqlite_utils.Database(memory=True)
+    db["_sqlite_migrations"].create(create_table, pk=pk)
+    assert db.table_names() == ["_sqlite_migrations"]
+    assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk))
+    migrations.apply(db)
+    assert db["_sqlite_migrations"].pks == ["id"]
+
+
+def test_pending_and_applied_are_read_only(migrations):
+    db = sqlite_utils.Database(memory=True)
+    assert [m.name for m in migrations.pending(db)] == ["m001", "m002"]
+    assert migrations.applied(db) == []
+    # Neither call should have created the tracking table
+    assert db.table_names() == []
+
+
+def test_duplicate_migration_name_errors():
+    migrations = Migrations("test")
+
+    @migrations()
+    def m001(db):
+        pass
+
+    with pytest.raises(ValueError) as ex:
+
+        @migrations(name="m001")
+        def m001_again(db):
+            pass
+
+    assert "m001" in str(ex.value)
+
+
+def test_stop_before_applied_migration_errors(migrations):
+    # Stopping before a migration that has already been applied is
+    # impossible to honor - previously the stop name was only checked
+    # against pending migrations, so everything after it was applied
+    db = sqlite_utils.Database(memory=True)
+    migrations.apply(db, stop_before="m002")  # applies m001 only
+    with pytest.raises(ValueError) as ex:
+        migrations.apply(db, stop_before="m001")
+    assert "m001" in str(ex.value)
+    assert "already been applied" in str(ex.value)
+    # Nothing else was applied
+    assert not db["cats"].exists()
+
+
+def test_stop_before_applied_migration_errors_before_any_apply(migrations):
+    # The error fires before any pending migration runs, even those that
+    # come before the already-applied stop target in registration order
+    db = sqlite_utils.Database(memory=True)
+    only_second = Migrations("test")
+
+    @only_second()
+    def m002(db):
+        db["cats"].create({"name": str})
+
+    only_second.apply(db)  # m002 applied, m001 still pending
+    with pytest.raises(ValueError):
+        migrations.apply(db, stop_before="m002")
+    assert not db["dogs"].exists()

tests/test_plugins.py

@@ -0,0 +1,128 @@
+from click.testing import CliRunner
+import click
+import importlib
+import pytest
+import sys
+from sqlite_utils import cli, Database, hookimpl, plugins
+
+
+def _supports_pragma_function_list():
+    db = Database(memory=True)
+    try:
+        db.execute("select * from pragma_function_list()")
+        return True
+    except Exception:
+        return False
+    finally:
+        db.close()
+
+
+def test_get_plugins_loads_setuptools_entrypoints_once(monkeypatch):
+    calls = []
+    monkeypatch.delattr(sys, "_called_from_test", raising=False)
+    monkeypatch.setattr(plugins, "_plugins_loaded", False)
+    monkeypatch.setattr(
+        plugins.pm,
+        "load_setuptools_entrypoints",
+        lambda group: calls.append(group) or 0,
+    )
+
+    plugins.get_plugins()
+    plugins.get_plugins()
+
+    assert calls == ["sqlite_utils"]
+
+
+def test_get_plugins_does_not_load_setuptools_entrypoints_in_tests(monkeypatch):
+    calls = []
+    monkeypatch.setattr(sys, "_called_from_test", True, raising=False)
+    monkeypatch.setattr(plugins, "_plugins_loaded", False)
+    monkeypatch.setattr(
+        plugins.pm,
+        "load_setuptools_entrypoints",
+        lambda group: calls.append(group) or 0,
+    )
+
+    assert plugins.get_plugins() == []
+    assert calls == []
+
+
+def test_register_commands():
+    importlib.reload(cli)
+    assert plugins.get_plugins() == []
+
+    class HelloWorldPlugin:
+        __name__ = "HelloWorldPlugin"
+
+        @hookimpl
+        def register_commands(self, cli):
+            @cli.command(name="hello-world")
+            def hello_world():
+                "Print hello world"
+                click.echo("Hello world!")
+
+    try:
+        plugins.pm.register(HelloWorldPlugin(), name="HelloWorldPlugin")
+        importlib.reload(cli)
+
+        assert plugins.get_plugins() == [
+            {"name": "HelloWorldPlugin", "hooks": ["register_commands"]}
+        ]
+
+        runner = CliRunner()
+        result = runner.invoke(cli.cli, ["hello-world"])
+        assert result.exit_code == 0
+        assert result.output == "Hello world!\n"
+
+    finally:
+        plugins.pm.unregister(name="HelloWorldPlugin")
+        importlib.reload(cli)
+        assert plugins.get_plugins() == []
+
+
+@pytest.mark.skipif(
+    not _supports_pragma_function_list(),
+    reason="Needs SQLite version that supports pragma_function_list()",
+)
+def test_prepare_connection():
+    importlib.reload(cli)
+    assert plugins.get_plugins() == []
+
+    class HelloFunctionPlugin:
+        __name__ = "HelloFunctionPlugin"
+
+        @hookimpl
+        def prepare_connection(self, conn):
+            conn.create_function("hello", 1, lambda name: f"Hello, {name}!")
+
+    db = Database(memory=True)
+
+    def _functions(db):
+        return [
+            row[0]
+            for row in db.execute(
+                "select distinct name from pragma_function_list() order by 1"
+            ).fetchall()
+        ]
+
+    assert "hello" not in _functions(db)
+
+    try:
+        plugins.pm.register(HelloFunctionPlugin(), name="HelloFunctionPlugin")
+
+        assert plugins.get_plugins() == [
+            {"name": "HelloFunctionPlugin", "hooks": ["prepare_connection"]}
+        ]
+
+        db = Database(memory=True)
+        assert "hello" in _functions(db)
+        result = db.execute('select hello("world")').fetchone()[0]
+        assert result == "Hello, world!"
+
+        # Test execute_plugins=False
+        db2 = Database(memory=True, execute_plugins=False)
+        assert "hello" not in _functions(db2)
+
+    finally:
+        plugins.pm.unregister(name="HelloFunctionPlugin")
+        assert plugins.get_plugins() == []

tests/test_query.py

@@ -0,0 +1,303 @@
+import pytest
+import types
+
+from sqlite_utils.utils import sqlite3
+
+
+def test_query(fresh_db):
+    fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}])
+    results = fresh_db.query("select * from dogs order by name desc")
+    assert isinstance(results, types.GeneratorType)
+    assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}]
+
+
+def test_query_executes_eagerly(fresh_db):
+    # The SQL runs when query() is called, not when the result is iterated,
+    # so errors are raised at the call site
+    with pytest.raises(sqlite3.OperationalError):
+        fresh_db.query("select * from missing_table")
+
+
+def test_query_rejects_statements_that_return_no_rows(fresh_db):
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    with pytest.raises(ValueError) as ex:
+        fresh_db.query("update dogs set name = 'Cleopaws'")
+    assert "execute()" in str(ex.value)
+    # The rejected update was rolled back, and no transaction is left open
+    assert not fresh_db.conn.in_transaction
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
+
+
+def test_query_rejected_ddl_is_rolled_back(fresh_db):
+    with pytest.raises(ValueError):
+        fresh_db.query("create table dogs (id integer primary key)")
+    assert not fresh_db.conn.in_transaction
+    assert fresh_db.table_names() == []
+
+
+def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db):
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    fresh_db.begin()
+    fresh_db.execute("insert into dogs (name) values ('Pancakes')")
+    with pytest.raises(ValueError):
+        fresh_db.query("update dogs set name = 'Cleopaws'")
+    # The transaction is still open and the earlier insert is intact
+    assert fresh_db.conn.in_transaction
+    fresh_db.commit()
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo", "Pancakes"]
+
+
+@pytest.mark.parametrize(
+    "sql",
+    [
+        "begin",
+        "commit",
+        "rollback",
+        "vacuum",
+        "detach database foo",
+        "/* comment */ commit",
+        "-- comment\nbegin",
+        "/* multi\nline */ -- and another\n  vacuum",
+        "\t /* a */ /* b */ savepoint s1",
+        "; commit",
+        ";;\n ; rollback",
+        "; /* comment */ vacuum",
+        "\ufeffbegin",
+    ],
+)
+def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql):
+    with pytest.raises(ValueError) as ex:
+        fresh_db.query(sql)
+    assert "execute()" in str(ex.value)
+    assert not fresh_db.conn.in_transaction
+
+
+def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db):
+    # A COMMIT hidden behind a leading comment must not slip past the
+    # keyword check - previously it committed the caller's open
+    # transaction before the ValueError was raised
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    fresh_db.begin()
+    fresh_db.execute("insert into dogs (name) values ('Pancakes')")
+    with pytest.raises(ValueError):
+        fresh_db.query("/* comment */ COMMIT")
+    # The explicit transaction is still open and can still be rolled back
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
+
+
+@pytest.mark.parametrize("sql", ["; COMMIT", "\ufeffCOMMIT"])
+def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql):
+    # sqlite3 tolerates empty statements and a UTF-8 BOM before the first
+    # real token, so the keyword scanner must skip them too - previously
+    # '; COMMIT' slipped past the check and committed the caller's open
+    # transaction before raising OperationalError
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    fresh_db.begin()
+    fresh_db.execute("insert into dogs (name) values ('Pancakes')")
+    with pytest.raises(ValueError):
+        fresh_db.query(sql)
+    # The explicit transaction is still open and can still be rolled back
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
+
+
+def test_query_error_leaves_no_transaction_open(fresh_db):
+    with pytest.raises(sqlite3.OperationalError):
+        fresh_db.query("select * from missing_table")
+    assert not fresh_db.conn.in_transaction
+
+
+def test_query_pragma(tmpdir):
+    from sqlite_utils import Database
+
+    db = Database(str(tmpdir / "test.db"))
+    # A row-returning PRAGMA works, including one that cannot run in a transaction
+    assert list(db.query("pragma journal_mode = wal")) == [{"journal_mode": "wal"}]
+    # A PRAGMA that returns no rows raises ValueError
+    with pytest.raises(ValueError):
+        db.query("pragma user_version = 5")
+    db.close()
+
+
+def test_query_rejected_pragma_still_takes_effect(fresh_db):
+    # Documented limitation: PRAGMAs run outside the savepoint guard,
+    # because some of them refuse to run inside a transaction - so a
+    # row-less PRAGMA takes effect even though it raises ValueError.
+    # If this test starts failing because the pragma was rolled back,
+    # the limitation has been fixed - update the docs in python-api.rst
+    # and the query() docstring to remove the carve-out
+    with pytest.raises(ValueError):
+        fresh_db.query("pragma user_version = 5")
+    assert fresh_db.execute("pragma user_version").fetchone()[0] == 5
+
+
+def test_query_comment_prefixed_pragma(tmpdir):
+    from sqlite_utils import Database
+
+    db = Database(str(tmpdir / "test.db"))
+    # A leading comment must not stop a PRAGMA being recognized as one -
+    # previously it was executed inside the savepoint guard, where
+    # journal mode changes are refused
+    assert list(db.query("-- set WAL mode\npragma journal_mode = wal")) == [
+        {"journal_mode": "wal"}
+    ]
+    db.close()
+
+
+def test_query_comment_prefixed_pragma_inside_transaction(fresh_db):
+    fresh_db.begin()
+    assert list(fresh_db.query("-- check version\npragma user_version")) == [
+        {"user_version": 0}
+    ]
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+
+
+@pytest.mark.parametrize(
+    "sql,expected",
+    [
+        ("select 1", "SELECT"),
+        ("  \t\n select 1", "SELECT"),
+        ("-- comment\nbegin", "BEGIN"),
+        ("/* one */ /* two */ pragma user_version", "PRAGMA"),
+        ("/* multi\nline */vacuum", "VACUUM"),
+        ("insert into t values (1)", "INSERT"),
+        ("-- only a comment", ""),
+        ("/* unterminated", ""),
+        ("", ""),
+        ("   ", ""),
+        ("123", ""),
+        ("; commit", "COMMIT"),
+        (";;\n ; rollback", "ROLLBACK"),
+        ("; -- comment\n begin", "BEGIN"),
+        ("\ufeffcommit", "COMMIT"),
+        ("\ufeff ; select 1", "SELECT"),
+        (";", ""),
+    ],
+)
+def test_first_keyword(sql, expected):
+    from sqlite_utils.db import _first_keyword
+
+    assert _first_keyword(sql) == expected
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_insert_returning(fresh_db):
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    rows = list(
+        fresh_db.query("insert into dogs (name) values ('Pancakes') returning name")
+    )
+    assert rows == [{"name": "Pancakes"}]
+    assert fresh_db["dogs"].count == 2
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_insert_returning_commits_without_iteration(tmpdir):
+    from sqlite_utils import Database
+
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["dogs"].insert({"name": "Cleo"})
+    # Never iterate over the results
+    db.query("insert into dogs (name) values ('Pancakes') returning name")
+    assert not db.conn.in_transaction
+    # A completely separate connection sees the new row straight away
+    other = sqlite3.connect(path)
+    assert other.execute("select count(*) from dogs").fetchone()[0] == 2
+    other.close()
+    db.close()
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_insert_returning_partial_iteration_still_commits(tmpdir):
+    from sqlite_utils import Database
+
+    path = str(tmpdir / "test.db")
+    db = Database(path)
+    db["dogs"].insert({"name": "Cleo"})
+    row = next(
+        db.query(
+            "insert into dogs (name) values ('Pancakes'), ('Marnie') returning name"
+        )
+    )
+    assert row == {"name": "Pancakes"}
+    assert not db.conn.in_transaction
+    other = sqlite3.connect(path)
+    assert other.execute("select count(*) from dogs").fetchone()[0] == 3
+    other.close()
+    db.close()
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_insert_returning_respects_explicit_transaction(fresh_db):
+    fresh_db["dogs"].insert({"name": "Cleo"})
+    fresh_db.begin()
+    rows = list(
+        fresh_db.query("insert into dogs (name) values ('Pancakes') returning name")
+    )
+    assert rows == [{"name": "Pancakes"}]
+    # Still inside the explicit transaction - not committed
+    assert fresh_db.conn.in_transaction
+    fresh_db.rollback()
+    assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
+
+
+def test_query_duplicate_column_names_are_deduped(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/624
+    fresh_db["one"].insert({"id": 1, "value": "left"})
+    fresh_db["two"].insert({"id": 2, "value": "right"})
+    rows = list(
+        fresh_db.query("select one.id, two.id, one.value, two.value from one, two")
+    )
+    assert rows == [{"id": 1, "id_2": 2, "value": "left", "value_2": "right"}]
+
+
+def test_query_deduped_column_avoids_existing_names(fresh_db):
+    # The renamed duplicate must not overwrite a real column called id_2
+    rows = list(fresh_db.query("select 1 as id, 2 as id, 3 as id_2"))
+    assert rows == [{"id": 1, "id_3": 2, "id_2": 3}]
+
+
+def test_execute_returning_dicts(fresh_db):
+    # Like db.query() but returns a list, included for backwards compatibility
+    # see https://github.com/simonw/sqlite-utils/issues/290
+    fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id")
+    assert fresh_db.execute_returning_dicts("select * from test") == [
+        {"id": 1, "bar": 2}
+    ]
+
+
+@pytest.mark.skipif(
+    sqlite3.sqlite_version_info < (3, 35, 0),
+    reason="RETURNING requires SQLite 3.35.0 or higher",
+)
+def test_query_preserves_error_from_transaction_destroying_trigger(fresh_db):
+    # RAISE(ROLLBACK) destroys the savepoint guard - the original
+    # IntegrityError must propagate, not "no such savepoint"
+    fresh_db.execute("create table t (id integer primary key, v text)")
+    fresh_db.execute("""
+        create trigger no_bad before insert on t
+        when new.v = 'bad'
+        begin
+            select raise(rollback, 'trigger says no');
+        end
+    """)
+    with pytest.raises(sqlite3.IntegrityError, match="trigger says no"):
+        fresh_db.query("insert into t (id, v) values (1, 'bad') returning id")
+    assert not fresh_db.conn.in_transaction
+    assert fresh_db.execute("select count(*) from t").fetchone()[0] == 0

tests/test_recipes.py

@@ -0,0 +1,146 @@
+from sqlite_utils import recipes
+from sqlite_utils.utils import sqlite3
+import json
+import pytest
+
+
+@pytest.fixture
+def dates_db(fresh_db):
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "dt": "5th October 2019 12:04"},
+            {"id": 2, "dt": "6th October 2019 00:05:06"},
+            {"id": 3, "dt": ""},
+            {"id": 4, "dt": None},
+        ],
+        pk="id",
+    )
+    return fresh_db
+
+
+def test_parsedate(dates_db):
+    dates_db["example"].convert("dt", recipes.parsedate)
+    assert list(dates_db["example"].rows) == [
+        {"id": 1, "dt": "2019-10-05"},
+        {"id": 2, "dt": "2019-10-06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ]
+
+
+def test_parsedatetime(dates_db):
+    dates_db["example"].convert("dt", recipes.parsedatetime)
+    assert list(dates_db["example"].rows) == [
+        {"id": 1, "dt": "2019-10-05T12:04:00"},
+        {"id": 2, "dt": "2019-10-06T00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ]
+
+
+@pytest.mark.parametrize(
+    "recipe,kwargs,expected",
+    (
+        ("parsedate", {}, "2005-03-04"),
+        ("parsedate", {"dayfirst": True}, "2005-04-03"),
+        ("parsedatetime", {}, "2005-03-04T00:00:00"),
+        ("parsedatetime", {"dayfirst": True}, "2005-04-03T00:00:00"),
+    ),
+)
+def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected):
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "dt": "03/04/05"},
+        ],
+        pk="id",
+    )
+    fresh_db["example"].convert(
+        "dt", lambda value: getattr(recipes, recipe)(value, **kwargs)
+    )
+    assert list(fresh_db["example"].rows) == [
+        {"id": 1, "dt": expected},
+    ]
+
+
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
+@pytest.mark.parametrize("fn", ("parsedate", "parsedatetime"))
+def test_dateparse_errors_raises(fresh_db, fn):
+    """Test that invalid dates raise errors when errors=None"""
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "dt": "invalid"},
+        ],
+        pk="id",
+    )
+    # Exception in SQLite callback surfaces as OperationalError
+    with pytest.raises(sqlite3.OperationalError):
+        fresh_db["example"].convert("dt", lambda value: getattr(recipes, fn)(value))
+
+
+@pytest.mark.parametrize("fn", ("parsedate", "parsedatetime"))
+@pytest.mark.parametrize("errors", (recipes.SET_NULL, recipes.IGNORE))
+def test_dateparse_errors_handled(fresh_db, fn, errors):
+    """Test error handling modes for invalid dates"""
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "dt": "invalid"},
+        ],
+        pk="id",
+    )
+    fresh_db["example"].convert(
+        "dt", lambda value: getattr(recipes, fn)(value, errors=errors)
+    )
+    rows = list(fresh_db["example"].rows)
+    expected = [{"id": 1, "dt": None if errors is recipes.SET_NULL else "invalid"}]
+    assert rows == expected
+
+
+@pytest.mark.parametrize("delimiter", [None, ";", "-"])
+def test_jsonsplit(fresh_db, delimiter):
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])},
+            {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])},
+        ],
+        pk="id",
+    )
+    if delimiter is not None:
+
+        def fn(value):
+            return recipes.jsonsplit(value, delimiter=delimiter)
+
+    else:
+        fn = recipes.jsonsplit
+
+    fresh_db["example"].convert("tags", fn)
+    assert list(fresh_db["example"].rows) == [
+        {"id": 1, "tags": '["foo", "bar"]'},
+        {"id": 2, "tags": '["bar", "baz"]'},
+    ]
+
+
+@pytest.mark.parametrize(
+    "type,expected",
+    (
+        (None, ["1", "2", "3"]),
+        (float, [1.0, 2.0, 3.0]),
+        (int, [1, 2, 3]),
+    ),
+)
+def test_jsonsplit_type(fresh_db, type, expected):
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "records": "1,2,3"},
+        ],
+        pk="id",
+    )
+    if type is not None:
+
+        def fn(value):
+            return recipes.jsonsplit(value, type=type)
+
+    else:
+        fn = recipes.jsonsplit
+
+    fresh_db["example"].convert("records", fn)
+    assert json.loads(fresh_db["example"].get(1)["records"]) == expected

tests/test_recreate.py

@@ -14,19 +14,25 @@ def test_recreate_ignored_for_in_memory():
 
 def test_recreate_not_allowed_for_connection():
     conn = sqlite3.connect(":memory:")
-    with pytest.raises(AssertionError):
+    try:
-        db = Database(conn, recreate=True)
+        with pytest.raises(ValueError):
+            Database(conn, recreate=True)
+    finally:
+        conn.close()
 
 
 @pytest.mark.parametrize(
-    "use_path,file_exists", [(True, True), (True, False), (False, True), (False, False)]
+    "use_path,create_file_first",
+    [(True, True), (True, False), (False, True), (False, False)],
 )
-def test_recreate(tmpdir, use_path, file_exists):
+def test_recreate(tmp_path, use_path, create_file_first):
-    filepath = str(tmpdir / "data.db")
+    filepath = str(tmp_path / "data.db")
     if use_path:
         filepath = pathlib.Path(filepath)
-    if file_exists:
+    if create_file_first:
-        Database(filepath)["t1"].insert({"foo": "bar"})
+        db = Database(filepath)
-        assert ["t1"] == Database(filepath).table_names()
+        db["t1"].insert({"foo": "bar"})
+        assert ["t1"] == db.table_names()
+        db.close()
     Database(filepath, recreate=True)["t2"].insert({"foo": "bar"})
     assert ["t2"] == Database(filepath).table_names()

tests/test_register_function.py

@@ -1,6 +1,8 @@
+# flake8: noqa
 import pytest
 import sys
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, call
+from sqlite_utils.utils import sqlite3
 
 
 def test_register_function(fresh_db):
@@ -12,6 +14,15 @@ def test_register_function(fresh_db):
     assert result == "olleh"
 
 
+def test_register_function_custom_name(fresh_db):
+    @fresh_db.register_function(name="revstr")
+    def reverse_string(s):
+        return "".join(reversed(list(s)))
+
+    result = fresh_db.execute('select revstr("hello")').fetchone()[0]
+    assert result == "olleh"
+
+
 def test_register_function_multiple_arguments(fresh_db):
     @fresh_db.register_function
     def a_times_b_plus_c(a, b, c):
@@ -30,20 +41,47 @@ def test_register_function_deterministic(fresh_db):
     assert result == "bob"
 
 
-@pytest.mark.skipif(
+def test_register_function_deterministic_tries_again_if_exception_raised(fresh_db):
-    sys.version_info < (3, 8), reason="deterministic=True was added in Python 3.8"
+    # Save the original connection so we can close it later
-)
+    original_conn = fresh_db.conn
-def test_register_function_deterministic_registered(fresh_db):
     fresh_db.conn = MagicMock()
     fresh_db.conn.create_function = MagicMock()
 
-    @fresh_db.register_function(deterministic=True)
+    try:
-    def to_lower_2(s):
-        return s.lower()
 
-    fresh_db.conn.create_function.assert_called_with(
+        @fresh_db.register_function(deterministic=True)
-        "to_lower_2", 1, to_lower_2, deterministic=True
+        def to_lower_2(s):
-    )
+            return s.lower()
+
+        fresh_db.conn.create_function.assert_called_with(
+            "to_lower_2", 1, to_lower_2, deterministic=True
+        )
+
+        first = True
+
+        def side_effect(*args, **kwargs):
+            # Raise exception only first time this is called
+            nonlocal first
+            if first:
+                first = False
+                raise sqlite3.NotSupportedError()
+
+        # But if sqlite3.NotSupportedError is raised, it tries again
+        fresh_db.conn.create_function.reset_mock()
+        fresh_db.conn.create_function.side_effect = side_effect
+
+        @fresh_db.register_function(deterministic=True)
+        def to_lower_3(s):
+            return s.lower()
+
+        # Should have been called once with deterministic=True and once without
+        assert fresh_db.conn.create_function.call_args_list == [
+            call("to_lower_3", 1, to_lower_3, deterministic=True),
+            call("to_lower_3", 1, to_lower_3),
+        ]
+    finally:
+        # Close the original connection that was replaced with the mock
+        original_conn.close()
 
 
 def test_register_function_replace(fresh_db):
@@ -53,16 +91,16 @@ def test_register_function_replace(fresh_db):
 
     assert "one" == fresh_db.execute("select one()").fetchone()[0]
 
-    # This will fail to replace the function:
+    # This will silently fail to replaec the function
     @fresh_db.register_function()
-    def one():
+    def one():  # noqa
         return "two"
 
     assert "one" == fresh_db.execute("select one()").fetchone()[0]
 
     # This will replace it
     @fresh_db.register_function(replace=True)
-    def one():
+    def one():  # noqa
         return "two"
 
     assert "two" == fresh_db.execute("select one()").fetchone()[0]

tests/test_rows.py

@@ -1,4 +1,3 @@
-from sqlite_utils.db import Index, View
 import pytest
 
 
@@ -105,3 +104,37 @@ def test_pks_and_rows_where_compound_pk(fresh_db):
         (("number", 1), {"type": "number", "number": 1, "plusone": 2}),
         (("number", 2), {"type": "number", "number": 2, "plusone": 3}),
     ]
+
+
+def test_rows_where_duplicate_select_columns_are_deduped(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/624
+    fresh_db["t"].insert({"id": 1, "name": "Cleo"})
+    rows = list(fresh_db["t"].rows_where(select="id, id, name"))
+    assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}]
+
+
+def test_pks_and_rows_where_view(fresh_db):
+    # pks_and_rows_where() lives on Queryable so views expose it, but
+    # SQLite views have no rowid. Modern SQLite (3.36+) raises an
+    # OperationalError from the generated SQL; older versions returned
+    # NULL for a view's rowid. Either way it must not fail earlier with
+    # an AttributeError from View lacking Table-only properties
+    from sqlite_utils.utils import sqlite3
+
+    fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db.create_view("dog_names", "select name from dogs")
+    try:
+        result = list(fresh_db["dog_names"].pks_and_rows_where())
+    except sqlite3.OperationalError:
+        pass  # SQLite 3.36+: no such column: rowid
+    else:
+        # Older SQLite returns NULL rowids for views
+        assert result == [(None, {"rowid": None, "name": "Cleo"})]
+
+
+def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db):
+    # Compound pks are returned in PRIMARY KEY declaration order
+    fresh_db.execute("create table t (b text, a text, primary key (a, b))")
+    fresh_db["t"].insert({"a": "A", "b": "B"})
+    pks_and_rows = list(fresh_db["t"].pks_and_rows_where())
+    assert pks_and_rows == [(("A", "B"), {"b": "B", "a": "A"})]

tests/test_rows_from_file.py

@@ -0,0 +1,54 @@
+from sqlite_utils.utils import rows_from_file, Format, RowError
+from io import BytesIO, StringIO
+import pytest
+
+
+@pytest.mark.parametrize(
+    "input,expected_format",
+    (
+        (b"id,name\n1,Cleo", Format.CSV),
+        (b"id\tname\n1\tCleo", Format.TSV),
+        (b'[{"id": "1", "name": "Cleo"}]', Format.JSON),
+    ),
+)
+def test_rows_from_file_detect_format(input, expected_format):
+    rows, format = rows_from_file(BytesIO(input))
+    assert format == expected_format
+    rows_list = list(rows)
+    assert rows_list == [{"id": "1", "name": "Cleo"}]
+
+
+@pytest.mark.parametrize(
+    "ignore_extras,extras_key,expected",
+    (
+        (True, None, [{"id": "1", "name": "Cleo"}]),
+        (False, "_rest", [{"id": "1", "name": "Cleo", "_rest": ["oops"]}]),
+        # expected of None means expect an error:
+        (False, False, None),
+    ),
+)
+def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expected):
+    try:
+        rows, format = rows_from_file(
+            BytesIO(b"id,name\r\n1,Cleo,oops"),
+            format=Format.CSV,
+            ignore_extras=ignore_extras,
+            extras_key=extras_key,
+        )
+        list_rows = list(rows)
+    except RowError:
+        if expected is None:
+            # This is fine,
+            return
+        else:
+            # We did not expect an error
+            raise
+    assert list_rows == expected
+
+
+def test_rows_from_file_error_on_string_io():
+    with pytest.raises(TypeError) as ex:
+        rows_from_file(StringIO("id,name\r\n1,Cleo"))  # type: ignore[arg-type]
+    assert ex.value.args == (
+        "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO",
+    )

tests/test_sniff.py

@@ -6,13 +6,13 @@ import pytest
 sniff_dir = pathlib.Path(__file__).parent / "sniff"
 
 
-@pytest.mark.parametrize("filepath", sniff_dir.glob("example*"))
+@pytest.mark.parametrize("filepath", sorted(sniff_dir.glob("example*")))
 def test_sniff(tmpdir, filepath):
     db_path = str(tmpdir / "test.db")
     runner = CliRunner()
     result = runner.invoke(
         cli.cli,
-        ["insert", db_path, "creatures", str(filepath), "--sniff"],
+        ["insert", db_path, "creatures", str(filepath), "--sniff", "--no-detect-types"],
         catch_exceptions=False,
     )
     assert result.exit_code == 0, result.stdout

tests/test_tracer.py

@@ -1,4 +1,3 @@
-import pytest
 from sqlite_utils import Database
 
 
@@ -7,58 +6,91 @@ def test_tracer():
     db = Database(
         memory=True, tracer=lambda sql, params: collected.append((sql, params))
     )
-    db["dogs"].insert({"name": "Cleopaws"})
+    dogs = db.table("dogs")
-    db["dogs"].enable_fts(["name"])
+    dogs.insert({"name": "Cleopaws"})
-    db["dogs"].search("Cleopaws")
+    dogs.enable_fts(["name"])
+    dogs.search("Cleopaws")
     assert collected == [
         ("PRAGMA recursive_triggers=on;", None),
         ("select name from sqlite_master where type = 'view'", None),
         ("select name from sqlite_master where type = 'table'", None),
-        ("CREATE TABLE [dogs] (\n   [name] TEXT\n);\n        ", None),
         ("select name from sqlite_master where type = 'view'", None),
-        ("INSERT INTO [dogs] ([name]) VALUES (?);", ["Cleopaws"]),
         ("select name from sqlite_master where type = 'view'", None),
+        ("select name from sqlite_master where type = 'table'", None),
+        ("select name from sqlite_master where type = 'view'", None),
+        ('CREATE TABLE "dogs" (\n   "name" TEXT\n);\n        ', None),
+        ("select name from sqlite_master where type = 'view'", None),
+        ('INSERT INTO "dogs" ("name") VALUES (?)', ["Cleopaws"]),
         (
-            "CREATE VIRTUAL TABLE [dogs_fts] USING FTS5 (\n    [name],\n    content=[dogs]\n)",
+            'CREATE VIRTUAL TABLE "dogs_fts" USING FTS5 (\n    "name",\n    content="dogs"\n)',
             None,
         ),
         (
-            "INSERT INTO [dogs_fts] (rowid, [name])\n    SELECT rowid, [name] FROM [dogs];",
+            'INSERT INTO "dogs_fts" (rowid, "name")\n    SELECT rowid, "name" FROM "dogs";',
             None,
         ),
-        ("select name from sqlite_master where type = 'view'", None),
     ]
 
 
 def test_with_tracer():
     collected = []
-    tracer = lambda sql, params: collected.append((sql, params))
+
+    def tracer(sql, params):
+        return collected.append((sql, params))
 
     db = Database(memory=True)
 
-    db["dogs"].insert({"name": "Cleopaws"})
+    dogs = db.table("dogs")
-    db["dogs"].enable_fts(["name"])
+
+    dogs.insert({"name": "Cleopaws"})
+    dogs.enable_fts(["name"])
 
     assert len(collected) == 0
 
     with db.tracer(tracer):
-        list(db["dogs"].search("Cleopaws"))
+        list(dogs.search("Cleopaws"))
 
-    assert len(collected) == 5
+    assert len(collected) == 4
     assert collected == [
-        ("select name from sqlite_master where type = 'view'", None),
         (
-            "SELECT name FROM sqlite_master\n    WHERE rootpage = 0\n    AND (\n        sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n        OR (\n            tbl_name = \"dogs\"\n            AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n        )\n    )",
+            "SELECT name FROM sqlite_master\n"
-            None,
+            "    WHERE rootpage = 0\n"
+            "    AND (\n"
+            "        sql LIKE :like\n"
+            "        OR sql LIKE :like2\n"
+            "        OR (\n"
+            "            tbl_name = :table\n"
+            "            AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n"
+            "        )\n"
+            "    )",
+            {
+                "like": "%VIRTUAL TABLE%USING FTS%content=[dogs]%",
+                "like2": '%VIRTUAL TABLE%USING FTS%content="dogs"%',
+                "table": "dogs",
+            },
         ),
         ("select name from sqlite_master where type = 'view'", None),
         ("select sql from sqlite_master where name = ?", ("dogs_fts",)),
         (
-            "with original as (\n    select\n        rowid,\n        *\n    from [dogs]\n)\nselect\n    [original].*\nfrom\n    [original]\n    join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n    [dogs_fts] match :query\norder by\n    [dogs_fts].rank",
+            'with "original" as (\n'
+            "    select\n"
+            "        rowid,\n"
+            "        *\n"
+            '    from "dogs"\n'
+            ")\n"
+            "select\n"
+            '    "original".*\n'
+            "from\n"
+            '    "original"\n'
+            '    join "dogs_fts" on "original".rowid = "dogs_fts".rowid\n'
+            "where\n"
+            '    "dogs_fts" match :query\n'
+            "order by\n"
+            '    "dogs_fts".rank',
             {"query": "Cleopaws"},
         ),
     ]
 
     # Outside the with block collected should not be appended to
-    db["dogs"].insert({"name": "Cleopaws"})
+    dogs.insert({"name": "Cleopaws"})
-    assert len(collected) == 5
+    assert len(collected) == 4

tests/test_transform.py

@@ -1,4 +1,6 @@
-from sqlite_utils.db import ForeignKey
+import sqlite3
+
+from sqlite_utils.db import ForeignKey, TransactionError, TransformError
 from sqlite_utils.utils import OperationalError
 import pytest
 
@@ -10,80 +12,90 @@ import pytest
         (
             {},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] TEXT\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Change column type
         (
             {"types": {"age": int}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] INTEGER\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" INTEGER\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Rename a column
         (
             {"rename": {"age": "dog_age"}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [dog_age] TEXT\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" TEXT\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Drop a column
         (
             {"drop": ["age"]},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name])\n   SELECT [id], [name] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n   SELECT "rowid", "id", "name" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Convert type AND rename column
         (
             {"types": {"age": int}, "rename": {"age": "dog_age"}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [dog_age] INTEGER\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" INTEGER\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Change primary key
         (
             {"pk": "age"},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT PRIMARY KEY\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT PRIMARY KEY\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Change primary key to a compound pk
         (
             {"pk": ("age", "name")},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT,\n   PRIMARY KEY ([age], [name])\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT,\n   PRIMARY KEY ("age", "name")\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Remove primary key, creating a rowid table
         (
             {"pk": None},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+            ],
+        ),
+        # Keeping the table
+        (
+            {"drop": ["age"], "keep_table": "kept_table"},
+            [
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n);',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n   SELECT "rowid", "id", "name" FROM "dogs";',
+                'ALTER TABLE "dogs" RENAME TO "kept_table";',
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
     ],
@@ -93,7 +105,10 @@ def test_transform_sql_table_with_primary_key(
     fresh_db, params, expected_sql, use_pragma_foreign_keys
 ):
     captured = []
-    tracer = lambda sql, params: captured.append((sql, params))
+
+    def tracer(sql, params):
+        return captured.append((sql, params))
+
     dogs = fresh_db["dogs"]
     if use_pragma_foreign_keys:
         fresh_db.conn.execute("PRAGMA foreign_keys=ON")
@@ -120,40 +135,40 @@ def test_transform_sql_table_with_primary_key(
         (
             {},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Change column type
         (
             {"types": {"age": int}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] INTEGER\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" INTEGER\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Rename a column
         (
             {"rename": {"age": "dog_age"}},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [dog_age] TEXT\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "dog_age" TEXT\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
         # Make ID a primary key
         (
             {"pk": "id"},
             [
-                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] TEXT\n);",
+                'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n);',
-                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
-                "DROP TABLE [dogs];",
+                'DROP TABLE "dogs";',
-                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+                'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
         ),
     ],
@@ -163,7 +178,10 @@ def test_transform_sql_table_with_no_primary_key(
     fresh_db, params, expected_sql, use_pragma_foreign_keys
 ):
     captured = []
-    tracer = lambda sql, params: captured.append((sql, params))
+
+    def tracer(sql, params):
+        return captured.append((sql, params))
+
     dogs = fresh_db["dogs"]
     if use_pragma_foreign_keys:
         fresh_db.conn.execute("PRAGMA foreign_keys=ON")
@@ -188,13 +206,13 @@ def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db):
     dogs.insert({"id": 1, "name": "Cleo", "age": "5"})
     assert (
         dogs.schema
-        == "CREATE TABLE [dogs] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT\n)"
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT\n)'
     )
     dogs.transform(pk="id")
     # Slight oddity: [dogs] becomes "dogs" during the rename:
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] TEXT\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n)'
     )
 
 
@@ -204,17 +222,51 @@ def test_transform_rename_pk(fresh_db):
     dogs.transform(rename={"id": "pk"})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [pk] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] TEXT\n)'
+        == 'CREATE TABLE "dogs" (\n   "pk" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n)'
     )
 
 
+def test_transform_preserves_keyword_literal_defaults(fresh_db):
+    # transform() used to requote keyword-literal defaults (DEFAULT TRUE became
+    # DEFAULT 'TRUE'), so a default insert stored the text 'TRUE' instead of the
+    # integer 1 -- silent value corruption on every rebuilt table.
+    fresh_db.execute(
+        "CREATE TABLE t ("
+        " id INTEGER PRIMARY KEY,"
+        " is_active INTEGER DEFAULT TRUE,"
+        " flag INTEGER DEFAULT FALSE,"
+        " note TEXT DEFAULT NULL"
+        ")"
+    )
+    table = fresh_db["t"]
+    table.insert({"id": 1})
+    before = fresh_db.execute("SELECT is_active, flag, note FROM t").fetchone()
+    assert before == (1, 0, None)
+
+    # Rebuild the table via an unrelated change.
+    table.transform(rename={"note": "note2"})
+
+    # The keyword literals stay unquoted in the schema ...
+    assert "DEFAULT TRUE" in table.schema
+    assert "DEFAULT FALSE" in table.schema
+    assert "DEFAULT NULL" in table.schema
+    assert "'TRUE'" not in table.schema
+
+    # ... and a fresh default insert still yields 1 / 0 / NULL, not strings.
+    table.insert({"id": 2})
+    after = fresh_db.execute(
+        "SELECT is_active, flag, note2 FROM t WHERE id = 2"
+    ).fetchone()
+    assert after == (1, 0, None)
+
+
 def test_transform_not_null(fresh_db):
     dogs = fresh_db["dogs"]
     dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id")
     dogs.transform(not_null={"name"})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT NOT NULL,\n   [age] TEXT\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT NOT NULL,\n   "age" TEXT\n)'
     )
 
 
@@ -224,7 +276,7 @@ def test_transform_remove_a_not_null(fresh_db):
     dogs.transform(not_null={"name": True, "age": False})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT NOT NULL,\n   [age] TEXT\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT NOT NULL,\n   "age" TEXT\n)'
     )
 
 
@@ -235,7 +287,7 @@ def test_transform_add_not_null_with_rename(fresh_db, not_null):
     dogs.transform(not_null=not_null, rename={"age": "dog_age"})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [dog_age] TEXT NOT NULL\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" TEXT NOT NULL\n)'
     )
 
 
@@ -245,7 +297,7 @@ def test_transform_defaults(fresh_db):
     dogs.transform(defaults={"age": 1})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] INTEGER DEFAULT 1\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" INTEGER DEFAULT 1\n)'
     )
 
 
@@ -255,7 +307,7 @@ def test_transform_defaults_and_rename_column(fresh_db):
     dogs.transform(rename={"age": "dog_age"}, defaults={"age": 1})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [dog_age] INTEGER DEFAULT 1\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" INTEGER DEFAULT 1\n)'
     )
 
 
@@ -265,7 +317,7 @@ def test_remove_defaults(fresh_db):
     dogs.transform(defaults={"age": None})
     assert (
         dogs.schema
-        == 'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] INTEGER\n)'
+        == 'CREATE TABLE "dogs" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" INTEGER\n)'
     )
 
 
@@ -313,22 +365,29 @@ def test_transform_foreign_keys_survive_renamed_column(
     ]
 
 
+def _add_country_city_continent(db):
+    db["country"].insert({"id": 1, "name": "France"}, pk="id")
+    db["continent"].insert({"id": 2, "name": "Europe"}, pk="id")
+    db["city"].insert({"id": 24, "name": "Paris"}, pk="id")
+
+
+_CAVEAU = {
+    "id": 32,
+    "name": "Caveau de la Huchette",
+    "country": 1,
+    "continent": 2,
+    "city": 24,
+}
+
+
 @pytest.mark.parametrize("use_pragma_foreign_keys", [False, True])
 def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys):
     if use_pragma_foreign_keys:
         fresh_db.conn.execute("PRAGMA foreign_keys=ON")
     # Create table with three foreign keys so we can drop two of them
-    fresh_db["country"].insert({"id": 1, "name": "France"}, pk="id")
+    _add_country_city_continent(fresh_db)
-    fresh_db["continent"].insert({"id": 2, "name": "Europe"}, pk="id")
-    fresh_db["city"].insert({"id": 24, "name": "Paris"}, pk="id")
     fresh_db["places"].insert(
-        {
+        _CAVEAU,
-            "id": 32,
-            "name": "Caveau de la Huchette",
-            "country": 1,
-            "continent": 2,
-            "city": 24,
-        },
         foreign_keys=("country", "continent", "city"),
     )
     assert fresh_db["places"].foreign_keys == [
@@ -368,6 +427,480 @@ def test_transform_verify_foreign_keys(fresh_db):
     # This should have rolled us back
     assert (
         fresh_db["authors"].schema
-        == "CREATE TABLE [authors] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT\n)"
+        == 'CREATE TABLE "authors" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n)'
     )
     assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True])
+def test_transform_on_delete_cascade_does_not_delete_records(
+    fresh_db, use_pragma_foreign_keys
+):
+    # Transforming a table drops and recreates it - if another table references
+    # it with ON DELETE CASCADE and PRAGMA foreign_keys is on, that drop must
+    # not cascade and delete the referencing records
+    if use_pragma_foreign_keys:
+        fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE
+        );
+        """)
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    # Transform the table on the other end of the cascading foreign key
+    fresh_db["authors"].transform(rename={"name": "author_name"})
+    assert list(fresh_db["authors"].rows) == [
+        {"id": 1, "author_name": "Ursula K. Le Guin"}
+    ]
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "The Dispossessed", "author_id": 1}
+    ]
+    # Transforming the table with the cascading foreign key should not
+    # delete its records either
+    fresh_db["books"].transform(rename={"title": "book_title"})
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "book_title": "The Dispossessed", "author_id": 1}
+    ]
+    if use_pragma_foreign_keys:
+        assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+@pytest.mark.parametrize("on_delete", ["CASCADE", "SET NULL", "SET DEFAULT", "cascade"])
+def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_delete):
+    # PRAGMA foreign_keys is a no-op inside a transaction, so transforming a
+    # table referenced by ON DELETE CASCADE / SET NULL / SET DEFAULT foreign
+    # keys inside an open transaction would fire those actions when the old
+    # table is dropped - transform() should refuse instead
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id) ON DELETE {}
+        );
+        """.format(on_delete))
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    previous_schema = fresh_db["authors"].schema
+    with fresh_db.atomic():
+        with pytest.raises(TransactionError) as excinfo:
+            fresh_db["authors"].transform(rename={"name": "author_name"})
+    message = str(excinfo.value)
+    assert "books" in message
+    assert "ON DELETE {}".format(on_delete.upper()) in message
+    # Nothing should have changed
+    assert fresh_db["authors"].schema == previous_schema
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "The Dispossessed", "author_id": 1}
+    ]
+    assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db):
+    # The copied table carries a foreign key referencing the original table
+    # name, so a self-referential cascade would wipe the copy too
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE categories (
+            id INTEGER PRIMARY KEY,
+            name TEXT,
+            parent_id INTEGER REFERENCES categories(id) ON DELETE CASCADE
+        );
+        """)
+    fresh_db["categories"].insert_all(
+        [
+            {"id": 1, "name": "Fiction", "parent_id": None},
+            {"id": 2, "name": "Science Fiction", "parent_id": 1},
+        ]
+    )
+    with fresh_db.atomic():
+        with pytest.raises(TransactionError) as excinfo:
+            fresh_db["categories"].transform(rename={"name": "title"})
+    assert "categories" in str(excinfo.value)
+    assert fresh_db["categories"].count == 2
+
+
+def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db):
+    # An inbound foreign key without a destructive ON DELETE action is safe
+    # inside a transaction thanks to PRAGMA defer_foreign_keys
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id)
+        );
+        """)
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    with fresh_db.atomic():
+        fresh_db["authors"].transform(rename={"name": "author_name"})
+    assert list(fresh_db["authors"].rows) == [
+        {"id": 1, "author_name": "Ursula K. Le Guin"}
+    ]
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "The Dispossessed", "author_id": 1}
+    ]
+    assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0]
+
+
+def test_transform_in_transaction_allowed_for_child_table(fresh_db):
+    # The table being transformed only has an outbound foreign key - dropping
+    # it fires no ON DELETE actions, so this is allowed inside a transaction
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE
+        );
+        """)
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    with fresh_db.atomic():
+        fresh_db["books"].transform(rename={"title": "book_title"})
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "book_title": "The Dispossessed", "author_id": 1}
+    ]
+
+
+def test_transform_in_transaction_allowed_with_foreign_keys_off(fresh_db):
+    # With PRAGMA foreign_keys off (the default) no cascades can fire, so
+    # transform inside a transaction is safe even with a CASCADE schema
+    fresh_db.executescript("""
+        CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE books (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE
+        );
+        """)
+    fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"})
+    fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1})
+    with fresh_db.atomic():
+        fresh_db["authors"].transform(rename={"name": "author_name"})
+    assert list(fresh_db["books"].rows) == [
+        {"id": 1, "title": "The Dispossessed", "author_id": 1}
+    ]
+
+
+def test_transform_add_foreign_keys_from_scratch(fresh_db):
+    _add_country_city_continent(fresh_db)
+    fresh_db["places"].insert(_CAVEAU)
+    # Should have no foreign keys
+    assert fresh_db["places"].foreign_keys == []
+    # Now add them using .transform()
+    fresh_db["places"].transform(add_foreign_keys=("country", "continent", "city"))
+    # Should now have all three:
+    assert fresh_db["places"].foreign_keys == [
+        ForeignKey(
+            table="places", column="city", other_table="city", other_column="id"
+        ),
+        ForeignKey(
+            table="places",
+            column="continent",
+            other_table="continent",
+            other_column="id",
+        ),
+        ForeignKey(
+            table="places", column="country", other_table="country", other_column="id"
+        ),
+    ]
+    assert fresh_db["places"].schema == (
+        'CREATE TABLE "places" (\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT,\n'
+        '   "country" INTEGER REFERENCES "country"("id"),\n'
+        '   "continent" INTEGER REFERENCES "continent"("id"),\n'
+        '   "city" INTEGER REFERENCES "city"("id")\n'
+        ")"
+    )
+
+
+@pytest.mark.parametrize(
+    "add_foreign_keys",
+    (
+        ("country", "continent"),
+        # Fully specified
+        (
+            ("country", "country", "id"),
+            ("continent", "continent", "id"),
+        ),
+    ),
+)
+def test_transform_add_foreign_keys_from_partial(fresh_db, add_foreign_keys):
+    _add_country_city_continent(fresh_db)
+    fresh_db["places"].insert(
+        _CAVEAU,
+        foreign_keys=("city",),
+    )
+    # Should have one foreign keys
+    assert fresh_db["places"].foreign_keys == [
+        ForeignKey(table="places", column="city", other_table="city", other_column="id")
+    ]
+    # Now add three more using .transform()
+    fresh_db["places"].transform(add_foreign_keys=add_foreign_keys)
+    # Should now have all three:
+    assert fresh_db["places"].foreign_keys == [
+        ForeignKey(
+            table="places", column="city", other_table="city", other_column="id"
+        ),
+        ForeignKey(
+            table="places",
+            column="continent",
+            other_table="continent",
+            other_column="id",
+        ),
+        ForeignKey(
+            table="places", column="country", other_table="country", other_column="id"
+        ),
+    ]
+
+
+@pytest.mark.parametrize(
+    "foreign_keys",
+    (
+        ("country", "continent"),
+        # Fully specified
+        (
+            ("country", "country", "id"),
+            ("continent", "continent", "id"),
+        ),
+    ),
+)
+def test_transform_replace_foreign_keys(fresh_db, foreign_keys):
+    _add_country_city_continent(fresh_db)
+    fresh_db["places"].insert(
+        _CAVEAU,
+        foreign_keys=("city",),
+    )
+    assert len(fresh_db["places"].foreign_keys) == 1
+    # Replace with two different ones
+    fresh_db["places"].transform(foreign_keys=foreign_keys)
+    assert fresh_db["places"].schema == (
+        'CREATE TABLE "places" (\n'
+        '   "id" INTEGER,\n'
+        '   "name" TEXT,\n'
+        '   "country" INTEGER REFERENCES "country"("id"),\n'
+        '   "continent" INTEGER REFERENCES "continent"("id"),\n'
+        '   "city" INTEGER\n'
+        ")"
+    )
+
+
+@pytest.mark.parametrize("table_type", ("id_pk", "rowid", "compound_pk"))
+def test_transform_preserves_rowids(fresh_db, table_type):
+    pk = None
+    if table_type == "id_pk":
+        pk = "id"
+    elif table_type == "compound_pk":
+        pk = ("id", "name")
+    elif table_type == "rowid":
+        pk = None
+    fresh_db["places"].insert_all(
+        [
+            {"id": "1", "name": "Paris", "country": "France"},
+            {"id": "2", "name": "London", "country": "UK"},
+            {"id": "3", "name": "New York", "country": "USA"},
+        ],
+        pk=pk,
+    )
+    # Now delete and insert a row to mix up the `rowid` sequence
+    fresh_db["places"].delete_where("id = ?", ["2"])
+    fresh_db["places"].insert({"id": "4", "name": "London", "country": "UK"})
+    previous_rows = list(
+        tuple(row) for row in fresh_db.execute("select rowid, id, name from places")
+    )
+    # Transform it
+    fresh_db["places"].transform(column_order=("country", "name"))
+    # Should be the same
+    next_rows = list(
+        tuple(row) for row in fresh_db.execute("select rowid, id, name from places")
+    )
+    assert previous_rows == next_rows
+
+
+@pytest.mark.parametrize(
+    "initial_strict,transform_strict,expected_strict",
+    (
+        (False, None, False),
+        (True, None, True),
+        (False, True, True),
+        (True, False, False),
+    ),
+)
+def test_transform_strict(fresh_db, initial_strict, transform_strict, expected_strict):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    dogs = fresh_db.table("dogs", strict=initial_strict)
+    dogs.insert({"id": 1, "name": "Cleo"})
+    assert dogs.strict is initial_strict
+    dogs.transform(strict=transform_strict)
+    assert dogs.strict is expected_strict
+
+
+def test_transform_to_strict_with_invalid_data(fresh_db):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    dogs = fresh_db["dogs"]
+    dogs.create({"id": int})
+    dogs.insert({"id": "not-an-integer"})
+
+    with pytest.raises(sqlite3.IntegrityError):
+        dogs.transform(strict=True)
+
+    assert dogs.strict is False
+    assert list(dogs.rows) == [{"id": "not-an-integer"}]
+    assert fresh_db.table_names() == ["dogs"]
+
+
+def test_transform_strict_updates_default(fresh_db):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    table = fresh_db.table("items", strict=True)
+    table.create({"id": int})
+
+    table.transform(strict=False)
+    assert table.strict is False
+
+    table.create({"id": int}, replace=True)
+    assert table.strict is False
+
+
+@pytest.mark.parametrize("method_name", ("transform", "transform_sql"))
+def test_transform_to_strict_not_supported(fresh_db, method_name):
+    table = fresh_db["items"]
+    table.create({"id": int})
+    fresh_db._supports_strict = False
+
+    with pytest.raises(TransformError, match="SQLite does not support STRICT tables"):
+        getattr(table, method_name)(strict=True)
+
+    assert table.strict is False
+
+
+@pytest.mark.parametrize(
+    "indexes, transform_params",
+    [
+        ([["name"]], {"types": {"age": str}}),
+        ([["name"], ["age", "breed"]], {"types": {"age": str}}),
+        ([], {"types": {"age": str}}),
+        ([["name"]], {"types": {"age": str}, "keep_table": "old_dogs"}),
+    ],
+)
+def test_transform_indexes(fresh_db, indexes, transform_params):
+    # https://github.com/simonw/sqlite-utils/issues/633
+    # New table should have same indexes as old table after transformation
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5, "breed": "Labrador"}, pk="id")
+
+    for index in indexes:
+        dogs.create_index(index)
+
+    indexes_before_transform = dogs.indexes
+
+    dogs.transform(**transform_params)
+
+    assert sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in dogs.indexes
+        ],
+        key=lambda x: x["name"],
+    ) == sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in indexes_before_transform
+        ],
+        key=lambda x: x["name"],
+    ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}"
+    if "keep_table" in transform_params:
+        assert all(
+            index.origin == "pk"
+            for index in fresh_db[transform_params["keep_table"]].indexes
+        )
+
+
+def test_transform_retains_indexes_with_foreign_keys(fresh_db):
+    dogs = fresh_db["dogs"]
+    owners = fresh_db["owners"]
+
+    dogs.insert({"id": 1, "name": "Cleo", "owner_id": 1}, pk="id")
+    owners.insert({"id": 1, "name": "Alice"}, pk="id")
+
+    dogs.create_index(["name"])
+
+    indexes_before_transform = dogs.indexes
+
+    fresh_db.add_foreign_keys([("dogs", "owner_id", "owners", "id")])  # calls transform
+
+    assert sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in dogs.indexes
+        ],
+        key=lambda x: x["name"],
+    ) == sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in indexes_before_transform
+        ],
+        key=lambda x: x["name"],
+    ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}"
+
+
+@pytest.mark.parametrize(
+    "transform_params",
+    [
+        {"rename": {"age": "dog_age"}},
+        {"drop": ["age"]},
+    ],
+)
+def test_transform_with_indexes_errors(fresh_db, transform_params):
+    # Should error with a compound (name, age) index if age is renamed or dropped
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id")
+
+    dogs.create_index(["name", "age"])
+
+    with pytest.raises(TransformError) as excinfo:
+        dogs.transform(**transform_params)
+
+    assert (
+        "Index 'idx_dogs_name_age' column 'age' is not in updated table 'dogs'. "
+        "You must manually drop this index prior to running this transformation"
+        in str(excinfo.value)
+    )
+
+
+def test_transform_with_unique_constraint_implicit_index(fresh_db):
+    dogs = fresh_db["dogs"]
+    # Create a table with a UNIQUE constraint on 'name', which creates an implicit index
+    fresh_db.execute("""
+        CREATE TABLE dogs (
+            id INTEGER PRIMARY KEY,
+            name TEXT UNIQUE,
+            age INTEGER
+        );
+    """)
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5})
+
+    # Attempt to transform the table without modifying 'name'
+    with pytest.raises(TransformError) as excinfo:
+        dogs.transform(types={"age": str})
+
+    assert (
+        "Index 'sqlite_autoindex_dogs_1' on table 'dogs' does not have a CREATE INDEX statement."
+        in str(excinfo.value)
+    )
+    assert (
+        "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation."
+        in str(excinfo.value)
+    )

tests/test_update.py

@@ -70,20 +70,21 @@ def test_update_alter(fresh_db):
     ] == list(table.rows)
 
 
-def test_update_alter_with_invalid_column_characters(fresh_db):
+def test_update_alter_with_special_column_characters(fresh_db):
+    # With double-quote escaping, columns with special characters are now valid
     table = fresh_db["table"]
     rowid = table.insert({"foo": "bar"}).last_pk
-    with pytest.raises(AssertionError):
+    table.update(rowid, {"new_col[abc]": 1.2}, alter=True)
-        table.update(rowid, {"new_col[abc]": 1.2}, alter=True)
+    assert list(table.rows) == [{"foo": "bar", "new_col[abc]": 1.2}]
 
 
 def test_update_with_no_values_sets_last_pk(fresh_db):
     table = fresh_db.table("dogs", pk="id")
     table.insert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Pancakes"}])
     table.update(1)
-    assert 1 == table.last_pk
+    assert table.last_pk == 1
     table.update(2)
-    assert 2 == table.last_pk
+    assert table.last_pk == 2
     with pytest.raises(NotFoundError):
         table.update(3)
 

tests/test_upsert.py

@@ -1,33 +1,46 @@
 from sqlite_utils.db import PrimaryKeyRequired
+from sqlite_utils import Database
 import pytest
 
 
-def test_upsert(fresh_db):
+@pytest.mark.parametrize("use_old_upsert", (False, True))
-    table = fresh_db["table"]
+def test_upsert(use_old_upsert):
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    table = db["table"]
     table.insert({"id": 1, "name": "Cleo"}, pk="id")
     table.upsert({"id": 1, "age": 5}, pk="id", alter=True)
-    assert [{"id": 1, "name": "Cleo", "age": 5}] == list(table.rows)
+    assert list(table.rows) == [{"id": 1, "name": "Cleo", "age": 5}]
-    assert 1 == table.last_pk
+    assert table.last_pk == 1
 
 
 def test_upsert_all(fresh_db):
     table = fresh_db["table"]
     table.upsert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Nixie"}], pk="id")
     table.upsert_all([{"id": 1, "age": 5}, {"id": 2, "age": 5}], pk="id", alter=True)
-    assert [
+    assert list(table.rows) == [
         {"id": 1, "name": "Cleo", "age": 5},
         {"id": 2, "name": "Nixie", "age": 5},
-    ] == list(table.rows)
+    ]
     assert table.last_pk is None
 
 
 def test_upsert_all_single_column(fresh_db):
     table = fresh_db["table"]
     table.upsert_all([{"name": "Cleo"}], pk="name")
-    assert [{"name": "Cleo"}] == list(table.rows)
+    assert list(table.rows) == [{"name": "Cleo"}]
     assert table.pks == ["name"]
 
 
+def test_upsert_all_not_null(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/538
+    fresh_db["comments"].upsert_all(
+        [{"id": 1, "name": "Cleo"}],
+        pk="id",
+        not_null=["name"],
+    )
+    assert list(fresh_db["comments"].rows) == [{"id": 1, "name": "Cleo"}]
+
+
 def test_upsert_error_if_no_pk(fresh_db):
     table = fresh_db["table"]
     with pytest.raises(PrimaryKeyRequired):
@@ -36,6 +49,89 @@ def test_upsert_error_if_no_pk(fresh_db):
         table.upsert({"id": 1, "name": "Cleo"})
 
 
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_empty_record_errors(use_old_upsert):
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    table = db["table"]
+    table.insert({"id": 1, "name": "Cleo"}, pk="id")
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert({}, pk="id")
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert_all([{}, {}], pk="id")
+    # No rows can have been inserted
+    assert table.count == 1
+
+
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_missing_pk_value_errors(use_old_upsert):
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    table = db["table"]
+    table.insert({"id": 1, "name": "Cleo"}, pk="id")
+    # Records that omit the pk column entirely
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert_all([{"name": "Pancakes"}, {"name": "Marnie"}], pk="id")
+    # A record with an explicit None pk value can never conflict
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert({"id": None, "name": "Pancakes"}, pk="id")
+    assert list(table.rows) == [{"id": 1, "name": "Cleo"}]
+
+
+def test_upsert_missing_compound_pk_value_errors(fresh_db):
+    table = fresh_db["table"]
+    table.insert({"a": "x", "b": "y", "v": 1}, pk=("a", "b"))
+    # Missing one component of the detected compound primary key
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert({"a": "x", "v": 2})
+    assert list(table.rows) == [{"a": "x", "b": "y", "v": 1}]
+
+
+def test_upsert_error_if_existing_table_has_no_pk(fresh_db):
+    table = fresh_db.create_table("table", {"id": int, "name": str})
+    with pytest.raises(PrimaryKeyRequired):
+        table.upsert({"id": 1, "name": "Cleo"})
+
+
+@pytest.mark.parametrize("use_old_upsert", (False, True))
+def test_upsert_uses_compound_pk_from_existing_table(use_old_upsert):
+    # https://github.com/simonw/sqlite-utils/issues/629
+    db = Database(memory=True, use_old_upsert=use_old_upsert)
+    db.execute("""
+        create table summary (
+            Source text,
+            Object text,
+            Category text,
+            Count integer,
+            primary key (Source, Object, Category)
+        )
+        """)
+    table = db["summary"]
+    table.upsert(
+        {
+            "Source": "Client A",
+            "Object": "Accounts",
+            "Category": "All",
+            "Count": 3,
+        }
+    )
+    assert table.last_pk == ("Client A", "Accounts", "All")
+    table.upsert(
+        {
+            "Source": "Client A",
+            "Object": "Accounts",
+            "Category": "All",
+            "Count": 4,
+        }
+    )
+    assert list(table.rows) == [
+        {
+            "Source": "Client A",
+            "Object": "Accounts",
+            "Category": "All",
+            "Count": 4,
+        }
+    ]
+
+
 def test_upsert_with_hash_id(fresh_db):
     table = fresh_db["table"]
     table.upsert({"foo": "bar"}, hash_id="pk")
@@ -45,6 +141,30 @@ def test_upsert_with_hash_id(fresh_db):
     assert "a5e744d0164540d33b1d7ea616c28f2fa97e754a" == table.last_pk
 
 
+@pytest.mark.parametrize("hash_id", (None, "custom_id"))
+def test_upsert_with_hash_id_columns(fresh_db, hash_id):
+    table = fresh_db["table"]
+    table.upsert({"a": 1, "b": 2, "c": 3}, hash_id=hash_id, hash_id_columns=("a", "b"))
+    assert list(table.rows) == [
+        {
+            hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49",
+            "a": 1,
+            "b": 2,
+            "c": 3,
+        }
+    ]
+    assert table.last_pk == "4acc71e0547112eb432f0a36fb1924c4a738cb49"
+    table.upsert({"a": 1, "b": 2, "c": 4}, hash_id=hash_id, hash_id_columns=("a", "b"))
+    assert list(table.rows) == [
+        {
+            hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49",
+            "a": 1,
+            "b": 2,
+            "c": 4,
+        }
+    ]
+
+
 def test_upsert_compound_primary_key(fresh_db):
     table = fresh_db["table"]
     table.upsert_all(
@@ -54,7 +174,7 @@ def test_upsert_compound_primary_key(fresh_db):
         ],
         pk=("species", "id"),
     )
-    assert None == table.last_pk
+    assert table.last_pk is None
     table.upsert({"species": "dog", "id": 1, "age": 5}, pk=("species", "id"))
     assert ("dog", 1) == table.last_pk
     assert [

tests/test_utils.py

@@ -1,4 +1,6 @@
 from sqlite_utils import utils
+import csv
+import io
 import pytest
 
 
@@ -22,6 +24,79 @@ def test_decode_base64_values(input, expected, should_be_is):
         assert actual == expected
 
 
-def test_find_spatialite():
+@pytest.mark.parametrize(
-    spatialite = utils.find_spatialite()
+    "size,expected",
-    assert spatialite is None or isinstance(spatialite, str)
+    (
+        (1, [["a"], ["b"], ["c"], ["d"]]),
+        (2, [["a", "b"], ["c", "d"]]),
+        (3, [["a", "b", "c"], ["d"]]),
+        (4, [["a", "b", "c", "d"]]),
+    ),
+)
+def test_chunks(size, expected):
+    input = ["a", "b", "c", "d"]
+    chunks = list(map(list, utils.chunks(input, size)))
+    assert chunks == expected
+
+
+def test_hash_record():
+    expected = "d383e7c0ba88f5ffcdd09be660de164b3847401a"
+    assert utils.hash_record({"name": "Cleo", "twitter": "CleoPaws"}) == expected
+    assert (
+        utils.hash_record(
+            {"name": "Cleo", "twitter": "CleoPaws", "age": 7}, keys=("name", "twitter")
+        )
+        == expected
+    )
+    assert (
+        utils.hash_record({"name": "Cleo", "twitter": "CleoPaws", "age": 7}) != expected
+    )
+
+
+def test_maximize_csv_field_size_limit():
+    # Reset to default in case other tests have changed it
+    csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT)
+    long_value = "a" * 131073
+    long_csv = "id,text\n1,{}".format(long_value)
+    fp = io.BytesIO(long_csv.encode("utf-8"))
+    # Using rows_from_file should error
+    with pytest.raises(csv.Error):
+        rows, _ = utils.rows_from_file(fp, utils.Format.CSV)
+        list(rows)
+    # But if we call maximize_csv_field_size_limit() first it should be OK:
+    utils.maximize_csv_field_size_limit()
+    fp2 = io.BytesIO(long_csv.encode("utf-8"))
+    rows2, _ = utils.rows_from_file(fp2, utils.Format.CSV)
+    rows_list2 = list(rows2)
+    assert len(rows_list2) == 1
+    assert rows_list2[0]["id"] == "1"
+    assert rows_list2[0]["text"] == long_value
+
+
+@pytest.mark.parametrize(
+    "input,expected",
+    (
+        ({"foo": {"bar": 1}}, {"foo_bar": 1}),
+        ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}),
+        ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}),
+    ),
+)
+def test_flatten(input, expected):
+    assert utils.flatten(input) == expected
+
+
+@pytest.mark.parametrize(
+    "input,expected",
+    (
+        ([], []),
+        (["id", "name"], ["id", "name"]),
+        (["id", "id"], ["id", "id_2"]),
+        (["id", "id", "id"], ["id", "id_2", "id_3"]),
+        # A renamed duplicate must not clobber a real column called id_2
+        (["id", "id", "id_2"], ["id", "id_3", "id_2"]),
+        (["id_2", "id", "id"], ["id_2", "id", "id_3"]),
+        (["id", "id", "id_2", "id_2"], ["id", "id_3", "id_2", "id_2_2"]),
+    ),
+)
+def test_dedupe_keys(input, expected):
+    assert utils.dedupe_keys(input) == expected

tests/test_wal.py

@@ -1,6 +1,6 @@
 import pytest
 from sqlite_utils import Database
-import sqlite3
+from sqlite_utils.db import TransactionError
 
 
 @pytest.fixture
@@ -22,3 +22,67 @@ def test_enable_disable_wal(db_path_tmpdir):
     db.disable_wal()
     assert "delete" == db.journal_mode
     assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()]
+
+
+def test_enable_wal_inside_transaction_raises(db_path_tmpdir):
+    db, path, tmpdir = db_path_tmpdir
+    db["test"].insert({"id": 1}, pk="id")
+    with pytest.raises(TransactionError):
+        with db.atomic():
+            db["test"].insert({"id": 2}, pk="id")
+            db.enable_wal()
+    # The atomic() block must have rolled back cleanly and the
+    # journal mode must be unchanged
+    assert db.journal_mode == "delete"
+    assert [r["id"] for r in db["test"].rows] == [1]
+
+
+def test_disable_wal_inside_transaction_raises(db_path_tmpdir):
+    db, path, tmpdir = db_path_tmpdir
+    db.enable_wal()
+    db["test"].insert({"id": 1}, pk="id")
+    with pytest.raises(TransactionError):
+        with db.atomic():
+            db["test"].insert({"id": 2}, pk="id")
+            db.disable_wal()
+    assert db.journal_mode == "wal"
+    assert [r["id"] for r in db["test"].rows] == [1]
+
+
+def test_ensure_autocommit_on(db_path_tmpdir):
+    db, path, tmpdir = db_path_tmpdir
+    previous_isolation_level = db.conn.isolation_level
+    assert previous_isolation_level is not None
+    with db.ensure_autocommit_on():
+        # isolation_level of None means driver-level autocommit mode
+        assert db.conn.isolation_level is None
+    # Restored afterwards
+    assert db.conn.isolation_level == previous_isolation_level
+
+
+def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir):
+    # Calling enable_wal() when WAL is already enabled is a no-op,
+    # so it is fine inside a transaction
+    db, path, tmpdir = db_path_tmpdir
+    db.enable_wal()
+    with db.atomic():
+        db["test"].insert({"id": 1}, pk="id")
+        db.enable_wal()
+    assert [r["id"] for r in db["test"].rows] == [1]
+
+
+def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir):
+    # Setting isolation_level commits any pending transaction as a side
+    # effect, silently breaking the caller's rollback guarantee - so
+    # entering autocommit mode with a transaction open is an error
+    db, path, tmpdir = db_path_tmpdir
+    db["test"].insert({"id": 1}, pk="id")
+    db.begin()
+    db.execute("insert into test (id) values (2)")
+    with pytest.raises(TransactionError):
+        with db.ensure_autocommit_on():
+            pass
+    # The transaction is still open and can still be rolled back
+    assert db.conn.in_transaction
+    db.rollback()
+    assert [r["id"] for r in db["test"].rows] == [1]

tests/utils.py

@@ -1,7 +0,0 @@
-import re
-
-COLLAPSE_RE = re.compile(r"\s+")
-
-
-def collapse_whitespace(s):
-    return COLLAPSE_RE.sub(" ", s)