Merge branch 'main' into fts-quote

.github/workflows/publish.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

.github/workflows/spellcheck.yml

@@ -0,0 +1,27 @@
+name: Check spelling in documentation
+
+on: [push, pull_request]
+
+jobs:
+  spellcheck:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - uses: actions/cache@v2
+      name: Configure pip caching
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        pip install -e '.[docs]'
+    - name: Check spelling
+      run: |
+        codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
+        codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt

.github/workflows/test-coverage.yml

@@ -0,0 +1,41 @@
+name: Calculate test coverage
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repo
+      uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - uses: actions/cache@v2
+      name: Configure pip caching
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -e .[test]
+        python -m pip install pytest-cov
+    - name: Run tests
+      run: |-
+        ls -lah
+        pytest --cov=sqlite_utils --cov-report xml:coverage.xml --cov-report term
+        ls -lah
+    - name: Upload coverage report
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: coverage.xml

.github/workflows/test.yml

@@ -25,12 +25,16 @@ jobs:
           ${{ runner.os }}-pip-
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install -e '.[test,mypy,flake8]'
     - name: Optionally install numpy
       if: matrix.numpy == 1
       run: pip install numpy
     - name: Run tests
       run: |
         pytest
+    - name: run mypy
+      run: mypy sqlite_utils
+    - name: run flake8
+      run: flake8
     - name: Check formatting
       run: black . --check

.readthedocs.yaml

@@ -0,0 +1,12 @@
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+  version: "3.8"
+  install:
+  - method: pip
+    path: .
+    extra_requirements:
+    - docs

README.md

@@ -1,10 +1,11 @@
 # sqlite-utils
 
 [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/)
-[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/latest/changelog.html)
+[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/stable/changelog.html)
 [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/)
 [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest)
-[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest)
+[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable)
+[![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE)
 
 Python CLI utility and library for manipulating SQLite databases.
@@ -56,7 +57,11 @@ You can import JSON data into a new database table like this:
 
 Or for data in a CSV file:
 
-    $ sqlite-utils insert dogs.db dogs docs.csv --csv
+    $ sqlite-utils insert dogs.db dogs dogs.csv --csv
+
+`sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command:
+
+    $ cat dogs.csv | sqlite-utils memory - "select name, age from dogs"
 
 See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands.
 

codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true

docs/Makefile

@@ -20,4 +20,4 @@ help:
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 livehtml:
-	sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0)
+	sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils

docs/changelog.rst

@@ -2,6 +2,184 @@
  Changelog
 ===========
 
+.. _v3_15.1:
+
+3.15.1 (2021-08-10)
+-------------------
+
+- Python library now includes type annotations on almost all of the methods, plus detailed docstrings describing each one. (:issue:`311`)
+- New :ref:`reference` documentation page, powered by those docstrings.
+- Fixed bug where ``.add_foreign_keys()`` failed to raise an error if called against a ``View``. (:issue:`313`)
+- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existent table. (:issue:`315`)
+
+.. _v3_15:
+
+3.15 (2021-08-09)
+-----------------
+
+- ``sqlite-utils insert --flatten`` option for :ref:`flattening nested JSON objects <cli_inserting_data_flatten>` to create tables with column names like ``topkey_nestedkey``. (:issue:`310`)
+- Fixed several spelling mistakes in the documentation, spotted `using codespell <https://til.simonwillison.net/python/codespell>`__.
+- Errors that occur while using the ``sqlite-utils`` CLI tool now show the responsible SQL and query parameters, if possible. (:issue:`309`)
+
+.. _v3_14:
+
+3.14 (2021-08-02)
+-----------------
+
+This release introduces the new :ref:`sqlite-utils convert command <cli_convert>` (:issue:`251`) and corresponding :ref:`table.convert(...) <python_api_convert>` Python method (:issue:`302`). These tools can be used to apply a Python conversion function to one or more columns of a table, either updating the column in place or using transformed data from that column to populate one or more other columns.
+
+This command-line example uses the Python standard library `textwrap module <https://docs.python.org/3/library/textwrap.html>`__ to wrap the content of the ``content`` column in the ``articles`` table to 100 characters::
+
+    $ sqlite-utils convert content.db articles content \
+        '"\n".join(textwrap.wrap(value, 100))' \
+        --import=textwrap
+
+The same operation in Python code looks like this:
+
+.. code-block:: python
+
+    import sqlite_utils, textwrap
+
+    db = sqlite_utils.Database("content.db")
+    db["articles"].convert("content", lambda v: "\n".join(textwrap.wrap(v, 100)))
+
+See the full documentation for the :ref:`sqlite-utils convert command <cli_convert>` and the :ref:`table.convert(...) <python_api_convert>` Python method for more details.
+
+Also in this release:
+
+- The new ``table.count_where(...)`` method, for counting rows in a table that match a specific SQL ``WHERE`` clause. (:issue:`305`)
+- New ``--silent`` option for the :ref:`sqlite-utils insert-files command <cli_insert_files>` to hide the terminal progress bar, consistent with the ``--silent`` option for ``sqlite-utils convert``. (:issue:`301`)
+
+.. _v3_13:
+
+3.13 (2021-07-24)
+-----------------
+
+- ``sqlite-utils schema my.db table1 table2`` command now accepts optional table names. (:issue:`299`)
+- ``sqlite-utils memory --help`` now describes the ``--schema`` option.
+
+.. _v3_12:
+
+3.12 (2021-06-25)
+-----------------
+
+- New :ref:`db.query(sql, params) <python_api_query>` method, which executes a SQL query and returns the results as an iterator over Python dictionaries. (:issue:`290`)
+- This project now uses ``flake8`` and has started to use ``mypy``. (:issue:`291`)
+- New documentation on :ref:`contributing <contributing>` to this project. (:issue:`292`)
+
+.. _v3_11:
+
+3.11 (2021-06-20)
+-----------------
+
+- New ``sqlite-utils memory data.csv --schema`` option, for outputting the schema of the in-memory database generated from one or more files. See :ref:`cli_memory_schema_dump_save`. (:issue:`288`)
+- Added :ref:`installation instructions <installation>`. (:issue:`286`)
+
+.. _v3_10:
+
+3.10 (2021-06-19)
+-----------------
+
+This release introduces the ``sqlite-utils memory`` command, which can be used to load CSV or JSON data into a temporary in-memory database and run SQL queries (including joins across multiple files) directly against that data.
+
+Also new: ``sqlite-utils insert --detect-types``, ``sqlite-utils dump``, ``table.use_rowid`` plus some smaller fixes.
+
+sqlite-utils memory
+~~~~~~~~~~~~~~~~~~~
+
+This example of ``sqlite-utils memory`` retrieves information about the all of the repositories in the `Dogsheep <https://github.com/dogsheep>`__ organization on GitHub using `this JSON API <https://api.github.com/users/dogsheep/repos>`__, sorts them by their number of stars and outputs a table of the top five (using ``-t``)::
+
+    $ curl -s 'https://api.github.com/users/dogsheep/repos' \
+      | sqlite-utils memory - '
+          select full_name, forks_count, stargazers_count
+          from stdin order by stargazers_count desc limit 5
+        ' -t
+    full_name                            forks_count    stargazers_count
+    ---------------------------------  -------------  ------------------
+    dogsheep/twitter-to-sqlite                    12                 225
+    dogsheep/github-to-sqlite                     14                 139
+    dogsheep/dogsheep-photos                       5                 116
+    dogsheep/dogsheep.github.io                    7                  90
+    dogsheep/healthkit-to-sqlite                   4                  85
+
+The tool works against files on disk as well. This example joins data from two CSV files::
+
+    $ cat creatures.csv
+    species_id,name
+    1,Cleo
+    2,Bants
+    2,Dori
+    2,Azi
+    $ cat species.csv
+    id,species_name
+    1,Dog
+    2,Chicken
+    $ sqlite-utils memory species.csv creatures.csv '
+      select * from creatures join species on creatures.species_id = species.id
+    '
+    [{"species_id": 1, "name": "Cleo", "id": 1, "species_name": "Dog"},
+     {"species_id": 2, "name": "Bants", "id": 2, "species_name": "Chicken"},
+     {"species_id": 2, "name": "Dori", "id": 2, "species_name": "Chicken"},
+     {"species_id": 2, "name": "Azi", "id": 2, "species_name": "Chicken"}]
+
+Here the ``species.csv`` file becomes the ``species`` table, the ``creatures.csv`` file becomes the ``creatures`` table and the output is JSON, the default output format.
+
+You can also use the ``--attach`` option to attach existing SQLite database files to the in-memory database, in order to join data from CSV or JSON directly against your existing tables.
+
+Full documentation of this new feature is available in :ref:`cli_memory`. (:issue:`272`)
+
+sqlite-utils insert \-\-detect-types
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`sqlite-utils insert <cli_inserting_data>` command can be used to insert data from JSON, CSV or TSV files into a SQLite database file. The new ``--detect-types`` option (shortcut ``-d``), when used in conjunction with a CSV or TSV import, will automatically detect if columns in the file are integers or floating point numbers as opposed to treating everything as a text column and create the new table with the corresponding schema. See :ref:`cli_insert_csv_tsv` for details. (:issue:`282`)
+
+Other changes
+~~~~~~~~~~~~~
+
+- **Bug fix**: ``table.transform()``, when run against a table without explicit primary keys, would incorrectly create a new version of the table with an explicit primary key column called ``rowid``. (:issue:`284`)
+- New ``table.use_rowid`` introspection property, see :ref:`python_api_introspection_use_rowid`. (:issue:`285`)
+- The new ``sqlite-utils dump file.db`` command outputs a SQL dump that can be used to recreate a database. (:issue:`274`)
+- ``-h`` now works as a shortcut for ``--help``, thanks Loren McIntyre. (:issue:`276`)
+- Now using `pytest-cov <https://pytest-cov.readthedocs.io/>`__ and `Codecov <https://about.codecov.io/>`__ to track test coverage - currently at 96%. (:issue:`275`)
+- SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors.
+
+.. _v3_9_1:
+
+3.9.1 (2021-06-12)
+------------------
+
+- Fixed bug when using ``table.upsert_all()`` to create a table with only a single column that is treated as the primary key. (:issue:`271`)
+
+.. _v3_9:
+
+3.9 (2021-06-11)
+----------------
+
+- New ``sqlite-utils schema`` command showing the full SQL schema for a database, see :ref:`Showing the schema (CLI)<cli_schema>`. (:issue:`268`)
+- ``db.schema`` introspection property exposing the same feature to the Python library, see :ref:`Showing the schema (Python library) <python_api_schema>`.
+
+.. _v3_8:
+
+3.8 (2021-06-02)
+----------------
+
+- New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (:issue:`263`)
+- ``table.xindexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (:issue:`261`)
+
+.. _v3_7:
+
+3.7 (2021-05-28)
+----------------
+
+- New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (:issue:`240`)
+- Fixed bug with ``table.add_foreign_key()`` against columns containing spaces. (:issue:`238`)
+- ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (:issue:`237`)
+- ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (:issue:`237`)
+- Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (:issue:`257`)
+- Suggest ``--alter`` if an error occurs caused by a missing column. (:issue:`259`)
+- Support creating indexes with columns in descending order, see :ref:`API documentation <python_api_create_index>` and :ref:`CLI documentation <cli_create_index>`. (:issue:`260`)
+- Correctly handle CSV files that start with a UTF-8 BOM. (:issue:`250`)
+
 .. _v3_6:
 
 3.6 (2021-02-18)
@@ -9,73 +187,73 @@
 
 This release adds the ability to execute queries joining data from more than one database file - similar to the cross database querying feature introduced in `Datasette 0.55 <https://docs.datasette.io/en/stable/changelog.html#v0-55>`__.
 
-- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation <python_api_attach>`. (`#113 <https://github.com/simonw/sqlite-utils/issues/113>`__)
+- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation <python_api_attach>`. (:issue:`113`)
-- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation <cli_attach>`. (`#236 <https://github.com/simonw/sqlite-utils/issues/236>`__)
+- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation <cli_query_attach>`. (:issue:`236`)
 
 .. _v3_5:
 
 3.5 (2021-02-14)
 ----------------
 
-- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (`#230 <https://github.com/simonw/sqlite-utils/issues/230>`__)
+- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`230`)
-- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (`#231 <https://github.com/simonw/sqlite-utils/issues/231>`__)
+- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (:issue:`231`)
-- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (`#228 <https://github.com/simonw/sqlite-utils/issues/228>`__)
+- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (:issue:`228`)
-- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven <https://github.com/nieuwenhoven>`__ for the fix. (`#234 <https://github.com/simonw/sqlite-utils/issues/234>`__)
+- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven <https://github.com/nieuwenhoven>`__ for the fix. (:issue:`234`)
-- Fixed bug importing CSV files with columns containing more than 128KB of data. (`#229 <https://github.com/simonw/sqlite-utils/issues/229>`__)
+- Fixed bug importing CSV files with columns containing more than 128KB of data. (:issue:`229`)
-- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven <https://github.com/nieuwenhoven>`__ for the Windows test fixes. (`#232 <https://github.com/simonw/sqlite-utils/issues/232>`__)
+- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven <https://github.com/nieuwenhoven>`__ for the Windows test fixes. (:issue:`232`)
 
 .. _v3_4_1:
 
 3.4.1 (2021-02-05)
 ------------------
 
-- Fixed a code import bug that slipped in to 3.4. (`#226 <https://github.com/simonw/sqlite-utils/issues/226>`__)
+- Fixed a code import bug that slipped in to 3.4. (:issue:`226`)
 
 .. _v3_4:
 
 3.4 (2021-02-05)
 ----------------
 
-- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (`#223 <https://github.com/simonw/sqlite-utils/issues/223>`__)
+- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`223`)
 
 .. _v3_3:
 
 3.3 (2021-01-17)
 ----------------
 
-- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (`#222 <https://github.com/simonw/sqlite-utils/issues/222>`__)
+- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (:issue:`222`)
 
 .. _v3_2_1:
 
 3.2.1 (2021-01-12)
 ------------------
 
-- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (`#221 <https://github.com/simonw/sqlite-utils/issues/221>`__)
+- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (:issue:`221`)
 
 .. _v3_2:
 
 3.2 (2021-01-03)
 ----------------
 
-This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (`#212 <https://github.com/simonw/sqlite-utils/issues/212>`__)
+This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (:issue:`212`)
 
 - ``table.enable_counts()`` method for enabling these triggers on a specific table.
-- ``db.enable_counts()`` method for enabling triggers on every table in the database. (`#213 <https://github.com/simonw/sqlite-utils/issues/213>`__)
+- ``db.enable_counts()`` method for enabling triggers on every table in the database. (:issue:`213`)
-- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (`#214 <https://github.com/simonw/sqlite-utils/issues/214>`__)
+- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (:issue:`214`)
-- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (`#218 <https://github.com/simonw/sqlite-utils/issues/218>`__)
+- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (:issue:`218`)
-- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (`#215 <https://github.com/simonw/sqlite-utils/issues/215>`__)
+- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (:issue:`215`)
 - ``table.has_counts_triggers`` property revealing if a table has been configured with the new ``_counts`` database triggers.
-- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (`#219 <https://github.com/simonw/sqlite-utils/issues/219>`__)
+- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (:issue:`219`)
-- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (`#217 <https://github.com/simonw/sqlite-utils/issues/217>`__)
+- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (:issue:`217`)
-- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (`#211 <https://github.com/simonw/sqlite-utils/issues/211>`__, `#216 <https://github.com/simonw/sqlite-utils/issues/216>`__)
+- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (:issue:`211`, :issue:`216`)
-- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (`#206 <https://github.com/simonw/sqlite-utils/issues/206>`__)
+- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (:issue:`206`)
 
 .. _v3_1_1:
 
 3.1.1 (2021-01-01)
 ------------------
 
-- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (`#209 <https://github.com/simonw/sqlite-utils/issues/209>`__)
+- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (:issue:`209`)
 - Documentation now lives on https://sqlite-utils.datasette.io/
 - README now includes ``brew install sqlite-utils`` installation method.
 
@@ -84,7 +262,7 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi
 3.1 (2020-12-12)
 ----------------
 
-- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 <https://github.com/simonw/sqlite-utils/issues/207>`__)
+- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (:issue:`207`)
 - New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`.
 - The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 <https://github.com/simonw/sqlite-utils/pull/204>`__)
 
@@ -93,28 +271,28 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi
 3.0 (2020-11-08)
 ----------------
 
-This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (`#192 <https://github.com/simonw/sqlite-utils/issues/192>`__)
+This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (:issue:`192`)
 
-The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (`#197 <https://github.com/simonw/sqlite-utils/issues/197>`__)
+The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (:issue:`197`)
 
 The release includes minor backwards-incompatible changes, hence the version bump to 3.0. Those changes, which should not affect most users, are:
 
 - The ``-c`` shortcut option for outputting CSV is no longer available. The full ``--csv`` option is required instead.
 - The ``-f`` shortcut for ``--fmt`` has also been removed - use ``--fmt``.
-- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (`#198 <https://github.com/simonw/sqlite-utils/issues/198>`__)
+- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (:issue:`198`)
 - The ``table.search()`` method now returns a generator over a list of Python dictionaries. It previously returned a list of tuples.
 
 Also in this release:
 
-- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (`#193 <https://github.com/simonw/sqlite-utils/issues/193>`__)
+- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (:issue:`193`)
-- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (`#196 <https://github.com/simonw/sqlite-utils/issues/196>`__)
+- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (:issue:`196`)
 - The new ``table.search_sql()`` method returns the SQL for searching a table, see :ref:`python_api_fts_search_sql`.
-- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (`#200 <https://github.com/simonw/sqlite-utils/issues/200>`__)
+- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (:issue:`200`)
 
 Changes since the 3.0a0 alpha release:
 
 - The ``sqlite-utils search`` command now defaults to returning every result, unless you add a ``--limit 20`` option.
-- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (`#201 <https://github.com/simonw/sqlite-utils/issues/201>`__)
+- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (:issue:`201`)
 
 .. _v2_23:
 
@@ -122,30 +300,30 @@ Changes since the 3.0a0 alpha release:
 -----------------
 
 - ``table.m2m(other_table, records)`` method now takes any iterable, not just a list or tuple. Thanks, Adam Wolf. (`#189 <https://github.com/simonw/sqlite-utils/pull/189>`__)
-- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (`#173 <https://github.com/simonw/sqlite-utils/issues/173>`__)
+- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (:issue:`173`)
-- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (`#191 <https://github.com/simonw/sqlite-utils/issues/191>`__)
+- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (:issue:`191`)
 
 .. _v2_22:
 
 2.22 (2020-10-16)
 -----------------
 
-- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (`#182 <https://github.com/simonw/sqlite-utils/issues/182>`__)
+- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (:issue:`182`)
-- The ``--load-extension`` option is now available to many more commands. (`#137 <https://github.com/simonw/sqlite-utils/issues/137>`__)
+- The ``--load-extension`` option is now available to many more commands. (:issue:`137`)
-- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (`#136 <https://github.com/simonw/sqlite-utils/issues/136>`__)
+- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (:issue:`136`)
-- Tests now also run against Python 3.9. (`#184 <https://github.com/simonw/sqlite-utils/issues/184>`__)
+- Tests now also run against Python 3.9. (:issue:`184`)
-- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (`#181 <https://github.com/simonw/sqlite-utils/issues/181>`__)
+- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (:issue:`181`)
 
 .. _v2_21:
 
 2.21 (2020-09-24)
 -----------------
 
-- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (`#172 <https://github.com/simonw/sqlite-utils/issues/172>`__)
+- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (:issue:`172`)
 - ``sqlite-utils extract`` no longer shows a progress bar, because it's fast enough not to need one.
-- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (`#175 <https://github.com/simonw/sqlite-utils/issues/175>`__)
+- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (:issue:`175`)
-- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (`#176 <https://github.com/simonw/sqlite-utils/issues/176>`__)
+- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (:issue:`176`)
-- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (`#177 <https://github.com/simonw/sqlite-utils/issues/177>`__)
+- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (:issue:`177`)
 - The table ``.disable_fts()``, ``.rebuild_fts()``, ``.delete()``, ``.delete_where()`` and ``.add_missing_columns()`` methods all now ``return self``, which means they can be chained together with other table operations.
 
 .. _v2_20:
@@ -153,7 +331,7 @@ Changes since the 3.0a0 alpha release:
 2.20 (2020-09-22)
 -----------------
 
-This release introduces two key new capabilities: **transform** (`#114 <https://github.com/simonw/sqlite-utils/issues/114>`__) and **extract** (`#42 <https://github.com/simonw/sqlite-utils/issues/42>`__).
+This release introduces two key new capabilities: **transform** (:issue:`114`) and **extract** (:issue:`42`).
 
 Transform
 ~~~~~~~~~
@@ -174,7 +352,7 @@ The Python library :ref:`extract() documentation <python_api_extract>` describes
 Other changes
 ~~~~~~~~~~~~~
 
-- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (`#162 <https://github.com/simonw/sqlite-utils/issues/162>`__)
+- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (:issue:`162`)
 - The ``table.rows_where()`` method now accepts an optional ``select=`` argument for specifying which columns should be selected, see :ref:`python_api_rows`.
 
 .. _v2_19:
@@ -182,31 +360,31 @@ Other changes
 2.19 (2020-09-20)
 -----------------
 
-- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (`#157 <https://github.com/simonw/sqlite-utils/issues/157>`__)
+- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (:issue:`157`)
-- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (`#160 <https://github.com/simonw/sqlite-utils/issues/160>`__)
+- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (:issue:`160`)
-- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (`#112 <https://github.com/simonw/sqlite-utils/issues/112>`__)
+- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (:issue:`112`)
 
 .. _v2_18:
 
 2.18 (2020-09-08)
 -----------------
 
-- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (`#155 <https://github.com/simonw/sqlite-utils/issues/155>`__)
+- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (:issue:`155`)
-- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (`#155 <https://github.com/simonw/sqlite-utils/issues/155>`__)
+- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (:issue:`155`)
 - ``table.optimize()`` method no longer deletes junk rows from the ``*_fts_docsize`` table. This was added in 2.17 but it turns out running ``table.rebuild_fts()`` is a better solution to this problem.
-- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (`#145 <https://github.com/simonw/sqlite-utils/issues/145>`__)
+- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (:issue:`145`)
 
 .. _v2_17:
 
 2.17 (2020-09-07)
 -----------------
 
-This release handles a bug where replacing rows in FTS tables could result in growing numbers of unneccessary rows in the associated ``*_fts_docsize`` table. (`#149 <https://github.com/simonw/sqlite-utils/issues/149>`__)
+This release handles a bug where replacing rows in FTS tables could result in growing numbers of unnecessary rows in the associated ``*_fts_docsize`` table. (:issue:`149`)
 
-- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (`#152 <https://github.com/simonw/sqlite-utils/issues/152>`__)
+- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (:issue:`152`)
-- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (`#153 <https://github.com/simonw/sqlite-utils/issues/153>`__)
+- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (:issue:`153`)
-- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (`#150 <https://github.com/simonw/sqlite-utils/issues/150>`__)
+- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (:issue:`150`)
-- Neater indentation for schema SQL. (`#148 <https://github.com/simonw/sqlite-utils/issues/148>`__)
+- Neater indentation for schema SQL. (:issue:`148`)
 - Documentation for ``sqlite_utils.AlterError`` exception thrown by in ``add_foreign_keys()``.
 
 .. _v2_16_1:
@@ -214,23 +392,23 @@ This release handles a bug where replacing rows in FTS tables could result in gr
 2.16.1 (2020-08-28)
 -------------------
 
-- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (`#139 <https://github.com/simonw/sqlite-utils/issues/139>`__)
+- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (:issue:`139`)
-- Continuous Integration is now powered by GitHub Actions. (`#143 <https://github.com/simonw/sqlite-utils/issues/143>`__)
+- Continuous Integration is now powered by GitHub Actions. (:issue:`143`)
 
 .. _v2_16:
 
 2.16 (2020-08-21)
 -----------------
 
-- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (`#134 <https://github.com/simonw/sqlite-utils/issues/134>`__)
+- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (:issue:`134`)
-- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (`#135 <https://github.com/simonw/sqlite-utils/issues/135>`__)
+- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (:issue:`135`)
 
 .. _v2_15_1:
 
 2.15.1 (2020-08-12)
 -------------------
 
-- Now available as a ``sdist`` package on PyPI in addition to a wheel. (`#133 <https://github.com/simonw/sqlite-utils/issues/133>`__)
+- Now available as a ``sdist`` package on PyPI in addition to a wheel. (:issue:`133`)
 
 .. _v2_15:
 
@@ -238,7 +416,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr
 -----------------
 
 - New ``db.enable_wal()`` and ``db.disable_wal()`` methods for enabling and disabling `Write-Ahead Logging <https://www.sqlite.org/wal.html>`__ for a database file - see :ref:`python_api_wal` in the Python API documentation.
-- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) <cli_wal>`. (`#132 <https://github.com/simonw/sqlite-utils/issues/132>`__)
+- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) <cli_wal>`. (:issue:`132`)
 
 .. _v2_14_1:
 
@@ -252,8 +430,8 @@ This release handles a bug where replacing rows in FTS tables could result in gr
 2.14 (2020-08-01)
 -----------------
 
-- The :ref:`insert-files command <cli_insert_files>` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (`#127 <https://github.com/simonw/sqlite-utils/issues/127>`__)
+- The :ref:`insert-files command <cli_insert_files>` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (:issue:`127`)
-- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() <python_api_fts>`. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (`#130 <https://github.com/simonw/sqlite-utils/issues/130>`__)
+- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() <python_api_fts>`. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (:issue:`130`)
 - You can also set a custom tokenizer using the :ref:`sqlite-utils enable-fts <cli_fts>` CLI command, via the new ``--tokenize`` option.
 
 .. _v2_13:
@@ -261,7 +439,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr
 2.13 (2020-07-29)
 -----------------
 
-- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (`#128 <https://github.com/simonw/sqlite-utils/issues/128>`__)
+- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (:issue:`128`)
 
 .. _v2_12:
 
@@ -270,11 +448,11 @@ This release handles a bug where replacing rows in FTS tables could result in gr
 
 The theme of this release is better tools for working with binary data. The new ``insert-files`` command can be used to insert binary files directly into a database table, and other commands have been improved with better support for BLOB columns.
 
-- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (`#122 <https://github.com/simonw/sqlite-utils/issues/122>`__)
+- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (:issue:`122`)
-- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (`#123 <https://github.com/simonw/sqlite-utils/issues/123>`__)
+- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (:issue:`123`)
-- JSON output now encodes BLOB values as special base64 obects - see :ref:`cli_query_json`. (`#125 <https://github.com/simonw/sqlite-utils/issues/125>`__)
+- JSON output now encodes BLOB values as special base64 objects - see :ref:`cli_query_json`. (:issue:`125`)
-- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (`#126 <https://github.com/simonw/sqlite-utils/issues/126>`__)
+- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (:issue:`126`)
-- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (`#124 <https://github.com/simonw/sqlite-utils/issues/124>`__)
+- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (:issue:`124`)
 
 .. _v2_11:
 
@@ -289,14 +467,14 @@ The theme of this release is better tools for working with binary data. The new
 2.10.1 (2020-06-23)
 -------------------
 
-- Added documentation for the ``table.pks`` introspection property. (`#116 <https://github.com/simonw/sqlite-utils/issues/116>`__)
+- Added documentation for the ``table.pks`` introspection property. (:issue:`116`)
 
 .. _v2_10:
 
 2.10 (2020-06-12)
 -----------------
 
-- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (`#115 <https://github.com/simonw/sqlite-utils/issues/115>`__)
+- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (:issue:`115`)
 
 .. _v2_9_1:
 
@@ -310,77 +488,77 @@ The theme of this release is better tools for working with binary data. The new
 2.9 (2020-05-10)
 ----------------
 
-- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (`#111 <https://github.com/simonw/sqlite-utils/issues/111>`__)
+- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (:issue:`111`)
 - New ``sqlite-utils drop-view`` command, see :ref:`cli_drop_view`.
-- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (`#110 <https://github.com/simonw/sqlite-utils/issues/110>`__)
+- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (:issue:`110`)
 
 .. _v2_8:
 
 2.8 (2020-05-03)
 ----------------
 
-- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (`#27 <https://github.com/simonw/sqlite-utils/issues/27>`__)
+- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (:issue:`27`)
-- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (`#107 <https://github.com/simonw/sqlite-utils/issues/107>`__)
+- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (:issue:`107`)
 
 .. _v2_7.2:
 
 2.7.2 (2020-05-02)
 ------------------
 
-- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (`#106 <https://github.com/simonw/sqlite-utils/issues/106>`__)
+- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (:issue:`106`)
 
 .. _v2_7.1:
 
 2.7.1 (2020-05-01)
 ------------------
 
-- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (`#105 <https://github.com/simonw/sqlite-utils/issues/105>`__)
+- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (:issue:`105`)
-- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (`#104 <https://github.com/simonw/sqlite-utils/issues/104>`__)
+- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (:issue:`104`)
-- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (`#102 <https://github.com/simonw/sqlite-utils/issues/102>`__)
+- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (:issue:`102`)
 
 .. _v2_7:
 
 2.7 (2020-04-17)
 ----------------
 
-- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (`#100 <https://github.com/simonw/sqlite-utils/issues/100>`__)
+- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (:issue:`100`)
 
 .. _v2_6:
 
 2.6 (2020-04-15)
 ----------------
 
-- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (`#76 <https://github.com/simonw/sqlite-utils/issues/76>`__)
+- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (:issue:`76`)
 
 .. _v2_5:
 
 2.5 (2020-04-12)
 ----------------
 
-- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (`#96 <https://github.com/simonw/sqlite-utils/issues/96>`__)
+- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (:issue:`96`)
-- ``table.last_pk`` is now only available for inserts or upserts of a single record. (`#98 <https://github.com/simonw/sqlite-utils/issues/98>`__)
+- ``table.last_pk`` is now only available for inserts or upserts of a single record. (:issue:`98`)
-- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (`#97 <https://github.com/simonw/sqlite-utils/issues/97>`__)
+- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (:issue:`97`)
 
 .. _v2_4_4:
 
 2.4.4 (2020-03-23)
 ------------------
 
-- Fixed bug where columns with only null values were not correctly created. (`#95 <https://github.com/simonw/sqlite-utils/issues/95>`__)
+- Fixed bug where columns with only null values were not correctly created. (:issue:`95`)
 
 .. _v2_4_3:
 
 2.4.3 (2020-03-23)
 ------------------
 
-- Column type suggestion code is no longer confused by null values. (`#94 <https://github.com/simonw/sqlite-utils/issues/94>`__)
+- Column type suggestion code is no longer confused by null values. (:issue:`94`)
 
 .. _v2_4_2:
 
 2.4.2 (2020-03-14)
 ------------------
 
-- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (`#92 <https://github.com/simonw/sqlite-utils/issues/92>`__)
+- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (:issue:`92`)
 - Documentation for ``NotFoundError`` thrown by ``table.get(pk)`` - see :ref:`python_api_get`.
 
 .. _v2_4_1:
@@ -388,45 +566,45 @@ The theme of this release is better tools for working with binary data. The new
 2.4.1 (2020-03-01)
 ------------------
 
-- ``table.enable_fts()`` now works with columns that contain spaces. (`#90 <https://github.com/simonw/sqlite-utils/issues/90>`__)
+- ``table.enable_fts()`` now works with columns that contain spaces. (:issue:`90`)
 
 .. _v2_4:
 
 2.4 (2020-02-26)
 ----------------
 
-- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (`#88 <https://github.com/simonw/sqlite-utils/issues/88>`__)
+- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (:issue:`88`)
-- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (`#88 <https://github.com/simonw/sqlite-utils/issues/88>`__)
+- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (:issue:`88`)
-- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (`#86 <https://github.com/simonw/sqlite-utils/issues/86>`__)
+- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (:issue:`86`)
-- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (`#87 <https://github.com/simonw/sqlite-utils/issues/87>`__)
+- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (:issue:`87`)
 
 .. _v2_3_1:
 
 2.3.1 (2020-02-10)
 ------------------
 
-``table.create_index()`` now works for columns that contain spaces. (`#85 <https://github.com/simonw/sqlite-utils/issues/85>`__)
+``table.create_index()`` now works for columns that contain spaces. (:issue:`85`)
 
 .. _v2_3:
 
 2.3 (2020-02-08)
 ----------------
 
-``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (`#83 <https://github.com/simonw/sqlite-utils/issues/83>`__)
+``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (:issue:`83`)
 
 .. _v2_2_1:
 
 2.2.1 (2020-02-06)
 ------------------
 
-Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (`#84 <https://github.com/simonw/sqlite-utils/issues/84>`__).
+Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (:issue:`84`).
 
 .. _v2_2:
 
 2.2 (2020-02-01)
 ----------------
 
-New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (`#81 <https://github.com/simonw/sqlite-utils/issues/81>`__).
+New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (:issue:`81`).
 
 This replaces the undocumented ``table.detect_column_types()`` method.
 
@@ -442,7 +620,7 @@ New feature: ``conversions={...}`` can be passed to the ``.insert()`` family of
 2.0.1 (2020-01-05)
 ------------------
 
-The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (`#73 <https://github.com/simonw/sqlite-utils/issues/73>`__).
+The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (:issue:`73`).
 
 .. _v2:
 
@@ -464,14 +642,14 @@ For full background on this change, see `issue #66 <https://github.com/simonw/sq
 1.12.1 (2019-11-06)
 -------------------
 
-- Fixed error thrown when ``.insert_all()`` and ``.upsert_all()`` were called with empty lists (`#52 <https://github.com/simonw/sqlite-utils/issues/52>`__)
+- Fixed error thrown when ``.insert_all()`` and ``.upsert_all()`` were called with empty lists (:issue:`52`)
 
 .. _v1_12:
 
 1.12 (2019-11-04)
 -----------------
 
-Python library utilities for deleting records (`#62 <https://github.com/simonw/sqlite-utils/issues/62>`__)
+Python library utilities for deleting records (:issue:`62`)
 
 - ``db["tablename"].delete(4)`` to delete by primary key, see :ref:`python_api_delete`
 - ``db["tablename"].delete_where("id > ?", [3])`` to delete by a where clause, see :ref:`python_api_delete_where`
@@ -485,14 +663,14 @@ Option to create triggers to automatically keep FTS tables up-to-date with newly
 
 - ``sqlite-utils enable-fts ... --create-triggers`` - see :ref:`Configuring full-text search using the CLI <cli_fts>`
 - ``db["tablename"].enable_fts(..., create_triggers=True)`` - see :ref:`Configuring full-text search using the Python library <python_api_fts>`
-- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (`#59 <https://github.com/simonw/sqlite-utils/issues/59>`__)
+- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (:issue:`59`)
 
 .. _v1_10:
 
 1.10 (2019-08-23)
 -----------------
 
-Ability to introspect and run queries against views (`#54 <https://github.com/simonw/sqlite-utils/issues/54>`__)
+Ability to introspect and run queries against views (:issue:`54`)
 
 - ``db.view_names()`` method and and ``db.views`` property
 - Separate ``View`` and ``Table`` classes, both subclassing new ``Queryable`` class
@@ -505,21 +683,21 @@ See :ref:`python_api_views`.
 1.9 (2019-08-04)
 ----------------
 
-- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (`#23 <https://github.com/simonw/sqlite-utils/issues/23>`__)
+- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (:issue:`23`)
 
 .. _v1_8:
 
 1.8 (2019-07-28)
 ----------------
 
-- ``table.update(pk, values)`` method: :ref:`python_api_update` (`#35 <https://github.com/simonw/sqlite-utils/issues/35>`__)
+- ``table.update(pk, values)`` method: :ref:`python_api_update` (:issue:`35`)
 
 .. _v1_7_1:
 
 1.7.1 (2019-07-28)
 ------------------
 
-- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (`#50 <https://github.com/simonw/sqlite-utils/issues/50>`__)
+- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (:issue:`50`)
 - Documentation and tests for ``table.drop()`` method: :ref:`python_api_drop`
 
 .. _v1_7:
@@ -529,8 +707,8 @@ See :ref:`python_api_views`.
 
 Support for lookup tables.
 
-- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (`#44 <https://github.com/simonw/sqlite-utils/issues/44>`__)
+- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (:issue:`44`)
-- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (`#46 <https://github.com/simonw/sqlite-utils/issues/46>`__)
+- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (:issue:`46`)
 - Use `pysqlite3 <https://github.com/coleifer/pysqlite3>`__ if it is available, otherwise use ``sqlite3`` from the standard library
 - Table options can now be passed to the new ``db.table(name, **options)`` factory function in addition to being passed to ``insert_all(records, **options)`` and friends - see :ref:`python_api_table_configuration`
 - In-memory databases can now be created using ``db = Database(memory=True)``
@@ -540,19 +718,19 @@ Support for lookup tables.
 1.6 (2019-07-18)
 ----------------
 
-- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (`#41 <https://github.com/simonw/sqlite-utils/issues/41>`__)
+- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (:issue:`41`)
 
 .. _v1_5:
 
 1.5 (2019-07-14)
 ----------------
 
-- Support for compound primary keys (`#36 <https://github.com/simonw/sqlite-utils/issues/36>`__)
+- Support for compound primary keys (:issue:`36`)
 
   - Configure these using the CLI tool by passing ``--pk`` multiple times
   - In Python, pass a tuple of columns to the ``pk=(..., ...)`` argument: :ref:`python_api_compound_primary_keys`
 
-- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (`#39 <https://github.com/simonw/sqlite-utils/issues/39>`__)
+- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (:issue:`39`)
 
 .. _v1_4_1:
 
@@ -566,14 +744,14 @@ Support for lookup tables.
 1.4 (2019-06-30)
 ----------------
 
-- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs <cli_index_foreign_keys>`) and ``db.index_foreign_keys()`` method (:ref:`docs <python_api_index_foreign_keys>`) (`#33 <https://github.com/simonw/sqlite-utils/issues/33>`__)
+- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs <cli_index_foreign_keys>`) and ``db.index_foreign_keys()`` method (:ref:`docs <python_api_index_foreign_keys>`) (:issue:`33`)
 
 .. _v1_3:
 
 1.3 (2019-06-28)
 ----------------
 
-- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation <python_api_add_foreign_keys>` (`#31 <https://github.com/simonw/sqlite-utils/issues/31>`__)
+- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation <python_api_add_foreign_keys>` (:issue:`31`)
 
 .. _v1_2_2:
 
@@ -587,15 +765,15 @@ Support for lookup tables.
 1.2.1 (2019-06-20)
 ------------------
 
-- Check the column exists before attempting to add a foreign key (`#29 <https://github.com/simonw/sqlite-utils/issues/29>`__)
+- Check the column exists before attempting to add a foreign key (:issue:`29`)
 
 .. _v1_2:
 
 1.2 (2019-06-12)
 ----------------
 
-- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by instrospecting the database. See :ref:`python_api_add_foreign_key` for details. (`#25 <https://github.com/simonw/sqlite-utils/issues/25>`__)
+- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by introspecting the database. See :ref:`python_api_add_foreign_key` for details. (:issue:`25`)
-- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (`#24 <https://github.com/simonw/sqlite-utils/issues/24>`__). Documentation: :ref:`Setting defaults and not null constraints (Python API) <python_api_defaults_not_null>`, :ref:`Setting defaults and not null constraints (CLI) <cli_defaults_not_null>`
+- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (:issue:`24`). Documentation: :ref:`Setting defaults and not null constraints (Python API) <python_api_defaults_not_null>`, :ref:`Setting defaults and not null constraints (CLI) <cli_defaults_not_null>`
 - Support for ``not_null_default=X`` / ``--not-null-default`` for setting a ``NOT NULL DEFAULT 'x'`` when adding a new column. Documentation: :ref:`Adding columns (Python API) <python_api_add_column>`, :ref:`Adding columns (CLI) <cli_add_column>`
 
 .. _v1_1:
@@ -603,8 +781,8 @@ Support for lookup tables.
 1.1 (2019-05-28)
 ----------------
 
-- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key alread exists (`#21 <https://github.com/simonw/sqlite-utils/issues/21>`__) - documentation: :ref:`Inserting data (Python API) <python_api_bulk_inserts>`, :ref:`Inserting data (CLI) <cli_inserting_data>`
+- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key already exists (:issue:`21`) - documentation: :ref:`Inserting data (Python API) <python_api_bulk_inserts>`, :ref:`Inserting data (CLI) <cli_inserting_data>`
-- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (`#16 <https://github.com/simonw/sqlite-utils/issues/16>`__) - documentation: :ref:`Adding columns (Python API) <python_api_add_column>`, :ref:`Adding columns (CLI) <cli_add_column>`
+- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (:issue:`16`) - documentation: :ref:`Adding columns (Python API) <python_api_add_column>`, :ref:`Adding columns (CLI) <cli_add_column>`
 
 .. _v1_0_1:
 

docs/cli.rst

@@ -8,25 +8,31 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite database
 
 .. contents:: :local:
 
-.. _cli_query_json:
+.. _cli_query:
 
-Running queries and returning JSON
+Running SQL queries
-==================================
+===================
 
-You can execute a SQL query against a database and get the results back as JSON like this::
+The ``sqlite-utils query`` command lets you run queries directly against a SQLite database file. This is the default subcommand, so the following two examples work the same way::
 
     $ sqlite-utils query dogs.db "select * from dogs"
+    $ sqlite-utils dogs.db "select * from dogs"
+
+.. _cli_query_json:
+
+Returning JSON
+--------------
+
+The default format returned for queries is JSON::
+
+    $ sqlite-utils dogs.db "select * from dogs"
     [{"id": 1, "age": 4, "name": "Cleo"},
      {"id": 2, "age": 2, "name": "Pancakes"}]
 
-This is the default command for ``sqlite-utils``, so you can instead use this::
+.. _cli_query_nl:
 
-    $ sqlite-utils dogs.db "select * from dogs"
+Newline-delimited JSON
-
+~~~~~~~~~~~~~~~~~~~~~~
-You can pass named parameters to the query using ``-p``::
-
-    $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6
-    [{":num * :num2": 30}]
 
 Use ``--nl`` to get back newline-delimited JSON objects::
 
@@ -34,7 +40,12 @@ Use ``--nl`` to get back newline-delimited JSON objects::
     {"id": 1, "age": 4, "name": "Cleo"}
     {"id": 2, "age": 2, "name": "Pancakes"}
 
-You can use ``--arrays`` to request ararys instead of objects::
+.. _cli_query_arrays:
+
+JSON arrays
+~~~~~~~~~~~
+
+You can use ``--arrays`` to request arrays instead of objects::
 
     $ sqlite-utils dogs.db "select * from dogs" --arrays
     [[1, 4, "Cleo"],
@@ -62,6 +73,11 @@ If you want to pretty-print the output further, you can pipe it through ``python
         }
     ]
 
+.. _cli_query_binary_json:
+
+Binary data in JSON
+~~~~~~~~~~~~~~~~~~~
+
 Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this::
 
     $ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool
@@ -75,25 +91,11 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b
         }
     ]
 
-If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the comand will return the number of affected rows::
-
-    $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'"   
-    [{"rows_affected": 1}]
-
-You can run queries against a temporary in-memory database by passing ``:memory:`` as the filename::
-
-    $ sqlite-utils :memory: "select sqlite_version()"
-    [{"sqlite_version()": "3.29.0"}]
-
-You can load SQLite extension modules using the `--load-extension` option::
-
-    $ sqlite-utils :memory: "select spatialite_version()" --load-extension=/usr/local/lib/mod_spatialite.dylib
-    [{"spatialite_version()": "4.3.0a"}]
 
 .. _cli_json_values:
 
 Nested JSON values
-------------------
+~~~~~~~~~~~~~~~~~~
 
 If one of your columns contains JSON, by default it will be returned as an escaped string::
 
@@ -124,24 +126,10 @@ You can use the ``--json-cols`` option to automatically detect these JSON column
         }
     ]
 
-.. _cli_attach:
-
-Attaching additional databases
-------------------------------
-
-SQLite supports cross-database SQL queries, which can join data from tables in more than one database file.
-
-You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk.
-
-This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database::
-
-    sqlite-utils dogs.db --attach books books.db \
-       'select * from sqlite_master union all select * from books.sqlite_master'
-
 .. _cli_query_csv:
 
-Running queries and returning CSV
+Returning CSV or TSV
-=================================
+--------------------
 
 You can use the ``--csv`` option to return results as CSV::
 
@@ -165,8 +153,8 @@ Use ``--tsv`` instead of ``--csv`` to get back tab-separated values::
 
 .. _cli_query_table:
 
-Running queries and outputting a table
+Table-formatted output
-======================================
+----------------------
 
 You can use the ``--table`` option (or ``-t`` shortcut) to output query results as a table::
 
@@ -190,8 +178,8 @@ For a full list of table format options, run ``sqlite-utils query --help``.
 
 .. _cli_query_raw:
 
-Returning raw data from a query, such as binary content
+Returning raw data, such as binary content
-=======================================================
+------------------------------------------
 
 If your table contains binary data in a ``BLOB`` you can use the ``--raw`` option to output specific columns directly to standard out.
 
@@ -199,6 +187,182 @@ For example, to retrieve a binary image from a ``BLOB`` column and store it in a
 
     $ sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg
 
+
+.. _cli_query_parameters:
+
+Using named parameters
+----------------------
+
+You can pass named parameters to the query using ``-p``::
+
+    $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6
+    [{":num * :num2": 30}]
+
+These will be correctly quoted and escaped in the SQL query, providing a safe way to combine other values with SQL.
+
+.. _cli_query_update_insert_delete:
+
+UPDATE, INSERT and DELETE
+-------------------------
+
+If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows::
+
+    $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'"
+    [{"rows_affected": 1}]
+
+SQLite extensions
+-----------------
+
+You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`.
+
+::
+
+    $ sqlite-utils dogs.db "select spatialite_version()" --load-extension=spatialite
+    [{"spatialite_version()": "4.3.0a"}]
+
+.. _cli_query_attach:
+
+Attaching additional databases
+------------------------------
+
+SQLite supports cross-database SQL queries, which can join data from tables in more than one database file.
+
+You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk.
+
+This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database::
+
+    sqlite-utils dogs.db --attach books books.db \
+       'select * from sqlite_master union all select * from books.sqlite_master'
+
+.. _cli_memory:
+
+Querying data directly using an in-memory database
+==================================================
+
+The ``sqlite-utils memory`` command works similar to ``sqlite-utils query``, but allows you to execute queries against an in-memory database.
+
+You can also pass this command CSV or JSON files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite.
+
+Without any extra arguments, this command executes SQL against the in-memory database directly::
+
+    $ sqlite-utils memory 'select sqlite_version()'
+    [{"sqlite_version()": "3.35.5"}]
+
+It takes all of the same output formatting options as :ref:`sqlite-utils query <cli_query>`: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``::
+
+    $ sqlite-utils memory 'select sqlite_version()' --csv             
+    sqlite_version()
+    3.35.5
+    $ sqlite-utils memory 'select sqlite_version()' --table --fmt grid
+    +--------------------+
+    | sqlite_version()   |
+    +====================+
+    | 3.35.5             |
+    +--------------------+
+
+.. _cli_memory_csv_json:
+
+Running queries directly against CSV or JSON
+--------------------------------------------
+
+If you have data in CSV or JSON format you can load it into an in-memory SQLite database and run queries against it directly in a single command using ``sqlite-utils memory`` like this::
+
+    $ sqlite-utils memory data.csv "select * from data"
+
+You can pass multiple files to the command if you want to run joins between data from different files::
+
+    $ sqlite-utils memory one.csv two.json "select * from one join two on one.id = two.other_id"
+
+If your data is JSON it should be the same format supported by the :ref:`sqlite-utils insert command <cli_inserting_data>` - so either a single JSON object (treated as a single row) or a list of JSON objects.
+
+CSV data can be comma- or tab- delimited.
+
+The in-memory tables will be named after the files without their extensions. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table::
+
+    $ sqlite-utils memory example.csv "select * from t"
+
+To read from standard input, use either ``-`` or ``stdin`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name::
+
+    $ cat example.csv | sqlite-utils memory - "select * from stdin"
+
+Incoming CSV data will be assumed to use ``utf-8``. If your data uses a different character encoding you can specify that with ``--encoding``::
+
+    $ cat example.csv | sqlite-utils memory - "select * from stdin" --encoding=latin-1
+
+If you are joining across multiple CSV files they must all use the same encoding.
+
+Column types will be automatically detected in CSV or TSV data, using the same mechanism as ``--detect-types`` described in :ref:`cli_insert_csv_tsv`. You can pass the ``--no-detect-types`` option to disable this automatic type detection and treat all CSV and TSV columns as ``TEXT``.
+
+.. _cli_memory_explicit:
+
+Explicitly specifying the format
+--------------------------------
+
+By default, ``sqlite-utils memory`` will attempt to detect the incoming data format (JSON, TSV or CSV) automatically.
+
+You can instead specify an explicit format by adding a ``:csv``, ``:tsv``, ``:json`` or ``:nl`` (for newline-delimited JSON) suffix to the filename. For example::
+
+    $ sqlite-utils memory one.dat:csv two.dat:nl "select * from one union select * from two"
+
+Here the contents of ``one.dat`` will be treated as CSV and the contents of ``two.dat`` will be treated as newline-delimited JSON.
+
+To explicitly specify the format for data piped into the tool on standard input, use ``stdin:format`` - for example::
+
+    $ cat one.dat | sqlite-utils memory stdin:csv "select * from stdin"
+
+.. _cli_memory_attach:
+
+Joining in-memory data against existing databases using \-\-attach
+------------------------------------------------------------------
+
+The :ref:`attach option <cli_query_attach>` can be used to attach database files to the in-memory connection, enabling joins between in-memory data loaded from a file and tables in existing SQLite database files. An example::
+
+    $ echo "id\n1\n3\n5" | sqlite-utils memory - --attach trees trees.db \
+      "select * from trees.trees where rowid in (select id from stdin)"
+
+Here the ``--attach trees trees.db`` option makes the ``trees.db`` database available with an alias of ``trees``.
+
+``select * from trees.trees where ...`` can then query the ``trees`` table in that database.
+
+The CSV data that was piped into the script is available in the ``stdin`` table, so  ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content.
+
+.. _cli_memory_schema_dump_save:
+
+\-\-schema, \-\-dump and \-\-save
+---------------------------------
+
+To see the schema that will be created for a file or multiple files, use ``--schema``::
+
+    % sqlite-utils memory dogs.csv --schema
+    CREATE TABLE [dogs] (
+        [id] INTEGER,
+        [age] INTEGER,
+        [name] TEXT
+    );
+    CREATE VIEW t1 AS select * from [dogs];
+    CREATE VIEW t AS select * from [dogs];
+
+You can output SQL that will both create the tables and insert the full data used to populate the in-memory database using ``--dump``::
+
+    % sqlite-utils memory dogs.csv --dump
+    BEGIN TRANSACTION;
+    CREATE TABLE [dogs] (
+        [id] INTEGER,
+        [age] INTEGER,
+        [name] TEXT
+    );
+    INSERT INTO "dogs" VALUES('1','4','Cleo');
+    INSERT INTO "dogs" VALUES('2','2','Pancakes');
+    CREATE VIEW t1 AS select * from [dogs];
+    CREATE VIEW t AS select * from [dogs];
+    COMMIT;
+
+Passing ``--save other.db`` will instead use that SQL to populate a new database file::
+
+    % sqlite-utils memory dogs.csv --save dogs.db
+
+These features are mainly intended as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`.
+
 .. _cli_rows:
 
 Returning all rows in a table
@@ -268,7 +432,7 @@ Use ``--schema`` to include the schema of each table::
                [age] INTEGER,
                [name] TEXT)
 
-The ``--nl``, ``--csv``, ``--tsv`` and ``--table`` options are all available.
+The ``--nl``, ``--csv``, ``--tsv``, ``--table`` and ``--fmt`` options are also available.
 
 .. _cli_views:
 
@@ -293,6 +457,33 @@ It takes the same options as the ``tables`` command:
 * ``--tsv``
 * ``--table``
 
+.. _cli_indexes:
+
+Listing indexes
+===============
+
+The ``indexes`` command lists any indexes configured for the database::
+
+    $ sqlite-utils indexes covid.db --table
+    table                             index_name                                                seqno    cid  name                 desc  coll      key
+    --------------------------------  ------------------------------------------------------  -------  -----  -----------------  ------  ------  -----
+    johns_hopkins_csse_daily_reports  idx_johns_hopkins_csse_daily_reports_combined_key             0     12  combined_key            0  BINARY      1
+    johns_hopkins_csse_daily_reports  idx_johns_hopkins_csse_daily_reports_country_or_region        0      1  country_or_region       0  BINARY      1
+    johns_hopkins_csse_daily_reports  idx_johns_hopkins_csse_daily_reports_province_or_state        0      2  province_or_state       0  BINARY      1
+    johns_hopkins_csse_daily_reports  idx_johns_hopkins_csse_daily_reports_day                      0      0  day                     0  BINARY      1
+    ny_times_us_counties              idx_ny_times_us_counties_date                                 0      0  date                    1  BINARY      1
+    ny_times_us_counties              idx_ny_times_us_counties_fips                                 0      3  fips                    0  BINARY      1
+    ny_times_us_counties              idx_ny_times_us_counties_county                               0      1  county                  0  BINARY      1
+    ny_times_us_counties              idx_ny_times_us_counties_state                                0      2  state                   0  BINARY      1
+
+It shows indexes across all tables. To see indexes for specific tables, list those after the database::
+
+    $ sqlite-utils indexes covid.db johns_hopkins_csse_daily_reports --table
+
+The command defaults to only showing the columns that are explicitly part of the index. To also include auxiliary columns use the ``--aux`` option - these columns will be listed with a ``key`` of ``0``.
+
+The command takes the same format options as the ``tables`` and ``views`` commands.
+
 .. _cli_triggers:
 
 Listing triggers
@@ -321,6 +512,24 @@ It defaults to showing triggers for all tables. To see triggers for one or more
 
 The command takes the same format options as the ``tables`` and ``views`` commands.
 
+.. _cli_schema:
+
+Showing the schema
+==================
+
+The ``sqlite-utils schema`` command shows the full SQL schema for the database::
+
+    $ sqlite-utils schema dogs.db
+    CREATE TABLE "dogs" (
+        [id] INTEGER PRIMARY KEY,
+        [name] TEXT
+    );
+
+This will show the schema for every table and index in the database. To view the schema just for a specified subset of tables pass those as additional arguments::
+
+    $ sqlite-utils schema dogs.db dogs chickens
+    ...
+
 .. _cli_analyze_tables:
 
 Analyzing tables
@@ -408,6 +617,21 @@ The ``_analyze_tables_`` table has the following schema::
         PRIMARY KEY ([table], [column])
     );
 
+The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most common and least common values that look like this::
+
+    [
+        ["Del Libertador, Av", 5068],
+        ["Alberdi Juan Bautista Av.", 4612],
+        ["Directorio Av.", 4552],
+        ["Rivadavia, Av", 4532],
+        ["Yerbal", 4512],
+        ["Cosquín", 4472],
+        ["Estado Plurinacional de Bolivia", 4440],
+        ["Gordillo Timoteo", 4424],
+        ["Montiel", 4360],
+        ["Condarco", 4288]
+    ]
+
 .. _cli_inserting_data:
 
 Inserting JSON data
@@ -445,6 +669,23 @@ If you feed it a JSON list it will insert multiple records. For example, if ``do
         }
     ]
 
+You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so::
+
+    $ sqlite-utils insert dogs.db dogs dogs.json --pk=id
+
+You can skip inserting any records that have a primary key that already exists using ``--ignore``::
+
+    $ sqlite-utils insert dogs.db dogs dogs.json --ignore
+
+You can delete all the existing rows in the table before inserting the new records using ``--truncate``::
+
+    $ sqlite-utils insert dogs.db dogs dogs.json --truncate
+
+.. _cli_inserting_data_binary:
+
+Inserting binary data
+---------------------
+
 You can insert binary data into a BLOB column by first encoding it using base64 and then structuring it like this::
 
     [
@@ -457,17 +698,10 @@ You can insert binary data into a BLOB column by first encoding it using base64
         }
     ]
 
-You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so::
+.. _cli_inserting_data_nl_json:
 
-    $ sqlite-utils insert dogs.db dogs dogs.json --pk=id
+Inserting newline-delimited JSON
-
+--------------------------------
-You can skip inserting any records that have a primary key that already exists using ``--ignore``::
-
-    $ sqlite-utils insert dogs.db dogs dogs.json --ignore
-
-You can delete all the existing rows in the table before inserting the new records using ``--truncate``::
-
-    $ sqlite-utils insert dogs.db dogs dogs.json --truncate
 
 You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette <https://datasette.io/>`__ can export newline-delimited JSON, you can combine the two tools like so::
 
@@ -488,6 +722,49 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite
     207368,920 Kirkham St,37.760210314285,-122.47073935813
     188702,1501 Evans Ave,37.7422086702947,-122.387293152263
 
+.. _cli_inserting_data_flatten:
+
+Flattening nested JSON objects
+------------------------------
+
+``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table.
+
+If your data is nested you can use the ``--flatten`` option to create columns that are derived from the nested data.
+
+Consider this example document, in a file called ``log.json``::
+
+    {
+        "httpRequest": {
+            "latency": "0.112114537s",
+            "requestMethod": "GET",
+            "requestSize": "534",
+            "status": 200
+        },
+        "insertId": "6111722f000b5b4c4d4071e2",
+        "labels": {
+            "service": "datasette-io"
+        }
+    }
+
+Inserting this into a table using ``sqlite-utils insert logs.db logs log.json`` will create a table with the following schema::
+
+    CREATE TABLE [logs] (
+       [httpRequest] TEXT,
+       [insertId] TEXT,
+       [labels] TEXT
+    );
+
+With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db logs log.json --flatten`` will create the following schema instead::
+
+    CREATE TABLE [logs] (
+       [httpRequest_latency] TEXT,
+       [httpRequest_requestMethod] TEXT,
+       [httpRequest_requestSize] TEXT,
+       [httpRequest_status] INTEGER,
+       [insertId] TEXT,
+       [labels_service] TEXT
+    );
+
 .. _cli_insert_csv_tsv:
 
 Inserting CSV or TSV data
@@ -507,6 +784,31 @@ Data is expected to be encoded as Unicode UTF-8. If your data is an another char
 
 A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option.
 
+By default every column inserted from a CSV or TSV file will be of type ``TEXT``. To automatically detect column types - resulting in a mix of ``TEXT``, ``INTEGER`` and ``FLOAT`` columns, use the ``--detect-types`` option (or its shortcut ``-d``).
+
+For example, given a ``creatures.csv`` file containing this::
+
+    name,age,weight
+    Cleo,6,45.5
+    Dori,1,3.5
+
+The following command::
+
+    $ sqlite-utils insert creatures.db creatures creatures.csv --csv --detect-types
+
+Will produce this schema::
+
+    $ sqlite-utils schema creatures.db
+    CREATE TABLE "creatures" (
+       [name] TEXT,
+       [age] INTEGER,
+       [weight] FLOAT
+    );
+
+You can set the ``SQLITE_UTILS_DETECT_TYPES`` environment variable if you want ``--detect-types`` to be the default behavior::
+
+    $ export SQLITE_UTILS_DETECT_TYPES=1
+
 .. _cli_insert_csv_tsv_delimiter:
 
 Alternative delimiters and quote characters
@@ -661,6 +963,128 @@ The ``-`` argument indicates data should be read from standard input. The string
 
 When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``.
 
+.. _cli_convert:
+
+Converting data in columns
+==========================
+
+The ``convert`` command can be used to transform the data in a specified column - for example to parse a date string into an ISO timestamp, or to split a string of tags into a JSON array.
+
+The command accepts a database, table, one or more columns and a string of Python code to be executed against the values from those columns. The following example would replace the values in the ``headline`` column in the ``articles`` table with an upper-case version::
+
+    $ sqlite-utils convert content.db articles headline 'value.upper()'
+
+The Python code is passed as a string. Within that Python code the ``value`` variable will be the value of the current column.
+
+The code you provide will be compiled into a function that takes ``value`` as a single argument. If you break your function body into multiple lines the last line should be a ``return`` statement::
+
+    $ sqlite-utils convert content.db articles headline '
+    value = str(value)
+    return value.upper()'
+
+You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options. This example uses the ``textwrap`` module to wrap the ``content`` column at 100 characters::
+
+    $ sqlite-utils convert content.db articles content \
+        '"\n".join(textwrap.wrap(value, 100))' \
+        --import=textwrap
+
+The transformation will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``::
+
+    $ sqlite-utils convert content.db articles headline 'value.upper()' \
+        --where "headline like '%cat%'"
+
+You can include named parameters in your where clause and populate them using one or more ``--param`` options::
+
+    $ sqlite-utils convert content.db articles headline 'value.upper()' \
+        --where "headline like :like" \
+        --param like '%cat%'
+
+The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database.
+
+.. _cli_convert_recipes:
+
+sqlite-utils convert recipes
+----------------------------
+
+Various built-in recipe functions are available for common operations. These are:
+
+``r.jsonsplit(value, delimiter=',', type=<class 'str'>)``
+  Convert a string like ``a,b,c`` into a JSON array ``["a", "b", "c"]``
+
+  The ``delimiter`` parameter can be used to specify a different delimiter.
+
+  The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4.5`` the following::
+
+      r.jsonsplit(value, type=float)
+
+  Would produce an array like this: ``[1.2, 3.0, 4.5]``
+
+``r.parsedate(value, dayfirst=False, yearfirst=False)``
+  Parse a date and convert it to ISO date format: ``yyyy-mm-dd``
+
+  In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted.
+
+``r.parsedatetime(value, dayfirst=False, yearfirst=False)``
+  Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS``
+
+These recipes can be used in the code passed to ``sqlite-utils convert`` like this::
+
+    $ sqlite-utils convert my.db mytable mycolumn \
+      'r.jsonsplit(value)'
+
+To use any of the documented parameters, do this::
+
+    $ sqlite-utils convert my.db mytable mycolumn \
+      'r.jsonsplit(value, delimiter=":")'
+
+.. _cli_convert_output:
+
+Saving the result to a different column
+---------------------------------------
+
+The ``--output`` and ``--output-type`` options can be used to save the result of the conversion to a separate column, which will be created if that column does not already exist::
+
+    $ sqlite-utils convert content.db articles headline 'value.upper()' \
+      --output headline_upper
+
+The type of the created column defaults to ``text``, but a different column type can be specified using ``--output-type``. This example will create a new floating point column called ``id_as_a_float`` with a copy of each item's ID increased by 0.5::
+
+    $ sqlite-utils convert content.db articles id 'float(value) + 0.5' \
+      --output id_as_a_float \
+      --output-type float
+
+You can drop the original column at the end of the operation by adding ``--drop``.
+
+.. _cli_convert_multi:
+
+Converting a column into multiple columns
+-----------------------------------------
+
+Sometimes you may wish to convert a single column into multiple derived columns. For example, you may have a ``location`` column containing ``latitude,longitude`` values which you wish to split out into separate ``latitude`` and ``longitude`` columns.
+
+You can achieve this using the ``--multi`` option to ``sqlite-utils convert``. This option expects your Python code to return a Python dictionary: new columns well be created and populated for each of the keys in that dictionary.
+
+For the ``latitude,longitude`` example you would use the following::
+
+    $ sqlite-utils convert demo.db places location \
+    'bits = value.split(",")
+    return {
+      "latitude": float(bits[0]),
+      "longitude": float(bits[1]),
+    }' --multi
+
+The type of the returned values will be taken into account when creating the new columns. In this example, the resulting database schema will look like this:
+
+.. code-block:: sql
+
+    CREATE TABLE [places] (
+        [location] TEXT,
+        [latitude] FLOAT,
+        [longitude] FLOAT
+    );
+
+The code function can also return ``None``, in which case its output will be ignored. You can drop the original column at the end of the operation by adding ``--drop``.
+
 .. _cli_create_table:
 
 Creating tables
@@ -861,45 +1285,44 @@ Here's a more complex example that makes use of these options. It converts `this
         --fk-column country_id \
         --rename country_long name
 
-After running the above, the command ``sqlite3 global.db .schema`` reveals the following schema:
+After running the above, the command ``sqlite-utils schema global.db`` reveals the following schema:
 
 .. code-block:: sql
 
     CREATE TABLE [countries] (
-        [id] INTEGER PRIMARY KEY,
+       [id] INTEGER PRIMARY KEY,
-        [country] TEXT,
+       [country] TEXT,
-        [name] TEXT
+       [name] TEXT
+    );
+    CREATE TABLE "power_plants" (
+       [country_id] INTEGER,
+       [name] TEXT,
+       [gppd_idnr] TEXT,
+       [capacity_mw] TEXT,
+       [latitude] TEXT,
+       [longitude] TEXT,
+       [primary_fuel] TEXT,
+       [other_fuel1] TEXT,
+       [other_fuel2] TEXT,
+       [other_fuel3] TEXT,
+       [commissioning_year] TEXT,
+       [owner] TEXT,
+       [source] TEXT,
+       [url] TEXT,
+       [geolocation_source] TEXT,
+       [wepp_id] TEXT,
+       [year_of_capacity_data] TEXT,
+       [generation_gwh_2013] TEXT,
+       [generation_gwh_2014] TEXT,
+       [generation_gwh_2015] TEXT,
+       [generation_gwh_2016] TEXT,
+       [generation_gwh_2017] TEXT,
+       [generation_data_source] TEXT,
+       [estimated_generation_gwh] TEXT,
+       FOREIGN KEY([country_id]) REFERENCES [countries]([id])
     );
     CREATE UNIQUE INDEX [idx_countries_country_name]
         ON [countries] ([country], [name]);
-    CREATE TABLE IF NOT EXISTS "power_plants" (
-        [rowid] INTEGER PRIMARY KEY,
-        [country_id] INTEGER,
-        [name] TEXT,
-        [gppd_idnr] TEXT,
-        [capacity_mw] TEXT,
-        [latitude] TEXT,
-        [longitude] TEXT,
-        [primary_fuel] TEXT,
-        [other_fuel1] TEXT,
-        [other_fuel2] TEXT,
-        [other_fuel3] TEXT,
-        [commissioning_year] TEXT,
-        [owner] TEXT,
-        [source] TEXT,
-        [url] TEXT,
-        [geolocation_source] TEXT,
-        [wepp_id] TEXT,
-        [year_of_capacity_data] TEXT,
-        [generation_gwh_2013] TEXT,
-        [generation_gwh_2014] TEXT,
-        [generation_gwh_2015] TEXT,
-        [generation_gwh_2016] TEXT,
-        [generation_gwh_2017] TEXT,
-        [generation_data_source] TEXT,
-        [estimated_generation_gwh] TEXT,
-        FOREIGN KEY(country_id) REFERENCES countries(id)
-    );
 
 .. _cli_create_view:
 
@@ -1033,6 +1456,14 @@ Use the ``--unique`` option to create a unique index.
 
 Use ``--if-not-exists`` to avoid attempting to create the index if one with that name already exists.
 
+To add an index on a column in descending order, prefix the column with a hyphen. Since this can be confused for a command-line option you need to construct that like this::
+
+    $ sqlite-utils create-index mydb.db mytable -- col1 -col2 col3
+
+This will create an index on that table on ``(col1, col2 desc, col3)``.
+
+If your column names are already prefixed with a hyphen you'll need to manually execute a ``CREATE INDEX`` SQL statement to add indexes to them rather than using this tool.
+
 .. _cli_fts:
 
 Configuring full-text search
@@ -1181,16 +1612,29 @@ You can disable WAL mode using ``disable-wal``::
 
 Both of these commands accept one or more database files as arguments.
 
+.. _cli_dump:
+
+Dumping the database to SQL
+===========================
+
+The ``dump`` command outputs a SQL dump of the schema and full contents of the specified database file::
+
+    $ sqlite-utils dump mydb.db
+    BEGIN TRANSACTION;
+    CREATE TABLE ...
+    ...
+    COMMIT;
+
 .. _cli_load_extension:
 
 Loading SQLite extensions
 =========================
 
-Many of these commands have the ablity to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``.
+Many of these commands have the ability to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``.
 
 This option can be applied multiple times to load multiple extensions.
 
 Since `SpatiaLite <https://www.gaia-gis.it/fossil/libspatialite/index>`__ is commonly used with SQLite, the value ``spatialite`` is special: it will search for SpatiaLite in the most common installation locations, saving you from needing to remember exactly where that module is located::
 
-    $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite
+    $ sqlite-utils memory "select spatialite_version()" --load-extension=spatialite
     [{"spatialite_version()": "4.3.0a"}]

docs/codespell-ignore-words.txt

@@ -0,0 +1 @@
+doub

docs/conf.py

@@ -30,7 +30,12 @@ from subprocess import Popen, PIPE
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = []
+extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"]
+autodoc_member_order = "bysource"
+
+extlinks = {
+    "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"),
+}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]

docs/contributing.rst

@@ -0,0 +1,67 @@
+.. _contributing:
+
+==============
+ Contributing
+==============
+
+To work on this library locally, first checkout the code. Then create a new virtual environment::
+
+    git clone git@github.com:simonw/sqlite-utils
+    cd sqlite-utils
+    python3 -mvenv venv
+    source venv/bin/activate
+
+Or if you are using ``pipenv``::
+
+    pipenv shell
+
+Within the virtual environment running ``sqlite-utils`` should run your locally editable version of the tool. You can use ``which sqlite-utils`` to confirm that you are running the version that lives in your virtual environment.
+
+.. _contributing_tests:
+
+Running the tests
+=================
+
+To install the dependencies and test dependencies::
+
+    pip install -e '.[test]'
+
+To run the tests::
+
+    pytest
+
+.. _contributing_docs:
+
+Building the documentation
+==========================
+
+To build the documentation, first install the documentation dependencies::
+
+    pip install -e '.[docs]'
+
+Then run ``make livehtml`` from the ``docs/`` directory to start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file::
+
+    cd docs
+    make livehtml
+
+.. _contributing_linting:
+
+Linting and formatting
+======================
+
+``sqlite-utils`` uses `Black <https://black.readthedocs.io/>`__ for code formatting, and `flake8 <https://flake8.pycqa.org/>`__ and `mypy <https://mypy.readthedocs.io/>`__ for linting and type checking.
+
+Black is installed as part of ``pip install -e '.[test]'`` - you can then format your code by running it in the root of the project::
+
+    black .
+
+To install ``mypy`` and ``flake8`` run the following::
+
+    pip install -e '.[flake8,mypy]'
+
+Both commands can then be run in the root of the project like this::
+
+    flake8
+    mypy sqlite_utils
+
+All three of these tools are run by our CI mechanism against every commit and pull request.

docs/index.rst

@@ -13,7 +13,7 @@
 .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
    :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE
 
-*Python utility functions for manipulating SQLite databases*
+*CLI tool and Python utility functions for manipulating SQLite databases*
 
 This library and command-line utility helps create SQLite databases from an existing collection of data.
 
@@ -29,8 +29,9 @@ Contents
 .. toctree::
    :maxdepth: 3
 
+   installation
    cli
    python-api
+   reference
+   contributing
    changelog
-
-Take a look at `this script <https://github.com/simonw/russian-ira-facebook-ads-datasette/blob/master/fetch_and_build_russian_ads.py>`_ for an example of this library in action.

docs/installation.rst

@@ -0,0 +1,40 @@
+.. _installation:
+
+==============
+ Installation
+==============
+
+``sqlite-utils`` is tested on Linux, macOS and Windows.
+
+.. _installation_homebrew:
+
+Using Homebrew
+==============
+
+The :ref:`sqlite-utils command-line tool <cli>` can be installed on macOS using Homebrew::
+
+    brew install sqlite-utils
+
+If you have it installed and want to upgrade to the most recent release, you can run::
+
+    brew upgrade sqlite-utils
+
+Then run ``sqlite-utils --version`` to confirm the installed version.
+
+.. _installation_pip:
+
+Using pip
+=========
+
+The `sqlite-utils package <https://pypi.org/project/sqlite-utils/>`__ on PyPI includes both the :ref:`sqlite_utils Python library <python_api>` and the ``sqlite-utils`` command-line tool. You can install them using ``pip`` like so::
+
+    pip install sqlite-utils
+
+.. _installation_pipx:
+
+Using pipx
+==========
+
+`pipx <https://pypi.org/project/pipx/>`__ is a tool for installing Python command-line applications in their own isolated environments. You can use ``pipx`` to install the ``sqlite-utils`` command-line tool like this::
+
+    pipx install sqlite-utils

docs/python-api.rst

@@ -1,8 +1,8 @@
 .. _python_api:
 
-============
+=============================
- Python API
+ sqlite_utils Python library
-============
+=============================
 
 .. contents:: :local:
 
@@ -59,12 +59,11 @@ You can attach an additional database using the ``.attach()`` method, providing
     db = Database("first.db")
     db.attach("second", "second.db")
     # Now you can run queries like this one:
-    cursor = db.execute("""
+    print(db.query("""
     select * from table_in_first
         union all
     select * from second.table_in_second
-    """)
+    """))
-    print(cursor.fetchall())
 
 You can reference tables in the attached database using the alias value you passed to ``db.attach(alias, filepath)`` as a prefix, for example the ``second.table_in_second`` reference in the SQL query above.
 
@@ -97,27 +96,77 @@ You can also turn on a tracer function temporarily for a block of code using the
 
 This example will print queries only for the duration of the ``with`` block.
 
-.. _python_api_execute:
+.. _python_api_executing_queries:
 
 Executing queries
 =================
 
-The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the tracer function if one has been registered.
+The ``Database`` class offers several methods for directly executing SQL queries.
+
+.. _python_api_query:
+
+db.query(sql, params)
+---------------------
+
+The ``db.query(sql)`` function executes a SQL query and returns an iterator over Python dictionaries representing the resulting rows:
+
+.. code-block:: python
+
+    db = Database(memory=True)
+    db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}])
+    for row in db.query("select * from dogs"):
+        print(row)
+    # Outputs:
+    # {'name': 'Cleo'}
+    # {'name': 'Pancakes'}
+
+.. _python_api_execute:
+
+db.execute(sql, params)
+-----------------------
+
+The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the :ref:`tracer function <python_api_tracing>` if one has been registered.
+
+``db.execute(sql)`` returns a `sqlite3.Cursor <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor>`__ that was used to execute the SQL.
 
 .. code-block:: python
 
     db = Database(memory=True)
     db["dogs"].insert({"name": "Cleo"})
-    db.execute("update dogs set name = 'Cleopaws'")
+    cursor = db.execute("update dogs set name = 'Cleopaws'")
+    print(cursor.rowcount)
+    # Outputs the number of rows affected by the update
+    # In this case 2
 
-You can pass parameters as an optional second argument, using either a list or a dictionary. These will be correctly quoted and escaped.
+Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor>`__.
+
+.. _python_api_parameters:
+
+Passing parameters
+------------------
+
+Both ``db.query()`` and ``db.execute()`` accept an optional second argument for parameters to be passed to the SQL query.
+
+This can take the form of either a tuple/list or a dictionary, depending on the type of parameters used in the query. Values passed in this way will be correctly quoted and escaped, helping avoid XSS vulnerabilities.
+
+``?`` parameters in the SQL query can be filled in using a list:
 
 .. code-block:: python
 
-    # Using ? and a list:
     db.execute("update dogs set name = ?", ["Cleopaws"])
-    # Or using :name and a dictionary:
+    # This will rename ALL dogs to be called "Cleopaws"
-    db.execute("update dogs set name = :name", {"name": "Cleopaws"})
+
+Named parameters using ``:name`` can be filled using a dictionary:
+
+.. code-block:: python
+
+    dog = next(db.query(
+        "select rowid, name from dogs where name = :name",
+        {"name": "Cleopaws"}
+    ))
+    # dog is now {'rowid': 1, 'name': 'Cleopaws'}
+
+In this example ``next()`` is used to retrieve the first result in the iterator returned by the ``db.query()`` method.
 
 .. _python_api_table:
 
@@ -202,7 +251,13 @@ You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``::
     ...     print(row)
     {'id': 1, 'age': 4, 'name': 'Cleo'}
 
-To return custom columns (instead of using ``select *``) pass ``select=``::
+The first argument is a fragment of SQL. The second, optional argument is values to be passed to that fragment - you can use ``?`` placeholders and pass an array, or you can use ``:named`` parameters and pass a dictionary, like this::
+
+    >>> for row in db["dogs"].rows_where("age > :age", {"age": 3}):
+    ...     print(row)
+    {'id': 1, 'age': 4, 'name': 'Cleo'}
+
+To return custom columns (instead of the default that uses ``select *``) pass ``select="column1, column2"``::
 
     >>> db = sqlite_utils.Database("dogs.db")
     >>> for row in db["dogs"].rows_where(select='name, age'):
@@ -231,6 +286,16 @@ This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an
     ...     print(row)
     {'id': 1, 'age': 4, 'name': 'Cleo'}
 
+.. _python_api_rows_count_where:
+
+Counting rows
+-------------
+
+To count the number of rows that would be returned by a where filter, use ``.count_where(where, where_args)``:
+
+    >>> db["dogs"].count_where("age > ?", [1]):
+    2
+
 .. _python_api_pks_and_rows_where:
 
 Listing rows with their primary keys
@@ -291,6 +356,21 @@ If the record does not exist a ``NotFoundError`` will be raised:
     except NotFoundError:
         print("Dog not found")
 
+.. _python_api_schema:
+
+Showing the schema
+==================
+
+The ``db.schema`` property returns the full SQL schema for the database as a string::
+
+    >>> db = sqlite_utils.Database("dogs.db")
+    >>> print(db.schema)
+    >>> print(db.schema)
+    CREATE TABLE "dogs" (
+        [id] INTEGER PRIMARY KEY,
+        [name] TEXT
+    );
+
 .. _python_api_creating_tables:
 
 Creating tables
@@ -603,7 +683,7 @@ The first argument to ``update()`` is the primary key. This can be a single valu
 
     >>> db["compound_dogs"].update((5, 3), {"name": "Updated"})
 
-The second argument is a dictonary of columns that should be updated, along with their new values.
+The second argument is a dictionary of columns that should be updated, along with their new values.
 
 You can cause any missing columns to be added automatically using ``alter=True``::
 
@@ -632,7 +712,7 @@ You can delete all records in a table that match a specific WHERE statement usin
 
     >>> db = sqlite_utils.Database("dogs.db")
     >>> # Delete every dog with age less than 3
-    >>> db["dogs"].delete_where("age < ?", [3]):
+    >>> db["dogs"].delete_where("age < ?", [3])
 
 Calling ``table.delete_where()`` with no other arguments will delete every row in the table.
 
@@ -647,7 +727,7 @@ For example, given the dogs database you could upsert the record for Cleo like s
 
 .. code-block:: python
 
-    db["dogs"].upsert([{
+    db["dogs"].upsert({
         "id": 1,
         "name": "Cleo",
         "twitter": "cleopaws",
@@ -666,6 +746,53 @@ An ``upsert_all()`` method is also available, which behaves like ``insert_all()`
 .. note::
     ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 <https://github.com/simonw/sqlite-utils/issues/66>`__ for details of this change.
 
+.. _python_api_convert:
+
+Converting data in columns
+==========================
+
+The ``table.convert(...)`` method can be used to apply a conversion function to the values in a column, either to update that column or to populate new columns. It is the Python library equivalent of the :ref:`sqlite-utils convert <cli_convert>` command.
+
+This feature works by registering a custom SQLite function that applies a Python transformation, then running a SQL query equivalent to ``UPDATE table SET column = convert_value(column);``
+
+To transform a specific column to uppercase, you would use the following:
+
+.. code-block:: python
+
+    db["dogs"].convert("name", lambda value: value.upper())
+
+You can pass a list of columns, in which case the transformation will be applied to each one:
+
+.. code-block:: python
+
+    db["dogs"].convert(["name", "twitter"], lambda value: value.upper())
+
+To save the output to of the transformation to a different column, use the ``output=`` parameter:
+
+.. code-block:: python
+
+    db["dogs"].convert("name", lambda value: value.upper(), output="name_upper")
+
+This will add the new column, if it does not already exist. You can pass ``output_type=int`` or some other type to control the type of the new column - otherwise it will default to text.
+
+If you want to drop the original column after saving the results in a separate output column, pass ``drop=True``.
+
+You can create multiple new columns from a single input column by passing ``multi=True`` and a conversion function that returns a Python dictionary. This example creates new ``upper`` and ``lower`` columns populated from the single ``title`` column:
+
+.. code-block:: python
+
+    table.convert(
+        "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True
+    )
+
+The ``.convert()`` method accepts optional ``where=`` and ``where_args=`` parameters which can be used to apply the conversion to a subset of rows specified by a where clause. Here's how to apply the conversion only to rows with an ``id`` that is higher than 20:
+
+.. code-block:: python
+
+    table.convert("title", lambda v: v.upper(), where="id > :id", where_args={"id": 20})
+
+These behave the same as the corresponding parameters to the :ref:`.rows_where() <python_api_rows>` method, so you can use ``?`` placeholders and a list of values instead of ``:named`` placeholders with a dictionary.
+
 .. _python_api_lookup_tables:
 
 Working with lookup tables
@@ -835,7 +962,7 @@ The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` metho
     The name of the column
 
 ``total_rows``
-    The total number of rows in the table`
+    The total number of rows in the table
 
 ``num_null``
     The number of rows for which this column is null
@@ -969,7 +1096,7 @@ Here's an example of this mechanism in action:
     ])
     db["books"].add_foreign_key("author_id", "authors", "id")
 
-The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you ommit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules:
+The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you omit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules:
 
 - If the column is of format ``author_id``, look for tables called ``author`` or ``authors``
 - If the column does not end in ``_id``, try looking for a table with the exact name of the column or that name with an added ``s``
@@ -1050,6 +1177,9 @@ The ``table.transform()`` method can do all of these things, by implementing a m
 
 The ``.transform()`` method takes a number of parameters, all of which are optional.
 
+Altering column types
+---------------------
+
 To alter the type of a column, use the ``types=`` argument:
 
 .. code-block:: python
@@ -1057,6 +1187,11 @@ To alter the type of a column, use the ``types=`` argument:
     # Convert the 'age' column to an integer, and 'weight' to a float
     table.transform(types={"age": int, "weight": float})
 
+See :ref:`python_api_add_column` for a list of available types.
+
+Renaming columns
+----------------
+
 The ``rename=`` parameter can rename columns:
 
 .. code-block:: python
@@ -1064,6 +1199,9 @@ The ``rename=`` parameter can rename columns:
     # Rename 'age' to 'initial_age':
     table.transform(rename={"age": "initial_age"})
 
+Dropping columns
+----------------
+
 To drop columns, pass them in the ``drop=`` set:
 
 .. code-block:: python
@@ -1071,6 +1209,9 @@ To drop columns, pass them in the ``drop=`` set:
     # Drop the 'age' column:
     table.transform(drop={"age"})
 
+Changing primary keys
+---------------------
+
 To change the primary key for a table, use ``pk=``. This can be passed a single column for a regular primary key, or a tuple of columns to create a compound primary key. Passing ``pk=None`` will remove the primary key and convert the table into a ``rowid`` table.
 
 .. code-block:: python
@@ -1078,6 +1219,9 @@ To change the primary key for a table, use ``pk=``. This can be passed a single
     # Make `user_id` the new primary key
     table.transform(pk="user_id")
 
+Changing not null status
+------------------------
+
 You can change the ``NOT NULL`` status of columns by using ``not_null=``. You can pass this a set of columns to make those columns ``NOT NULL``:
 
 .. code-block:: python
@@ -1095,6 +1239,9 @@ If you want to take existing ``NOT NULL`` columns and change them to allow null
     # Make age allow NULL and switch weight to being NOT NULL:
     table.transform(not_null={"age": False, "weight": True})
 
+Altering column defaults
+------------------------
+
 The ``defaults=`` parameter can be used to set or change the defaults for different columns:
 
 .. code-block:: python
@@ -1105,6 +1252,9 @@ The ``defaults=`` parameter can be used to set or change the defaults for differ
     # Now remove the default from that column:
     table.transform(defaults={"age": None})
 
+Changing column order
+---------------------
+
 The ``column_order=`` parameter can be used to change the order of the columns. If you pass the names of a subset of the columns those will go first and columns you omitted will appear in their existing order after them.
 
 .. code-block:: python
@@ -1112,6 +1262,9 @@ The ``column_order=`` parameter can be used to change the order of the columns.
     # Change column order
     table.transform(column_order=("name", "age", "id")
 
+Dropping foreign key constraints
+--------------------------------
+
 You can use ``.transform()`` to remove foreign key constraints from a table.
 
 This example drops two foreign keys - the one from ``places.country`` to ``country.id`` and the one from ``places.continent`` to ``continent.id``:
@@ -1443,6 +1596,11 @@ If you have loaded an existing table or view, you can use introspection to find
     >>> db["PlantType"]
     <Table PlantType (id, value)>
 
+.. _python_api_introspection_exists:
+
+.exists()
+---------
+
 The ``.exists()`` method can be used to find out if a table exists or not::
 
     >>> db["PlantType"].exists()
@@ -1450,6 +1608,11 @@ The ``.exists()`` method can be used to find out if a table exists or not::
     >>> db["PlantType2"].exists()
     False
 
+.. _python_api_introspection_count:
+
+.count
+------
+
 The ``.count`` property shows the current number of rows (``select count(*) from table``)::
 
     >>> db["PlantType"].count
@@ -1457,25 +1620,60 @@ The ``.count`` property shows the current number of rows (``select count(*) from
     >>> db["Street_Tree_List"].count
     189144
 
-This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.execute_count()`` instead of accessing the property.
+This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.count_where()`` instead of accessing the property.
 
-The ``.columns`` property shows the columns in the table or view::
+.. _python_api_introspection_columns:
+
+.columns
+--------
+
+The ``.columns`` property shows the columns in the table or view. It returns a list of ``Column(cid, name, type, notnull, default_value, is_pk)`` named tuples.
+
+::
 
     >>> db["PlantType"].columns
     [Column(cid=0, name='id', type='INTEGER', notnull=0, default_value=None, is_pk=1),
      Column(cid=1, name='value', type='TEXT', notnull=0, default_value=None, is_pk=0)]
 
-The ``.columns_dict`` property returns a dictionary version of this with just the names and types::
+.. _python_api_introspection_columns_dict:
+
+.columns_dict
+-------------
+
+The ``.columns_dict`` property returns a dictionary version of the columns with just the names and Python types::
 
     >>> db["PlantType"].columns_dict
     {'id': <class 'int'>, 'value': <class 'str'>}
 
+.. _python_api_introspection_pks:
+
+.pks
+----
+
 The ``.pks`` property returns a list of strings naming the primary key columns for the table::
 
     >>> db["PlantType"].pks
     ['id']
 
-The ``.foreign_keys`` property shows if the table has any foreign key relationships. It is not available on views.
+If a table has no primary keys but is a `rowid table <https://www.sqlite.org/rowidtable.html>`__, this property will return ``['rowid']``.
+
+.. _python_api_introspection_use_rowid:
+
+.use_rowid
+----------
+
+Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly defined primary keys must use that ``rowid`` as the primary key for identifying individual rows. The ``.use_rowid`` property checks to see if a table needs to use the ``rowid`` in this way - it returns ``True`` if the table has no explicitly defined primary keys and ``False`` otherwise.
+
+    >>> db["PlantType"].use_rowid
+    False
+
+
+.. _python_api_introspection_foreign_keys:
+
+.foreign_keys
+-------------
+
+The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views.
 
 ::
 
@@ -1487,6 +1685,11 @@ The ``.foreign_keys`` property shows if the table has any foreign key relationsh
      ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'),
      ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')]
 
+.. _python_api_introspection_schema:
+
+.schema
+-------
+
 The ``.schema`` property outputs the table's schema as a SQL string::
 
     >>> print(db["Street_Tree_List"].schema)
@@ -1517,7 +1720,12 @@ The ``.schema`` property outputs the table's schema as a SQL string::
         FOREIGN KEY ("qCareAssistant") REFERENCES [qCareAssistant](id),
         FOREIGN KEY ("qLegalStatus") REFERENCES [qLegalStatus](id))
 
-The ``.indexes`` property shows you all indexes created for a table. It is not available on views.
+.. _python_api_introspection_indexes:
+
+.indexes
+--------
+
+The ``.indexes`` property returns all indexes created for a table, as a list of ``Index(seq, name, unique, origin, partial, columns)`` named tuples. It is not available on views.
 
 ::
 
@@ -1529,7 +1737,39 @@ The ``.indexes`` property shows you all indexes created for a table. It is not a
      Index(seq=4, name='"Street_Tree_List_qCaretaker"', unique=0, origin='c', partial=0, columns=['qCaretaker']),
      Index(seq=5, name='"Street_Tree_List_PlantType"', unique=0, origin='c', partial=0, columns=['PlantType'])]
 
-The ``.triggers`` property lists database triggers. It can be used on both database and table objects.
+.. _python_api_introspection_xindexes:
+
+.xindexes
+---------
+
+The ``.xindexes`` property returns more detailed information about the indexes on the table, using the SQLite `PRAGMA index_xinfo() <https://sqlite.org/pragma.html#pragma_index_xinfo>`__ mechanism. It returns a list of ``XIndex(name, columns)`` named tuples, where ``columns`` is a list of ``XIndexColumn(seqno, cid, name, desc, coll, key)`` named tuples.
+
+::
+
+    >>> db["ny_times_us_counties"].xindexes
+    [
+        XIndex(
+            name='idx_ny_times_us_counties_date',
+            columns=[
+                XIndexColumn(seqno=0, cid=0, name='date', desc=1, coll='BINARY', key=1),
+                XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0)
+            ]
+        ),
+        XIndex(
+            name='idx_ny_times_us_counties_fips',
+            columns=[
+                XIndexColumn(seqno=0, cid=3, name='fips', desc=0, coll='BINARY', key=1),
+                XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0)
+            ]
+        )
+    ]
+
+.. _python_api_introspection_triggers:
+
+.triggers
+---------
+
+The ``.triggers`` property lists database triggers. It can be used on both database and table objects. It returns a list of ``Trigger(name, table, sql)`` named tuples.
 
 ::
 
@@ -1540,6 +1780,11 @@ The ``.triggers`` property lists database triggers. It can be used on both datab
     >>> db.triggers
     ... similar output to db["authors"].triggers
 
+.. _python_api_introspection_triggers_dict:
+
+.triggers_dict
+--------------
+
 The ``.triggers_dict`` property returns the triggers for that table as a dictionary mapping their names to their SQL definitions.
 
 ::
@@ -1558,6 +1803,11 @@ The same property exists on the database, and will return all triggers across al
      'authors_ad': 'CREATE TRIGGER [authors_ad] AFTER DELETE...',
      'authors_au': 'CREATE TRIGGER [authors_au] AFTER UPDATE'}
 
+.. _python_api_introspection_detect_fts:
+
+.detect_fts()
+-------------
+
 The ``detect_fts()`` method returns the associated SQLite FTS table name, if one exists for this table. If the table has not been configured for full-text search it returns ``None``.
 
 ::
@@ -1565,12 +1815,22 @@ The ``detect_fts()`` method returns the associated SQLite FTS table name, if one
     >>> db["authors"].detect_fts()
     "authors_fts"
 
+.. _python_api_introspection_virtual_table_using:
+
+.virtual_table_using
+--------------------
+
 The ``.virtual_table_using`` property reveals if a table is a virtual table. It returns ``None`` for regular tables and the upper case version of the type of virtual table otherwise. For example::
 
     >>> db["authors"].enable_fts(["name"])
     >>> db["authors_fts"].virtual_table_using
     "FTS5"
 
+.. _python_api_introspection_has_counts_triggers:
+
+.has_counts_triggers
+--------------------
+
 The ``.has_counts_triggers`` property shows if a table has been configured with triggers for updating a ``_counts`` table, as described in :ref:`python_api_cached_table_counts`.
 
 ::
@@ -1848,6 +2108,8 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y
 
     db.reset_counts()
 
+.. _python_api_create_index:
+
 Creating indexes
 ================
 
@@ -1866,6 +2128,17 @@ By default the index will be named ``idx_{table-name}_{columns}`` - if you want
         index_name="good_dogs_by_age"
     )
 
+To create an index in descending order for a column, wrap the column name in ``db.DescIndex()`` like this:
+
+.. code-block:: python
+
+    from sqlite_utils.db import DescIndex
+
+    db["dogs"].create_index(
+        ["is_good_dog", DescIndex("age")],
+        index_name="good_dogs_by_age"
+    )
+
 You can create a unique index by passing ``unique=True``:
 
 .. code-block:: python
@@ -2050,12 +2323,24 @@ If you want to deliberately replace the registered function with a new implement
     def reverse_string(s):
         return s[::-1]
 
+Exceptions that occur inside a user-defined function default to returning the following error::
+
+    Unexpected error: user-defined function raised exception
+
+You can cause ``sqlite3`` to return more useful errors, including the traceback from the custom function, by executing the following before your custom functions are executed:
+
+.. code-block:: python
+
+    from sqlite_utils.utils import sqlite3
+
+    sqlite3.enable_callback_tracebacks(True)
+
 .. _python_api_quote:
 
 Quoting strings for use in SQL
 ==============================
 
-In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.execute()``, as described in :ref:`python_api_execute`.
+In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.query()``, as described in :ref:`python_api_parameters`.
 
 If that option isn't relevant to your use-case you can to quote a string for use with SQLite using the ``db.quote()`` method, like so:
 

docs/reference.rst

@@ -0,0 +1,70 @@
+.. _reference:
+
+===============
+ API Reference
+===============
+
+.. contents:: :local:
+
+.. _reference_db_database:
+
+sqlite_utils.db.Database
+========================
+
+.. autoclass:: sqlite_utils.db.Database
+    :members:
+    :undoc-members:
+    :special-members: __getitem__
+    :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys
+
+.. _reference_db_queryable:
+
+sqlite_utils.db.Queryable
+=========================
+
+:ref:`Table <reference_db_table>` and :ref:`View <reference_db_view>` are  both subclasses of ``Queryable``, providing access to the following methods:
+
+.. autoclass:: sqlite_utils.db.Queryable
+    :members:
+    :undoc-members:
+    :exclude-members: execute_count
+
+.. _reference_db_table:
+
+sqlite_utils.db.Table
+=====================
+
+.. autoclass:: sqlite_utils.db.Table
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :exclude-members: guess_foreign_column, value_or_default, build_insert_queries_and_params, insert_chunk, add_missing_columns
+
+.. _reference_db_view:
+
+sqlite_utils.db.View
+====================
+
+.. autoclass:: sqlite_utils.db.View
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. _reference_db_other:
+
+Other
+=====
+
+.. _reference_db_other_column:
+
+sqlite_utils.db.Column
+----------------------
+
+.. autoclass:: sqlite_utils.db.Column
+
+.. _reference_db_other_column_details:
+
+sqlite_utils.db.ColumnDetails
+-----------------------------
+
+.. autoclass:: sqlite_utils.db.ColumnDetails

setup.cfg

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 160
+extend-ignore = E203 # for Black

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import io
 import os
 
-VERSION = "3.6"
+VERSION = "3.15.1"
 
 
 def get_long_description():
@@ -22,11 +22,19 @@ setup(
     version=VERSION,
     license="Apache License, Version 2.0",
     packages=find_packages(exclude=["tests", "tests.*"]),
-    install_requires=["sqlite-fts4", "click", "click-default-group", "tabulate"],
+    install_requires=[
+        "sqlite-fts4",
+        "click",
+        "click-default-group",
+        "tabulate",
+        "dateutils",
+    ],
     setup_requires=["pytest-runner"],
     extras_require={
         "test": ["pytest", "black", "hypothesis"],
-        "docs": ["sphinx_rtd_theme", "sphinx-autobuild"],
+        "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"],
+        "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"],
+        "flake8": ["flake8"],
     },
     entry_points="""
         [console_scripts]

sqlite_utils/cli.py

@@ -1,12 +1,14 @@
 import base64
 import click
-from click_default_group import DefaultGroup
+from click_default_group import DefaultGroup  # type: ignore
 from datetime import datetime
 import hashlib
 import pathlib
 import sqlite_utils
-from sqlite_utils.db import AlterError
+from sqlite_utils.db import AlterError, BadMultiValues, DescIndex
+from sqlite_utils import recipes
 import textwrap
+import inspect
 import io
 import itertools
 import json
@@ -14,7 +16,18 @@ import os
 import sys
 import csv as csv_std
 import tabulate
-from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values
+from .utils import (
+    file_progress,
+    find_spatialite,
+    sqlite3,
+    decode_base64_values,
+    progressbar,
+    rows_from_file,
+    Format,
+    TypeTracker,
+)
+
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 
 VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB")
 
@@ -31,7 +44,7 @@ It's often worth trying: --encoding=latin-1
 """.strip()
 
 
-# Increase CSV field size limit to maximim possible
+# Increase CSV field size limit to maximum possible
 # https://stackoverflow.com/a/15063941
 field_size_limit = sys.maxsize
 
@@ -89,7 +102,12 @@ def load_extension_option(fn):
     )(fn)
 
 
-@click.group(cls=DefaultGroup, default="query", default_if_no_args=True)
+@click.group(
+    cls=DefaultGroup,
+    default="query",
+    default_if_no_args=True,
+    context_settings=CONTEXT_SETTINGS,
+)
 @click.version_option()
 def cli():
     "Commands for interacting with a SQLite database"
@@ -245,17 +263,6 @@ def views(
     )
 
 
-@cli.command()
-@click.argument(
-    "path",
-    type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
-    required=True,
-)
-def vacuum(path):
-    """Run VACUUM against the database"""
-    sqlite_utils.Database(path).vacuum()
-
-
 @cli.command()
 @click.argument(
     "path",
@@ -308,6 +315,21 @@ def vacuum(path):
     sqlite_utils.Database(path).vacuum()
 
 
+@cli.command()
+@click.argument(
+    "path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@load_extension_option
+def dump(path, load_extension):
+    """Output a SQL dump of the schema and full contents of the database"""
+    db = sqlite_utils.Database(path)
+    _load_extensions(db, load_extension)
+    for line in db.conn.iterdump():
+        click.echo(line)
+
+
 @cli.command(name="add-column")
 @click.argument(
     "path",
@@ -450,11 +472,21 @@ def index_foreign_keys(path, load_extension):
 )
 @load_extension_option
 def create_index(path, table, column, name, unique, if_not_exists, load_extension):
-    "Add an index to the specified table covering the specified columns"
+    """
+    Add an index to the specified table covering the specified columns.
+    Use "sqlite-utils create-index mydb -- -column" to specify descending
+    order for a column.
+    """
     db = sqlite_utils.Database(path)
     _load_extensions(db, load_extension)
+    # Treat -prefix as descending for columns
+    columns = []
+    for col in column:
+        if col.startswith("-"):
+            col = DescIndex(col[1:])
+        columns.append(col)
     db[table].create_index(
-        column, index_name=name, unique=unique, if_not_exists=if_not_exists
+        columns, index_name=name, unique=unique, if_not_exists=if_not_exists
     )
 
 
@@ -611,6 +643,7 @@ def insert_upsert_options(fn):
                 "--pk", help="Columns to use as the primary key, e.g. id", multiple=True
             ),
             click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"),
+            click.option("--flatten", is_flag=True, help="Flatten nested JSON objects"),
             click.option("-c", "--csv", is_flag=True, help="Expect CSV"),
             click.option("--tsv", is_flag=True, help="Expect TSV"),
             click.option("--delimiter", help="Delimiter to use for CSV files"),
@@ -644,6 +677,13 @@ def insert_upsert_options(fn):
                 "--encoding",
                 help="Character encoding for input, defaults to utf-8",
             ),
+            click.option(
+                "-d",
+                "--detect-types",
+                is_flag=True,
+                envvar="SQLITE_UTILS_DETECT_TYPES",
+                help="Detect types for columns in CSV/TSV data",
+            ),
             load_extension_option,
             click.option("--silent", is_flag=True, help="Do not show progress bar"),
         )
@@ -658,6 +698,7 @@ def insert_upsert_implementation(
     json_file,
     pk,
     nl,
+    flatten,
     csv,
     tsv,
     delimiter,
@@ -673,6 +714,7 @@ def insert_upsert_implementation(
     not_null=None,
     default=None,
     encoding=None,
+    detect_types=None,
     load_extension=None,
     silent=False,
 ):
@@ -682,13 +724,16 @@ def insert_upsert_implementation(
         csv = True
     if (nl + csv + tsv) >= 2:
         raise click.ClickException("Use just one of --nl, --csv or --tsv")
+    if (csv or tsv) and flatten:
+        raise click.ClickException("--flatten cannot be used with --csv or --tsv")
     if encoding and not (csv or tsv):
         raise click.ClickException("--encoding must be used with --csv or --tsv")
-    encoding = encoding or "utf-8"
-    buffered = io.BufferedReader(json_file, buffer_size=4096)
-    decoded = io.TextIOWrapper(buffered, encoding=encoding)
     if pk and len(pk) == 1:
         pk = pk[0]
+    encoding = encoding or "utf-8-sig"
+    buffered = io.BufferedReader(json_file, buffer_size=4096)
+    decoded = io.TextIOWrapper(buffered, encoding=encoding)
+    tracker = None
     if csv or tsv:
         if sniff:
             # Read first 2048 bytes and use that to detect
@@ -710,6 +755,9 @@ def insert_upsert_implementation(
             else:
                 headers = first_row
             docs = (dict(zip(headers, row)) for row in reader)
+            if detect_types:
+                tracker = TypeTracker()
+                docs = tracker.wrap(docs)
     else:
         try:
             if nl:
@@ -722,6 +770,8 @@ def insert_upsert_implementation(
             raise click.ClickException(
                 "Invalid JSON - use --csv for CSV or --tsv for TSV files"
             )
+        if flatten:
+            docs = (dict(_flatten(doc)) for doc in docs)
 
     extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate}
     if not_null:
@@ -732,9 +782,52 @@ def insert_upsert_implementation(
         extra_kwargs["upsert"] = upsert
     # Apply {"$base64": true, ...} decoding, if needed
     docs = (decode_base64_values(doc) for doc in docs)
-    db[table].insert_all(
+    try:
-        docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs
+        db[table].insert_all(
-    )
+            docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs
+        )
+    except Exception as e:
+        if (
+            isinstance(e, sqlite3.OperationalError)
+            and e.args
+            and "has no column named" in e.args[0]
+        ):
+            raise click.ClickException(
+                "{}\n\nTry using --alter to add additional columns".format(e.args[0])
+            )
+        # If we can find sql= and parameters= arguments, show those
+        variables = _find_variables(e.__traceback__, ["sql", "parameters"])
+        if "sql" in variables and "parameters" in variables:
+            raise click.ClickException(
+                "{}\n\nsql = {}\nparameters = {}".format(
+                    str(e), variables["sql"], variables["parameters"]
+                )
+            )
+        else:
+            raise
+    if tracker is not None:
+        db[table].transform(types=tracker.types)
+
+
+def _flatten(d):
+    for key, value in d.items():
+        if isinstance(value, dict):
+            for key2, value2 in _flatten(value):
+                yield key + "_" + key2, value2
+        else:
+            yield key, value
+
+
+def _find_variables(tb, vars):
+    to_find = list(vars)
+    found = {}
+    for var in to_find:
+        if var in tb.tb_frame.f_locals:
+            vars.remove(var)
+            found[var] = tb.tb_frame.f_locals[var]
+    if vars and tb.tb_next:
+        found.update(_find_variables(tb.tb_next, vars))
+    return found
 
 
 @cli.command()
@@ -760,6 +853,7 @@ def insert(
     json_file,
     pk,
     nl,
+    flatten,
     csv,
     tsv,
     delimiter,
@@ -769,6 +863,7 @@ def insert(
     batch_size,
     alter,
     encoding,
+    detect_types,
     load_extension,
     silent,
     ignore,
@@ -790,6 +885,7 @@ def insert(
             json_file,
             pk,
             nl,
+            flatten,
             csv,
             tsv,
             delimiter,
@@ -803,6 +899,7 @@ def insert(
             replace=replace,
             truncate=truncate,
             encoding=encoding,
+            detect_types=detect_types,
             load_extension=load_extension,
             silent=silent,
             not_null=not_null,
@@ -820,6 +917,7 @@ def upsert(
     json_file,
     pk,
     nl,
+    flatten,
     csv,
     tsv,
     batch_size,
@@ -831,6 +929,7 @@ def upsert(
     not_null,
     default,
     encoding,
+    detect_types,
     load_extension,
     silent,
 ):
@@ -846,6 +945,7 @@ def upsert(
             json_file,
             pk,
             nl,
+            flatten,
             csv,
             tsv,
             delimiter,
@@ -905,7 +1005,17 @@ def upsert(
 def create_table(
     path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension
 ):
-    "Add an index to the specified table covering the specified columns"
+    """
+    Add a table with the specified columns. Columns should be specified using
+    name, type pairs, for example:
+
+    \b
+    sqlite-utils create-table my.db people \\
+        id integer \\
+        name text \\
+        height float \\
+        photo blob --pk id
+    """
     db = sqlite_utils.Database(path)
     _load_extensions(db, load_extension)
     if len(columns) % 2 == 1:
@@ -1060,8 +1170,168 @@ def query(
         db.attach(alias, attach_path)
     _load_extensions(db, load_extension)
     db.register_fts4_bm25()
+
+    _execute_query(
+        db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols
+    )
+
+
+@cli.command()
+@click.argument(
+    "paths",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+    required=False,
+    nargs=-1,
+)
+@click.argument("sql")
+@click.option(
+    "--attach",
+    type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)),
+    multiple=True,
+    help="Additional databases to attach - specify alias and filepath",
+)
+@output_options
+@click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row")
+@click.option(
+    "-p",
+    "--param",
+    multiple=True,
+    type=(str, str),
+    help="Named :parameters for SQL query",
+)
+@click.option(
+    "--encoding",
+    help="Character encoding for CSV input, defaults to utf-8",
+)
+@click.option(
+    "-n",
+    "--no-detect-types",
+    is_flag=True,
+    help="Treat all CSV/TSV columns as TEXT",
+)
+@click.option("--schema", is_flag=True, help="Show SQL schema for in-memory database")
+@click.option("--dump", is_flag=True, help="Dump SQL for in-memory database")
+@click.option(
+    "--save",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    help="Save in-memory database to this file",
+)
+@load_extension_option
+def memory(
+    paths,
+    sql,
+    attach,
+    nl,
+    arrays,
+    csv,
+    tsv,
+    no_headers,
+    table,
+    fmt,
+    json_cols,
+    raw,
+    param,
+    encoding,
+    no_detect_types,
+    schema,
+    dump,
+    save,
+    load_extension,
+):
+    """Execute SQL query against an in-memory database, optionally populated by imported data
+
+    To import data from CSV, TSV or JSON files pass them on the command-line:
+
+    \b
+        sqlite-utils memory one.csv two.json \\
+            "select * from one join two on one.two_id = two.id"
+
+    For data piped into the tool from standard input, use "-" or "stdin":
+
+    \b
+        cat animals.csv | sqlite-utils memory - \\
+            "select * from stdin where species = 'dog'"
+
+    The format of the data will be automatically detected. You can specify the format
+    explicitly using :json, :csv, :tsv or :nl (for newline-delimited JSON) - for example:
+
+    \b
+        cat animals.csv | sqlite-utils memory stdin:csv places.dat:nl \\
+            "select * from stdin where place_id in (select id from places)"
+
+    Use --schema to view the SQL schema of any imported files:
+
+    \b
+        sqlite-utils memory animals.csv --schema
+    """
+    db = sqlite_utils.Database(memory=True)
+    # If --dump or --save used but no paths detected, assume SQL query is a path:
+    if (dump or save or schema) and not paths:
+        paths = [sql]
+        sql = None
+    for i, path in enumerate(paths):
+        # Path may have a :format suffix
+        if ":" in path and path.rsplit(":", 1)[-1].upper() in Format.__members__:
+            path, suffix = path.rsplit(":", 1)
+            format = Format[suffix.upper()]
+        else:
+            format = None
+        if path in ("-", "stdin"):
+            csv_fp = sys.stdin.buffer
+            csv_table = "stdin"
+        else:
+            csv_path = pathlib.Path(path)
+            csv_table = csv_path.stem
+            csv_fp = csv_path.open("rb")
+        rows, format_used = rows_from_file(csv_fp, format=format, encoding=encoding)
+        tracker = None
+        if format_used in (Format.CSV, Format.TSV) and not no_detect_types:
+            tracker = TypeTracker()
+            rows = tracker.wrap(rows)
+        db[csv_table].insert_all(rows, alter=True)
+        if tracker is not None:
+            db[csv_table].transform(types=tracker.types)
+        # Add convenient t / t1 / t2 views
+        view_names = ["t{}".format(i + 1)]
+        if i == 0:
+            view_names.append("t")
+        for view_name in view_names:
+            if not db[view_name].exists():
+                db.create_view(view_name, "select * from [{}]".format(csv_table))
+
+    if dump:
+        for line in db.conn.iterdump():
+            click.echo(line)
+        return
+
+    if schema:
+        click.echo(db.schema)
+        return
+
+    if save:
+        db2 = sqlite_utils.Database(save)
+        for line in db.conn.iterdump():
+            db2.execute(line)
+        return
+
+    for alias, attach_path in attach:
+        db.attach(alias, attach_path)
+    _load_extensions(db, load_extension)
+    db.register_fts4_bm25()
+
+    _execute_query(
+        db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols
+    )
+
+
+def _execute_query(
+    db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols
+):
     with db.conn:
-        cursor = db.execute(sql, dict(param))
+        try:
+            cursor = db.execute(sql, dict(param))
+        except sqlite3.OperationalError as e:
+            raise click.ClickException(str(e))
         if cursor.description is None:
             # This was an update/insert
             headers = ["rows_affected"]
@@ -1260,6 +1530,90 @@ def triggers(
     )
 
 
+@cli.command()
+@click.argument(
+    "path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("tables", nargs=-1)
+@click.option("--aux", is_flag=True, help="Include auxiliary columns")
+@output_options
+@load_extension_option
+@click.pass_context
+def indexes(
+    ctx,
+    path,
+    tables,
+    aux,
+    nl,
+    arrays,
+    csv,
+    tsv,
+    no_headers,
+    table,
+    fmt,
+    json_cols,
+    load_extension,
+):
+    "Show indexes for this database"
+    sql = """
+    select
+      sqlite_master.name as "table",
+      indexes.name as index_name,
+      xinfo.*
+    from sqlite_master
+      join pragma_index_list(sqlite_master.name) indexes
+      join pragma_index_xinfo(index_name) xinfo
+    where
+      sqlite_master.type = 'table'
+    """
+    if tables:
+        quote = sqlite_utils.Database(memory=True).quote
+        sql += " and sqlite_master.name in ({})".format(
+            ", ".join(quote(table) for table in tables)
+        )
+    if not aux:
+        sql += " and xinfo.key = 1"
+    ctx.invoke(
+        query,
+        path=path,
+        sql=sql,
+        nl=nl,
+        arrays=arrays,
+        csv=csv,
+        tsv=tsv,
+        no_headers=no_headers,
+        table=table,
+        fmt=fmt,
+        json_cols=json_cols,
+        load_extension=load_extension,
+    )
+
+
+@cli.command()
+@click.argument(
+    "path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("tables", nargs=-1, required=False)
+@load_extension_option
+def schema(
+    path,
+    tables,
+    load_extension,
+):
+    "Show full schema for this database or for specified tables"
+    db = sqlite_utils.Database(path)
+    _load_extensions(db, load_extension)
+    if tables:
+        for table in tables:
+            click.echo(db[table].schema)
+    else:
+        click.echo(db.schema)
+
+
 @cli.command()
 @click.argument(
     "path",
@@ -1269,9 +1623,12 @@ def triggers(
 @click.argument("table")
 @click.option(
     "--type",
-    type=(str, str),
+    type=(
+        str,
+        click.Choice(["INTEGER", "TEXT", "FLOAT", "BLOB"], case_sensitive=False),
+    ),
     multiple=True,
-    help="Change column type to X",
+    help="Change column type to INTEGER, TEXT, FLOAT or BLOB",
 )
 @click.option("--drop", type=str, multiple=True, help="Drop this column")
 @click.option(
@@ -1432,9 +1789,20 @@ def extract(
 @click.option("--replace", is_flag=True, help="Replace files with matching primary key")
 @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key")
 @click.option("--name", type=str, help="File name to use")
+@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar")
 @load_extension_option
 def insert_files(
-    path, table, file_or_dir, column, pk, alter, replace, upsert, name, load_extension
+    path,
+    table,
+    file_or_dir,
+    column,
+    pk,
+    alter,
+    replace,
+    upsert,
+    name,
+    silent,
+    load_extension,
 ):
     """
     Insert one or more files using BLOB columns in the specified table
@@ -1471,7 +1839,7 @@ def insert_files(
     # Load all paths so we can show a progress bar
     paths_and_relative_paths = list(yield_paths_and_relative_paths())
 
-    with click.progressbar(paths_and_relative_paths) as bar:
+    with progressbar(paths_and_relative_paths, silent=silent) as bar:
 
         def to_insert():
             for path, relative_path in bar:
@@ -1593,6 +1961,159 @@ def analyze_tables(
         click.echo(details)
 
 
+def _generate_convert_help():
+    help = textwrap.dedent(
+        """
+    Convert columns using Python code you supply. For example:
+
+    \b
+    $ sqlite-utils convert my.db mytable mycolumn \\
+        '"\\n".join(textwrap.wrap(value, 10))' \\
+        --import=textwrap
+
+    "value" is a variable with the column value to be converted.
+
+    The following common operations are available as recipe functions:
+    """
+    ).strip()
+    recipe_names = [
+        n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")
+    ]
+    for name in recipe_names:
+        fn = getattr(recipes, name)
+        help += "\n\nr.{}{}\n\n  {}".format(
+            name, str(inspect.signature(fn)), fn.__doc__
+        )
+    help += "\n\n"
+    help += textwrap.dedent(
+        """
+    You can use these recipes like so:
+
+    \b
+    $ sqlite-utils convert my.db mytable mycolumn \\
+        'r.jsonsplit(value, delimiter=":")'
+    """
+    ).strip()
+    return help
+
+
+@cli.command(help=_generate_convert_help())
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("table", type=str)
+@click.argument("columns", type=str, nargs=-1, required=True)
+@click.argument("code", type=str)
+@click.option(
+    "--import", "imports", type=str, multiple=True, help="Python modules to import"
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Show results of running this against first 10 rows"
+)
+@click.option(
+    "--multi", is_flag=True, help="Populate columns for keys in returned dictionary"
+)
+@click.option("--where", help="Optional where clause")
+@click.option(
+    "-p",
+    "--param",
+    multiple=True,
+    type=(str, str),
+    help="Named :parameters for where clause",
+)
+@click.option("--output", help="Optional separate column to populate with the output")
+@click.option(
+    "--output-type",
+    help="Column type to use for the output column",
+    default="text",
+    type=click.Choice(["integer", "float", "blob", "text"]),
+)
+@click.option("--drop", is_flag=True, help="Drop original column afterwards")
+@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar")
+def convert(
+    db_path,
+    table,
+    columns,
+    code,
+    imports,
+    dry_run,
+    multi,
+    where,
+    param,
+    output,
+    output_type,
+    drop,
+    silent,
+):
+    sqlite3.enable_callback_tracebacks(True)
+    db = sqlite_utils.Database(db_path)
+    if output is not None and len(columns) > 1:
+        raise click.ClickException("Cannot use --output with more than one column")
+    if multi and len(columns) > 1:
+        raise click.ClickException("Cannot use --multi with more than one column")
+    if drop and not (output or multi):
+        raise click.ClickException("--drop can only be used with --output or --multi")
+    # If single line and no 'return', add the return
+    if "\n" not in code and not code.strip().startswith("return "):
+        code = "return {}".format(code)
+    where_args = dict(param) if param else []
+    # Compile the code into a function body called fn(value)
+    new_code = ["def fn(value):"]
+    for line in code.split("\n"):
+        new_code.append("    {}".format(line))
+    code_o = compile("\n".join(new_code), "<string>", "exec")
+    locals = {}
+    globals = {"r": recipes, "recipes": recipes}
+    for import_ in imports:
+        globals[import_] = __import__(import_)
+    exec(code_o, globals, locals)
+    fn = locals["fn"]
+    if dry_run:
+        # Pull first 20 values for first column and preview them
+        db.conn.create_function("preview_transform", 1, lambda v: fn(v) if v else v)
+        sql = """
+            select
+                [{column}] as value,
+                preview_transform([{column}]) as preview
+            from [{table}]{where} limit 10
+        """.format(
+            column=columns[0],
+            table=table,
+            where=" where {}".format(where) if where is not None else "",
+        )
+        for row in db.conn.execute(sql, where_args).fetchall():
+            click.echo(str(row[0]))
+            click.echo(" --- becomes:")
+            click.echo(str(row[1]))
+            click.echo()
+        count = db[table].count_where(
+            where=where,
+            where_args=where_args,
+        )
+        click.echo("Would affect {} row{}".format(count, "" if count == 1 else "s"))
+    else:
+        try:
+            db[table].convert(
+                columns,
+                fn,
+                where=where,
+                where_args=where_args,
+                output=output,
+                output_type=output_type,
+                drop=drop,
+                multi=multi,
+                show_progress=not silent,
+            )
+        except BadMultiValues as e:
+            raise click.ClickException(
+                "When using --multi code must return a Python dictionary - returned: {}".format(
+                    repr(e.values)
+                )
+            )
+
+
 def _render_common(title, values):
     if values is None:
         return ""

sqlite_utils/recipes.py

@@ -0,0 +1,19 @@
+from dateutil import parser
+import json
+
+
+def parsedate(value, dayfirst=False, yearfirst=False):
+    "Parse a date and convert it to ISO date format: yyyy-mm-dd"
+    return (
+        parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat()
+    )
+
+
+def parsedatetime(value, dayfirst=False, yearfirst=False):
+    "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS"
+    return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat()
+
+
+def jsonsplit(value, delimiter=",", type=str):
+    'Convert a string like a,b,c into a JSON array ["a", "b", "c"]'
+    return json.dumps([type(s.strip()) for s in value.split(delimiter)])

sqlite_utils/utils.py

@@ -1,16 +1,22 @@
 import base64
-import click
 import contextlib
+import csv
+import enum
 import io
+import json
 import os
+from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type
+
+import click
 
 try:
-    import pysqlite3 as sqlite3
+    import pysqlite3 as sqlite3  # type: ignore
-    import pysqlite3.dbapi2
+    import pysqlite3.dbapi2  # type: ignore
 
     OperationalError = pysqlite3.dbapi2.OperationalError
 except ImportError:
-    import sqlite3
+    # https://github.com/python/mypy/issues/1153#issuecomment-253842414
+    import sqlite3  # type: ignore
 
     OperationalError = sqlite3.OperationalError
 
@@ -25,8 +31,11 @@ def suggest_column_types(records):
     for record in records:
         for key, value in record.items():
             all_column_types.setdefault(key, set()).add(type(value))
-    column_types = {}
+    return types_for_column_types(all_column_types)
 
+
+def types_for_column_types(all_column_types):
+    column_types = {}
     for key, types in all_column_types.items():
         # Ignore null values if at least one other type present:
         if len(types) > 1:
@@ -105,9 +114,167 @@ class UpdateWrapper:
 
 @contextlib.contextmanager
 def file_progress(file, silent=False, **kwargs):
-    if silent or file.fileno() == 0:  # 0 = stdin
+    if silent:
+        yield file
+        return
+    # file.fileno() throws an exception in our test suite
+    try:
+        fileno = file.fileno()
+    except io.UnsupportedOperation:
+        yield file
+        return
+    if fileno == 0:  # 0 means stdin
         yield file
     else:
         file_length = os.path.getsize(file.name)
         with click.progressbar(length=file_length, **kwargs) as bar:
             yield UpdateWrapper(file, bar.update)
+
+
+class Format(enum.Enum):
+    CSV = 1
+    TSV = 2
+    JSON = 3
+    NL = 4
+
+
+class RowsFromFileError(Exception):
+    pass
+
+
+class RowsFromFileBadJSON(RowsFromFileError):
+    pass
+
+
+def rows_from_file(
+    fp: BinaryIO,
+    format: Optional[Format] = None,
+    dialect: Optional[Type[csv.Dialect]] = None,
+    encoding: Optional[str] = None,
+) -> Tuple[Iterable[dict], Format]:
+    if format == Format.JSON:
+        decoded = json.load(fp)
+        if isinstance(decoded, dict):
+            decoded = [decoded]
+        if not isinstance(decoded, list):
+            raise RowsFromFileBadJSON("JSON must be a list or a dictionary")
+        return decoded, Format.JSON
+    elif format == Format.NL:
+        return (json.loads(line) for line in fp if line.strip()), Format.NL
+    elif format == Format.CSV:
+        use_encoding: str = encoding or "utf-8-sig"
+        decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding)
+        if dialect is not None:
+            reader = csv.DictReader(decoded_fp, dialect=dialect)
+        else:
+            reader = csv.DictReader(decoded_fp)
+        return reader, Format.CSV
+    elif format == Format.TSV:
+        return (
+            rows_from_file(
+                fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding
+            )[0],
+            Format.TSV,
+        )
+    elif format is None:
+        # Detect the format, then call this recursively
+        buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096)
+        first_bytes = buffered.peek(2048).strip()
+        if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"):
+            # TODO: Detect newline-JSON
+            return rows_from_file(buffered, format=Format.JSON)
+        else:
+            dialect = csv.Sniffer().sniff(
+                first_bytes.decode(encoding or "utf-8-sig", "ignore")
+            )
+            return rows_from_file(
+                buffered, format=Format.CSV, dialect=dialect, encoding=encoding
+            )
+    else:
+        raise RowsFromFileError("Bad format")
+
+
+class TypeTracker:
+    def __init__(self):
+        self.trackers = {}
+
+    def wrap(self, iterator):
+        for row in iterator:
+            for key, value in row.items():
+                tracker = self.trackers.setdefault(key, ValueTracker())
+                tracker.evaluate(value)
+            yield row
+
+    @property
+    def types(self):
+        return {key: tracker.guessed_type for key, tracker in self.trackers.items()}
+
+
+class ValueTracker:
+    def __init__(self):
+        self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()}
+
+    @classmethod
+    def get_tests(cls):
+        return [
+            key.split("test_")[-1]
+            for key in cls.__dict__.keys()
+            if key.startswith("test_")
+        ]
+
+    def test_integer(self, value):
+        try:
+            int(value)
+            return True
+        except (ValueError, TypeError):
+            return False
+
+    def test_float(self, value):
+        try:
+            float(value)
+            return True
+        except (ValueError, TypeError):
+            return False
+
+    def __repr__(self):
+        return self.guessed_type + ": possibilities = " + repr(self.couldbe)
+
+    @property
+    def guessed_type(self):
+        options = set(self.couldbe.keys())
+        # Return based on precedence
+        for key in self.get_tests():
+            if key in options:
+                return key
+        return "text"
+
+    def evaluate(self, value):
+        if not value or not self.couldbe:
+            return
+        not_these = []
+        for name, test in self.couldbe.items():
+            if not test(value):
+                not_these.append(name)
+        for key in not_these:
+            del self.couldbe[key]
+
+
+class NullProgressBar:
+    def __init__(self, *args):
+        self.args = args
+
+    def __iter__(self):
+        yield from self.args[0]
+
+    def update(self, value):
+        pass
+
+
+@contextlib.contextmanager
+def progressbar(*args, **kwargs):
+    silent = kwargs.pop("silent")
+    if silent:
+        yield NullProgressBar(*args)
+    else:
+        with click.progressbar(*args, **kwargs) as bar:
+            yield bar

tests/test_analyze_tables.py

@@ -1,10 +1,8 @@
-from sqlite_utils.db import Database, ForeignKey, ColumnDetails
+from sqlite_utils.db import Database, ColumnDetails
 from sqlite_utils import cli
-from sqlite_utils.utils import OperationalError
 from click.testing import CliRunner
 import pytest
 import sqlite3
-import textwrap
 
 
 @pytest.fixture
@@ -132,6 +130,7 @@ def test_analyze_table_save(db_to_analyze_path):
     result = CliRunner().invoke(
         cli.cli, ["analyze-tables", db_to_analyze_path, "--save"]
     )
+    assert result.exit_code == 0
     rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows)
     assert rows == [
         {

tests/test_cli.py

@@ -1,6 +1,7 @@
 from sqlite_utils import cli, Database
 from sqlite_utils.db import Index, ForeignKey
 from click.testing import CliRunner
+from unittest import mock
 import json
 import os
 import pytest
@@ -24,6 +25,22 @@ def db_path(tmpdir):
     return path
 
 
+@pytest.mark.parametrize(
+    "options",
+    (
+        ["-h"],
+        ["--help"],
+        ["insert", "-h"],
+        ["insert", "--help"],
+    ),
+)
+def test_help(options):
+    result = CliRunner().invoke(cli.cli, options)
+    assert result.exit_code == 0
+    assert result.output.startswith("Usage: ")
+    assert "-h, --help" in result.output
+
+
 def test_tables(db_path):
     result = CliRunner().invoke(cli.cli, ["tables", db_path])
     assert '[{"table": "Gosh"},\n {"table": "Gosh2"}]' == result.output.strip()
@@ -208,6 +225,17 @@ def test_create_index(db_path):
     )
 
 
+def test_create_index_desc(db_path):
+    db = Database(db_path)
+    assert [] == db["Gosh"].indexes
+    result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "--", "-c1"])
+    assert result.exit_code == 0
+    assert (
+        db.execute("select sql from sqlite_master where type='index'").fetchone()[0]
+        == "CREATE INDEX [idx_Gosh_c1]\n    ON [Gosh] ([c1] desc)"
+    )
+
+
 @pytest.mark.parametrize(
     "col_name,col_type,expected_schema",
     (
@@ -354,6 +382,21 @@ def test_add_column_foreign_key(db_path):
     assert "table 'bobcats' does not exist" in str(result.exception)
 
 
+def test_suggest_alter_if_column_missing(db_path):
+    db = Database(db_path)
+    db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "authors", "-"],
+        input='{"id": 2, "name": "Barry", "age": 43}',
+    )
+    assert result.exit_code != 0
+    assert result.output.strip() == (
+        "Error: table authors has no column named age\n\n"
+        "Try using --alter to add additional columns"
+    )
+
+
 def test_index_foreign_keys(db_path):
     test_add_column_foreign_key(db_path)
     db = Database(db_path)
@@ -367,7 +410,7 @@ def test_index_foreign_keys(db_path):
 
 def test_enable_fts(db_path):
     db = Database(db_path)
-    assert None == db["Gosh"].detect_fts()
+    assert db["Gosh"].detect_fts() is None
     result = CliRunner().invoke(
         cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"]
     )
@@ -377,7 +420,7 @@ def test_enable_fts(db_path):
     # Table names with restricted chars are handled correctly.
     # colons and dots are restricted characters for table names.
     db["http://example.com"].create({"c1": str, "c2": str, "c3": str})
-    assert None == db["http://example.com"].detect_fts()
+    assert db["http://example.com"].detect_fts() is None
     result = CliRunner().invoke(
         cli.cli,
         [
@@ -476,6 +519,13 @@ def test_vacuum(db_path):
     assert 0 == result.exit_code
 
 
+def test_dump(db_path):
+    result = CliRunner().invoke(cli.cli, ["dump", db_path])
+    assert result.exit_code == 0
+    assert result.output.startswith("BEGIN TRANSACTION;")
+    assert result.output.strip().endswith("COMMIT;")
+
+
 @pytest.mark.parametrize("tables", ([], ["Gosh"], ["Gosh2"]))
 def test_optimize(db_path, tables):
     db = Database(db_path)
@@ -563,8 +613,8 @@ def test_insert_simple(tmpdir):
     open(json_path, "w").write(json.dumps({"name": "Cleo", "age": 4}))
     result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path])
     assert 0 == result.exit_code
-    assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts(
+    assert [{"age": 4, "name": "Cleo"}] == list(
-        "select * from dogs"
+        Database(db_path).query("select * from dogs")
     )
     db = Database(db_path)
     assert ["dogs"] == db.table_names()
@@ -579,8 +629,8 @@ def test_insert_from_stdin(tmpdir):
         input=json.dumps({"name": "Cleo", "age": 4}),
     )
     assert 0 == result.exit_code
-    assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts(
+    assert [{"age": 4, "name": "Cleo"}] == list(
-        "select * from dogs"
+        Database(db_path).query("select * from dogs")
     )
 
 
@@ -598,6 +648,34 @@ def test_insert_invalid_json_error(tmpdir):
     )
 
 
+def test_insert_json_flatten(tmpdir):
+    db_path = str(tmpdir / "flat.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "items", "-", "--flatten"],
+        input=json.dumps({"nested": {"data": 4}}),
+    )
+    assert result.exit_code == 0
+    assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}]
+
+
+def test_insert_json_flatten_nl(tmpdir):
+    db_path = str(tmpdir / "flat.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "items", "-", "--flatten", "--nl"],
+        input="\n".join(
+            json.dumps(item)
+            for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}]
+        ),
+    )
+    assert result.exit_code == 0
+    assert list(Database(db_path).query("select * from items")) == [
+        {"nested_data": 4, "nested_other": None},
+        {"nested_data": None, "nested_other": 3},
+    ]
+
+
 def test_insert_with_primary_key(db_path, tmpdir):
     json_path = str(tmpdir / "dog.json")
     open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4}))
@@ -605,9 +683,9 @@ def test_insert_with_primary_key(db_path, tmpdir):
         cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
     )
     assert 0 == result.exit_code
-    assert [{"id": 1, "age": 4, "name": "Cleo"}] == Database(
+    assert [{"id": 1, "age": 4, "name": "Cleo"}] == list(
-        db_path
+        Database(db_path).query("select * from dogs")
-    ).execute_returning_dicts("select * from dogs")
+    )
     db = Database(db_path)
     assert ["id"] == db["dogs"].pks
 
@@ -621,7 +699,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir):
     )
     assert 0 == result.exit_code
     db = Database(db_path)
-    assert dogs == db.execute_returning_dicts("select * from dogs order by id")
+    assert dogs == list(db.query("select * from dogs order by id"))
     assert ["id"] == db["dogs"].pks
 
 
@@ -637,7 +715,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir):
     )
     assert 0 == result.exit_code
     db = Database(db_path)
-    assert dogs == db.execute_returning_dicts("select * from dogs order by breed, id")
+    assert dogs == list(db.query("select * from dogs order by breed, id"))
     assert {"breed", "id"} == set(db["dogs"].pks)
     assert (
         "CREATE TABLE [dogs] (\n"
@@ -682,7 +760,7 @@ def test_insert_binary_base64(db_path):
     )
     assert 0 == result.exit_code, result.output
     db = Database(db_path)
-    actual = db.execute_returning_dicts("select content from files")
+    actual = list(db.query("select content from files"))
     assert actual == [{"content": b"hello"}]
 
 
@@ -697,7 +775,7 @@ def test_insert_newline_delimited(db_path):
     assert [
         {"foo": "bar", "n": 1},
         {"foo": "baz", "n": 2},
-    ] == db.execute_returning_dicts("select foo, n from from_json_nl")
+    ] == list(db.query("select foo, n from from_json_nl"))
 
 
 def test_insert_ignore(db_path, tmpdir):
@@ -716,9 +794,7 @@ def test_insert_ignore(db_path, tmpdir):
     )
     assert 0 == result.exit_code, result.output
     # ... but it should actually have no effect
-    assert [{"id": 1, "name": "Cleo"}] == db.execute_returning_dicts(
+    assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs"))
-        "select * from dogs"
-    )
 
 
 @pytest.mark.parametrize(
@@ -781,8 +857,9 @@ def test_insert_replace(db_path, tmpdir):
     )
     assert 0 == result.exit_code, result.output
     assert 21 == db["dogs"].count
-    assert insert_replace_dogs == db.execute_returning_dicts(
+    assert (
-        "select * from dogs where id in (1, 2, 21) order by id"
+        list(db.query("select * from dogs where id in (1, 2, 21) order by id"))
+        == insert_replace_dogs
     )
 
 
@@ -797,7 +874,7 @@ def test_insert_truncate(db_path):
     assert [
         {"foo": "bar", "n": 1},
         {"foo": "baz", "n": 2},
-    ] == db.execute_returning_dicts("select foo, n from from_json_nl")
+    ] == list(db.query("select foo, n from from_json_nl"))
     # Truncate and insert new rows
     result = CliRunner().invoke(
         cli.cli,
@@ -816,7 +893,7 @@ def test_insert_truncate(db_path):
     assert [
         {"foo": "bam", "n": 3},
         {"foo": "bat", "n": 4},
-    ] == db.execute_returning_dicts("select foo, n from from_json_nl")
+    ] == list(db.query("select foo, n from from_json_nl"))
 
 
 def test_insert_alter(db_path, tmpdir):
@@ -847,7 +924,7 @@ def test_insert_alter(db_path, tmpdir):
         {"foo": "bar", "n": 1, "baz": None},
         {"foo": "baz", "n": 2, "baz": None},
         {"foo": "bar", "baz": 5, "n": None},
-    ] == db.execute_returning_dicts("select foo, n, baz from from_json_nl")
+    ] == list(db.query("select foo, n, baz from from_json_nl"))
 
 
 @pytest.mark.parametrize(
@@ -917,7 +994,23 @@ def test_query_json(db_path, sql, args, expected):
     assert expected == result.output.strip()
 
 
-LOREM_IPSUM_COMPRESSED = b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8ef\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3\x85>\x8c\xa4i\x8d\xdaTu\x7f<c\xc9\xf5L\x0f\xd7E\xad/\x9b\x9eI^2\x93\x1a\x9b\xf6F^\n\xd7\xd4\x8f\xca\xfb\x90.\xdd/\xfd\x94\xd4\x11\x87I8\x1a\xaf\xd1S?\x06\x88\xa7\xecBo\xbb$\xbb\t\xe9\xf4\xe8\xe4\x98U\x1bM\x19S\xbe\xa4e\x991x\xfcx\xf6\xe2#\x9e\x93h'&%YK(i)\x7f\t\xc5@N7\xbf+\x1b\xb5\xdd\x10\r\x9e\xb1\xf0y\xa1\xf7W\x92a\xe2;\xc6\xc8\xa0\xa7\xc4\x92\xe2\\\xf2\xa1\x99m\xdf\x88)\xc6\xec\x9a\xa5\xed\x14wR\xf1h\xf22x\xcfM\xfdv\xd3\xa4LY\x96\xcc\xbd[{\xd9m\xf0\x0eH#\x8e\xf5\x9b\xab\xd7\xcb\xe9t\x05\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}"
+LOREM_IPSUM_COMPRESSED = (
+    b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8e"
+    b"f\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J"
+    b"\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b"
+    b"$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3"
+    b"\x85>\x8c\xa4i\x8d\xdaTu\x7f<c\xc9\xf5L\x0f\xd7E\xad/\x9b\x9eI^2\x93\x1a\x9b"
+    b"\xf6F^\n\xd7\xd4\x8f\xca\xfb\x90.\xdd/\xfd\x94\xd4\x11\x87I8\x1a\xaf\xd1S?\x06"
+    b"\x88\xa7\xecBo\xbb$\xbb\t\xe9\xf4\xe8\xe4\x98U\x1bM\x19S\xbe\xa4e\x991x\xfc"
+    b"x\xf6\xe2#\x9e\x93h'&%YK(i)\x7f\t\xc5@N7\xbf+\x1b\xb5\xdd\x10\r\x9e\xb1\xf0"
+    b"y\xa1\xf7W\x92a\xe2;\xc6\xc8\xa0\xa7\xc4\x92\xe2\\\xf2\xa1\x99m\xdf\x88)\xc6"
+    b"\xec\x9a\xa5\xed\x14wR\xf1h\xf22x\xcfM\xfdv\xd3\xa4LY\x96\xcc\xbd[{\xd9m\xf0"
+    b"\x0eH#\x8e\xf5\x9b\xab\xd7\xcb\xe9t\x05\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03"
+    b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03"
+    b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03"
+    b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03"
+    b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}"
+)
 
 
 def test_query_json_binary(db_path):
@@ -939,7 +1032,18 @@ def test_query_json_binary(db_path):
             "sz": 16984,
             "data": {
                 "$base64": True,
-                "encoded": "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uIjnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3fiCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9",
+                "encoded": (
+                    (
+                        "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH"
+                        "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+"
+                        "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I"
+                        "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI"
+                        "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f"
+                        "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8"
+                        "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A"
+                        "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9"
+                    )
+                ),
             },
         }
     ]
@@ -1023,7 +1127,7 @@ def test_query_load_extension(use_spatialite_shortcut):
     # Without --load-extension:
     result = CliRunner().invoke(cli.cli, [":memory:", "select spatialite_version()"])
     assert result.exit_code == 1
-    assert "no such function: spatialite_version" in repr(result)
+    assert "no such function: spatialite_version" in result.output
     # With --load-extension:
     if use_spatialite_shortcut:
         load_extension = "spatialite"
@@ -1108,17 +1212,32 @@ def test_upsert(db_path, tmpdir):
         {"id": 1, "age": 5},
         {"id": 2, "age": 5},
     ]
-    open(json_path, "w").write(json.dumps(insert_dogs))
+    open(json_path, "w").write(json.dumps(upsert_dogs))
     result = CliRunner().invoke(
         cli.cli,
         ["upsert", db_path, "dogs", json_path, "--pk", "id"],
         catch_exceptions=False,
     )
     assert 0 == result.exit_code, result.output
-    assert [
+    assert list(db.query("select * from dogs order by id")) == [
-        {"id": 1, "name": "Cleo", "age": 4},
+        {"id": 1, "name": "Cleo", "age": 5},
-        {"id": 2, "name": "Nixie", "age": 4},
+        {"id": 2, "name": "Nixie", "age": 5},
-    ] == db.execute_returning_dicts("select * from dogs order by id")
+    ]
+
+
+def test_upsert_flatten(tmpdir):
+    db_path = str(tmpdir / "flat.db")
+    db = Database(db_path)
+    db["upsert_me"].insert({"id": 1, "name": "Example"}, pk="id")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["upsert", db_path, "upsert_me", "-", "--flatten", "--pk", "id", "--alter"],
+        input=json.dumps({"id": 1, "nested": {"two": 2}}),
+    )
+    assert result.exit_code == 0
+    assert list(db.query("select * from upsert_me")) == [
+        {"id": 1, "name": "Example", "nested_two": 2}
+    ]
 
 
 def test_upsert_alter(db_path, tmpdir):
@@ -1137,7 +1256,11 @@ def test_upsert_alter(db_path, tmpdir):
         cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"]
     )
     assert 1 == result.exit_code
-    assert "no such column: age" == str(result.exception)
+    assert (
+        "Error: no such column: age\n\n"
+        "sql = UPDATE [dogs] SET [age] = ? WHERE [id] = ?\n"
+        "parameters = [5, 1]"
+    ) == result.output.strip()
     # Should succeed with --alter
     result = CliRunner().invoke(
         cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"]
@@ -1145,7 +1268,7 @@ def test_upsert_alter(db_path, tmpdir):
     assert 0 == result.exit_code
     assert [
         {"id": 1, "name": "Cleo", "age": 5},
-    ] == db.execute_returning_dicts("select * from dogs order by id")
+    ] == list(db.query("select * from dogs order by id"))
 
 
 @pytest.mark.parametrize(
@@ -1499,7 +1622,7 @@ def test_query_update(db_path, args, expected):
         cli.cli, [db_path, "update dogs set age = 5 where name = 'Cleo'"] + args
     )
     assert expected == result.output.strip()
-    assert db.execute_returning_dicts("select * from dogs") == [
+    assert list(db.query("select * from dogs")) == [
         {"id": 1, "age": 5, "name": "Cleo"},
     ]
 
@@ -1547,47 +1670,112 @@ def test_add_foreign_keys(db_path):
     [
         (
             [],
-            "CREATE TABLE \"dogs\" (\n   [id] INTEGER PRIMARY KEY,\n   [age] INTEGER NOT NULL DEFAULT '1',\n   [name] TEXT\n)",
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [age] INTEGER NOT NULL DEFAULT '1',\n"
+                "   [name] TEXT\n"
+                ")"
+            ),
         ),
         (
             ["--type", "age", "text"],
-            "CREATE TABLE \"dogs\" (\n   [id] INTEGER PRIMARY KEY,\n   [age] TEXT NOT NULL DEFAULT '1',\n   [name] TEXT\n)",
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [age] TEXT NOT NULL DEFAULT '1',\n"
+                "   [name] TEXT\n"
+                ")"
+            ),
         ),
         (
             ["--drop", "age"],
-            'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT\n)',
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [name] TEXT\n"
+                ")"
+            ),
         ),
         (
             ["--rename", "age", "age2", "--rename", "id", "pk"],
-            "CREATE TABLE \"dogs\" (\n   [pk] INTEGER PRIMARY KEY,\n   [age2] INTEGER NOT NULL DEFAULT '1',\n   [name] TEXT\n)",
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [pk] INTEGER PRIMARY KEY,\n"
+                "   [age2] INTEGER NOT NULL DEFAULT '1',\n"
+                "   [name] TEXT\n"
+                ")"
+            ),
         ),
         (
             ["--not-null", "name"],
-            "CREATE TABLE \"dogs\" (\n   [id] INTEGER PRIMARY KEY,\n   [age] INTEGER NOT NULL DEFAULT '1',\n   [name] TEXT NOT NULL\n)",
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [age] INTEGER NOT NULL DEFAULT '1',\n"
+                "   [name] TEXT NOT NULL\n"
+                ")"
+            ),
         ),
         (
             ["--not-null-false", "age"],
-            "CREATE TABLE \"dogs\" (\n   [id] INTEGER PRIMARY KEY,\n   [age] INTEGER DEFAULT '1',\n   [name] TEXT\n)",
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [age] INTEGER DEFAULT '1',\n"
+                "   [name] TEXT\n"
+                ")"
+            ),
         ),
         (
             ["--pk", "name"],
-            "CREATE TABLE \"dogs\" (\n   [id] INTEGER,\n   [age] INTEGER NOT NULL DEFAULT '1',\n   [name] TEXT PRIMARY KEY\n)",
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [id] INTEGER,\n"
+                "   [age] INTEGER NOT NULL DEFAULT '1',\n"
+                "   [name] TEXT PRIMARY KEY\n"
+                ")"
+            ),
         ),
         (
             ["--pk-none"],
-            "CREATE TABLE \"dogs\" (\n   [id] INTEGER,\n   [age] INTEGER NOT NULL DEFAULT '1',\n   [name] TEXT\n)",
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [id] INTEGER,\n"
+                "   [age] INTEGER NOT NULL DEFAULT '1',\n"
+                "   [name] TEXT\n"
+                ")"
+            ),
         ),
         (
             ["--default", "name", "Turnip"],
-            "CREATE TABLE \"dogs\" (\n   [id] INTEGER PRIMARY KEY,\n   [age] INTEGER NOT NULL DEFAULT '1',\n   [name] TEXT DEFAULT 'Turnip'\n)",
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [age] INTEGER NOT NULL DEFAULT '1',\n"
+                "   [name] TEXT DEFAULT 'Turnip'\n"
+                ")"
+            ),
         ),
         (
             ["--default-none", "age"],
-            'CREATE TABLE "dogs" (\n   [id] INTEGER PRIMARY KEY,\n   [age] INTEGER NOT NULL,\n   [name] TEXT\n)',
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [age] INTEGER NOT NULL,\n"
+                "   [name] TEXT\n"
+                ")"
+            ),
         ),
         (
             ["-o", "name", "--column-order", "age", "-o", "id"],
-            "CREATE TABLE \"dogs\" (\n   [name] TEXT,\n   [age] INTEGER NOT NULL DEFAULT '1',\n   [id] INTEGER PRIMARY KEY\n)",
+            (
+                'CREATE TABLE "dogs" (\n'
+                "   [name] TEXT,\n"
+                "   [age] INTEGER NOT NULL DEFAULT '1',\n"
+                "   [id] INTEGER PRIMARY KEY\n"
+                ")"
+            ),
         ),
     ],
 )
@@ -1636,9 +1824,13 @@ def test_transform_drop_foreign_key(db_path):
     print(result.output)
     assert result.exit_code == 0
     schema = db["places"].schema
-    assert (
+    assert schema == (
-        schema
+        'CREATE TABLE "places" (\n'
-        == 'CREATE TABLE "places" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [country] INTEGER,\n   [city] INTEGER REFERENCES [city]([id])\n)'
+        "   [id] INTEGER PRIMARY KEY,\n"
+        "   [name] TEXT,\n"
+        "   [country] INTEGER,\n"
+        "   [city] INTEGER REFERENCES [city]([id])\n"
+        ")"
     )
 
 
@@ -1652,22 +1844,48 @@ _common_other_schema = (
     [
         (
             [],
-            'CREATE TABLE "trees" (\n   [id] INTEGER PRIMARY KEY,\n   [address] TEXT,\n   [species_id] INTEGER,\n   FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)',
+            (
+                'CREATE TABLE "trees" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [address] TEXT,\n"
+                "   [species_id] INTEGER,\n"
+                "   FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
+                ")"
+            ),
             _common_other_schema,
         ),
         (
             ["--table", "custom_table"],
-            'CREATE TABLE "trees" (\n   [id] INTEGER PRIMARY KEY,\n   [address] TEXT,\n   [custom_table_id] INTEGER,\n   FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n)',
+            (
+                'CREATE TABLE "trees" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [address] TEXT,\n"
+                "   [custom_table_id] INTEGER,\n"
+                "   FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n"
+                ")"
+            ),
             "CREATE TABLE [custom_table] (\n   [id] INTEGER PRIMARY KEY,\n   [species] TEXT\n)",
         ),
         (
             ["--fk-column", "custom_fk"],
-            'CREATE TABLE "trees" (\n   [id] INTEGER PRIMARY KEY,\n   [address] TEXT,\n   [custom_fk] INTEGER,\n   FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n)',
+            (
+                'CREATE TABLE "trees" (\n'
+                "   [id] INTEGER PRIMARY KEY,\n"
+                "   [address] TEXT,\n"
+                "   [custom_fk] INTEGER,\n"
+                "   FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n"
+                ")"
+            ),
             _common_other_schema,
         ),
         (
             ["--rename", "name", "name2"],
-            'CREATE TABLE "trees" (\n   [id] INTEGER PRIMARY KEY,\n   [address] TEXT,\n   [species_id] INTEGER,\n   FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)',
+            'CREATE TABLE "trees" (\n'
+            "   [id] INTEGER PRIMARY KEY,\n"
+            "   [address] TEXT,\n"
+            "   [species_id] INTEGER,\n"
+            "   FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
+            ")",
             "CREATE TABLE [species] (\n   [id] INTEGER PRIMARY KEY,\n   [species] TEXT\n)",
         ),
     ],
@@ -1776,7 +1994,87 @@ def test_search(tmpdir, fts, extra_arg, expected):
     assert result.output.replace("\r", "") == expected
 
 
-_TRIGGERS_EXPECTED = '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah AFTER INSERT ON articles\\nBEGIN\\n    UPDATE counter SET count = count + 1;\\nEND"}]\n'
+def test_indexes(tmpdir):
+    db_path = str(tmpdir / "test.db")
+    db = Database(db_path)
+    db.conn.executescript(
+        """
+        create table Gosh (c1 text, c2 text, c3 text);
+        create index Gosh_idx on Gosh(c2, c3 desc);
+    """
+    )
+    result = CliRunner().invoke(
+        cli.cli,
+        ["indexes", str(db_path)],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    assert json.loads(result.output) == [
+        {
+            "table": "Gosh",
+            "index_name": "Gosh_idx",
+            "seqno": 0,
+            "cid": 1,
+            "name": "c2",
+            "desc": 0,
+            "coll": "BINARY",
+            "key": 1,
+        },
+        {
+            "table": "Gosh",
+            "index_name": "Gosh_idx",
+            "seqno": 1,
+            "cid": 2,
+            "name": "c3",
+            "desc": 1,
+            "coll": "BINARY",
+            "key": 1,
+        },
+    ]
+    result2 = CliRunner().invoke(
+        cli.cli,
+        ["indexes", str(db_path), "--aux"],
+        catch_exceptions=False,
+    )
+    assert result2.exit_code == 0
+    assert json.loads(result2.output) == [
+        {
+            "table": "Gosh",
+            "index_name": "Gosh_idx",
+            "seqno": 0,
+            "cid": 1,
+            "name": "c2",
+            "desc": 0,
+            "coll": "BINARY",
+            "key": 1,
+        },
+        {
+            "table": "Gosh",
+            "index_name": "Gosh_idx",
+            "seqno": 1,
+            "cid": 2,
+            "name": "c3",
+            "desc": 1,
+            "coll": "BINARY",
+            "key": 1,
+        },
+        {
+            "table": "Gosh",
+            "index_name": "Gosh_idx",
+            "seqno": 2,
+            "cid": -1,
+            "name": None,
+            "desc": 0,
+            "coll": "BINARY",
+            "key": 0,
+        },
+    ]
+
+
+_TRIGGERS_EXPECTED = (
+    '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah '
+    'AFTER INSERT ON articles\\nBEGIN\\n    UPDATE counter SET count = count + 1;\\nEND"}]\n'
+)
 
 
 @pytest.mark.parametrize(
@@ -1813,6 +2111,60 @@ def test_triggers(tmpdir, extra_args, expected):
     assert result.output == expected
 
 
+@pytest.mark.parametrize(
+    "options,expected",
+    (
+        (
+            [],
+            (
+                "CREATE TABLE [dogs] (\n"
+                "   [id] INTEGER,\n"
+                "   [name] TEXT\n"
+                ");\n"
+                "CREATE TABLE [chickens] (\n"
+                "   [id] INTEGER,\n"
+                "   [name] TEXT,\n"
+                "   [breed] TEXT\n"
+                ");\n"
+                "CREATE INDEX [idx_chickens_breed]\n"
+                "    ON [chickens] ([breed]);\n"
+            ),
+        ),
+        (
+            ["dogs"],
+            ("CREATE TABLE [dogs] (\n" "   [id] INTEGER,\n" "   [name] TEXT\n" ")\n"),
+        ),
+        (
+            ["chickens", "dogs"],
+            (
+                "CREATE TABLE [chickens] (\n"
+                "   [id] INTEGER,\n"
+                "   [name] TEXT,\n"
+                "   [breed] TEXT\n"
+                ")\n"
+                "CREATE TABLE [dogs] (\n"
+                "   [id] INTEGER,\n"
+                "   [name] TEXT\n"
+                ")\n"
+            ),
+        ),
+    ),
+)
+def test_schema(tmpdir, options, expected):
+    db_path = str(tmpdir / "test.db")
+    db = Database(db_path)
+    db["dogs"].create({"id": int, "name": str})
+    db["chickens"].create({"id": int, "name": str, "breed": str})
+    db["chickens"].create_index(["breed"])
+    result = CliRunner().invoke(
+        cli.cli,
+        ["schema", db_path] + options,
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    assert result.output == expected
+
+
 def test_long_csv_column_value(tmpdir):
     db_path = str(tmpdir / "test.db")
     csv_path = str(tmpdir / "test.csv")
@@ -1889,3 +2241,85 @@ def test_attach(tmpdir):
         {"id": 1, "text": "foo"},
         {"id": 1, "text": "bar"},
     ]
+
+
+def test_csv_insert_bom(tmpdir):
+    db_path = str(tmpdir / "test.db")
+    bom_csv_path = str(tmpdir / "bom.csv")
+    with open(bom_csv_path, "wb") as fp:
+        fp.write(b"\xef\xbb\xbfname,age\nCleo,5")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "broken", bom_csv_path, "--encoding", "utf-8", "--csv"],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    result2 = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "fixed", bom_csv_path, "--csv"],
+        catch_exceptions=False,
+    )
+    assert result2.exit_code == 0
+    db = Database(db_path)
+    tables = db.execute("select name, sql from sqlite_master").fetchall()
+    assert tables == [
+        ("broken", "CREATE TABLE [broken] (\n   [\ufeffname] TEXT,\n   [age] TEXT\n)"),
+        ("fixed", "CREATE TABLE [fixed] (\n   [name] TEXT,\n   [age] TEXT\n)"),
+    ]
+
+
+@pytest.mark.parametrize("option_or_env_var", (None, "-d", "--detect-types"))
+def test_insert_detect_types(tmpdir, option_or_env_var):
+    db_path = str(tmpdir / "test.db")
+    data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5"
+    extra = []
+    if option_or_env_var:
+        extra = [option_or_env_var]
+
+    def _test():
+        result = CliRunner().invoke(
+            cli.cli,
+            ["insert", db_path, "creatures", "-", "--csv"] + extra,
+            catch_exceptions=False,
+            input=data,
+        )
+        assert result.exit_code == 0
+        db = Database(db_path)
+        assert list(db["creatures"].rows) == [
+            {"name": "Cleo", "age": 6, "weight": 45.5},
+            {"name": "Dori", "age": 1, "weight": 3.5},
+        ]
+
+    if option_or_env_var is None:
+        # Use environemnt variable instead of option
+        with mock.patch.dict(os.environ, {"SQLITE_UTILS_DETECT_TYPES": "1"}):
+            _test()
+    else:
+        _test()
+
+
+@pytest.mark.parametrize(
+    "input,expected",
+    (
+        ({"foo": {"bar": 1}}, {"foo_bar": 1}),
+        ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}),
+        ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}),
+    ),
+)
+def test_flatten_helper(input, expected):
+    assert dict(cli._flatten(input)) == expected
+
+
+def test_integer_overflow_error(tmpdir):
+    db_path = str(tmpdir / "test.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "items", "-"],
+        input=json.dumps({"bignumber": 34223049823094832094802398430298048240}),
+    )
+    assert result.exit_code == 1
+    assert result.output == (
+        "Error: Python int too large to convert to SQLite INTEGER\n\n"
+        "sql = INSERT INTO [items] ([bignumber]) VALUES (?);\n"
+        "parameters = [34223049823094832094802398430298048240]\n"
+    )

tests/test_cli_convert.py

@@ -0,0 +1,517 @@
+from click.testing import CliRunner
+from sqlite_utils import cli
+import sqlite_utils
+import json
+import textwrap
+import pathlib
+import pytest
+
+
+@pytest.fixture
+def test_db_and_path(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["example"].insert_all(
+        [
+            {"id": 1, "dt": "5th October 2019 12:04"},
+            {"id": 2, "dt": "6th October 2019 00:05:06"},
+            {"id": 3, "dt": ""},
+            {"id": 4, "dt": None},
+        ],
+        pk="id",
+    )
+    return db, db_path
+
+
+@pytest.fixture
+def fresh_db_and_path(tmpdir):
+    db_path = str(pathlib.Path(tmpdir) / "data.db")
+    db = sqlite_utils.Database(db_path)
+    return db, db_path
+
+
+@pytest.mark.parametrize(
+    "code",
+    [
+        "return value.replace('October', 'Spooktober')",
+        # Return is optional:
+        "value.replace('October', 'Spooktober')",
+    ],
+)
+def test_convert_single_line(test_db_and_path, code):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(cli.cli, ["convert", db_path, "example", "dt", code])
+    assert 0 == result.exit_code, result.output
+    assert [
+        {"id": 1, "dt": "5th Spooktober 2019 12:04"},
+        {"id": 2, "dt": "6th Spooktober 2019 00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ] == list(db["example"].rows)
+
+
+def test_convert_multiple_lines(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "v = value.replace('October', 'Spooktober')\nreturn v.upper()",
+        ],
+    )
+    assert 0 == result.exit_code, result.output
+    assert [
+        {"id": 1, "dt": "5TH SPOOKTOBER 2019 12:04"},
+        {"id": 2, "dt": "6TH SPOOKTOBER 2019 00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ] == list(db["example"].rows)
+
+
+def test_convert_import(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "return re.sub('O..', 'OXX', value)",
+            "--import",
+            "re",
+        ],
+    )
+    assert 0 == result.exit_code, result.output
+    assert [
+        {"id": 1, "dt": "5th OXXober 2019 12:04"},
+        {"id": 2, "dt": "6th OXXober 2019 00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ] == list(db["example"].rows)
+
+
+def test_convert_dryrun(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "return re.sub('O..', 'OXX', value)",
+            "--import",
+            "re",
+            "--dry-run",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == (
+        "5th October 2019 12:04\n"
+        " --- becomes:\n"
+        "5th OXXober 2019 12:04\n"
+        "\n"
+        "6th October 2019 00:05:06\n"
+        " --- becomes:\n"
+        "6th OXXober 2019 00:05:06\n"
+        "\n"
+        "\n"
+        " --- becomes:\n"
+        "\n"
+        "\n"
+        "None\n"
+        " --- becomes:\n"
+        "None\n\n"
+        "Would affect 4 rows"
+    )
+    # But it should not have actually modified the table data
+    assert list(db["example"].rows) == [
+        {"id": 1, "dt": "5th October 2019 12:04"},
+        {"id": 2, "dt": "6th October 2019 00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ]
+    # Test with a where clause too
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "return re.sub('O..', 'OXX', value)",
+            "--import",
+            "re",
+            "--dry-run",
+            "--where",
+            "id = :id",
+            "-p",
+            "id",
+            "4",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output.strip().split("\n")[-1] == "Would affect 1 row"
+
+
+@pytest.mark.parametrize("drop", (True, False))
+def test_convert_output_column(test_db_and_path, drop):
+    db, db_path = test_db_and_path
+    args = [
+        "convert",
+        db_path,
+        "example",
+        "dt",
+        "value.replace('October', 'Spooktober')",
+        "--output",
+        "newcol",
+    ]
+    if drop:
+        args += ["--drop"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert 0 == result.exit_code, result.output
+    expected = [
+        {
+            "id": 1,
+            "dt": "5th October 2019 12:04",
+            "newcol": "5th Spooktober 2019 12:04",
+        },
+        {
+            "id": 2,
+            "dt": "6th October 2019 00:05:06",
+            "newcol": "6th Spooktober 2019 00:05:06",
+        },
+        {"id": 3, "dt": "", "newcol": ""},
+        {"id": 4, "dt": None, "newcol": None},
+    ]
+    if drop:
+        for row in expected:
+            del row["dt"]
+    assert list(db["example"].rows) == expected
+
+
+@pytest.mark.parametrize(
+    "output_type,expected",
+    (
+        ("text", [(1, "1"), (2, "2"), (3, "3"), (4, "4")]),
+        ("float", [(1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0)]),
+        ("integer", [(1, 1), (2, 2), (3, 3), (4, 4)]),
+        (None, [(1, "1"), (2, "2"), (3, "3"), (4, "4")]),
+    ),
+)
+def test_convert_output_column_output_type(test_db_and_path, output_type, expected):
+    db, db_path = test_db_and_path
+    args = [
+        "convert",
+        db_path,
+        "example",
+        "id",
+        "value",
+        "--output",
+        "new_id",
+    ]
+    if output_type:
+        args += ["--output-type", output_type]
+    result = CliRunner().invoke(
+        cli.cli,
+        args,
+    )
+    assert 0 == result.exit_code, result.output
+    assert expected == list(db.execute("select id, new_id from example"))
+
+
+@pytest.mark.parametrize(
+    "options,expected_error",
+    [
+        (
+            [
+                "dt",
+                "id",
+                "value.replace('October', 'Spooktober')",
+                "--output",
+                "newcol",
+            ],
+            "Cannot use --output with more than one column",
+        ),
+        (
+            [
+                "dt",
+                "value.replace('October', 'Spooktober')",
+                "--output",
+                "newcol",
+                "--output-type",
+                "invalid",
+            ],
+            "Error: Invalid value for '--output-type'",
+        ),
+        (
+            [
+                "value.replace('October', 'Spooktober')",
+            ],
+            "Missing argument 'COLUMNS...'",
+        ),
+    ],
+)
+def test_convert_output_error(test_db_and_path, options, expected_error):
+    db_path = test_db_and_path[1]
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+        ]
+        + options,
+    )
+    assert result.exit_code != 0
+    assert expected_error in result.output
+
+
+@pytest.mark.parametrize("drop", (True, False))
+def test_convert_multi(fresh_db_and_path, drop):
+    db, db_path = fresh_db_and_path
+    db["creatures"].insert_all(
+        [
+            {"id": 1, "name": "Simon"},
+            {"id": 2, "name": "Cleo"},
+        ],
+        pk="id",
+    )
+    args = [
+        "convert",
+        db_path,
+        "creatures",
+        "name",
+        "--multi",
+        '{"upper": value.upper(), "lower": value.lower()}',
+    ]
+    if drop:
+        args += ["--drop"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 0, result.output
+    expected = [
+        {"id": 1, "name": "Simon", "upper": "SIMON", "lower": "simon"},
+        {"id": 2, "name": "Cleo", "upper": "CLEO", "lower": "cleo"},
+    ]
+    if drop:
+        for row in expected:
+            del row["name"]
+    assert list(db["creatures"].rows) == expected
+
+
+def test_convert_multi_complex_column_types(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["rows"].insert_all(
+        [
+            {"id": 1},
+            {"id": 2},
+            {"id": 3},
+            {"id": 4},
+        ],
+        pk="id",
+    )
+    code = textwrap.dedent(
+        """
+    if value == 1:
+        return {"is_str": "", "is_float": 1.2, "is_int": None}
+    elif value == 2:
+        return {"is_float": 1, "is_int": 12}
+    elif value == 3:
+        return {"is_bytes": b"blah"}
+    """
+    )
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "rows",
+            "id",
+            "--multi",
+            code,
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["rows"].rows) == [
+        {"id": 1, "is_str": "", "is_float": 1.2, "is_int": None, "is_bytes": None},
+        {"id": 2, "is_str": None, "is_float": 1.0, "is_int": 12, "is_bytes": None},
+        {
+            "id": 3,
+            "is_str": None,
+            "is_float": None,
+            "is_int": None,
+            "is_bytes": b"blah",
+        },
+        {"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None},
+    ]
+    assert db["rows"].schema == (
+        "CREATE TABLE [rows] (\n"
+        "   [id] INTEGER PRIMARY KEY\n"
+        ", [is_str] TEXT, [is_float] FLOAT, [is_int] INTEGER, [is_bytes] BLOB)"
+    )
+
+
+@pytest.mark.parametrize("delimiter", [None, ";", "-"])
+def test_recipe_jsonsplit(tmpdir, delimiter):
+    db_path = str(pathlib.Path(tmpdir) / "data.db")
+    db = sqlite_utils.Database(db_path)
+    db["example"].insert_all(
+        [
+            {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])},
+            {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])},
+        ],
+        pk="id",
+    )
+    code = "r.jsonsplit(value)"
+    if delimiter:
+        code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter)
+    args = ["convert", db_path, "example", "tags", code]
+    result = CliRunner().invoke(cli.cli, args)
+    assert 0 == result.exit_code, result.output
+    assert list(db["example"].rows) == [
+        {"id": 1, "tags": '["foo", "bar"]'},
+        {"id": 2, "tags": '["bar", "baz"]'},
+    ]
+
+
+@pytest.mark.parametrize(
+    "type,expected_array",
+    (
+        (None, ["1", "2", "3"]),
+        ("float", [1.0, 2.0, 3.0]),
+        ("int", [1, 2, 3]),
+    ),
+)
+def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array):
+    db, db_path = fresh_db_and_path
+    db["example"].insert_all(
+        [
+            {"id": 1, "records": "1,2,3"},
+        ],
+        pk="id",
+    )
+    code = "r.jsonsplit(value)"
+    if type:
+        code = "recipes.jsonsplit(value, type={})".format(type)
+    args = ["convert", db_path, "example", "records", code]
+    result = CliRunner().invoke(cli.cli, args)
+    assert 0 == result.exit_code, result.output
+    assert json.loads(db["example"].get(1)["records"]) == expected_array
+
+
+@pytest.mark.parametrize("drop", (True, False))
+def test_recipe_jsonsplit_output(fresh_db_and_path, drop):
+    db, db_path = fresh_db_and_path
+    db["example"].insert_all(
+        [
+            {"id": 1, "records": "1,2,3"},
+        ],
+        pk="id",
+    )
+    code = "r.jsonsplit(value)"
+    args = ["convert", db_path, "example", "records", code, "--output", "tags"]
+    if drop:
+        args += ["--drop"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert 0 == result.exit_code, result.output
+    expected = {
+        "id": 1,
+        "records": "1,2,3",
+        "tags": '["1", "2", "3"]',
+    }
+    if drop:
+        del expected["records"]
+    assert db["example"].get(1) == expected
+
+
+def test_cannot_use_drop_without_multi_or_output(fresh_db_and_path):
+    args = ["convert", fresh_db_and_path[1], "example", "records", "value", "--drop"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 1, result.output
+    assert "Error: --drop can only be used with --output or --multi" in result.output
+
+
+def test_cannot_use_multi_with_more_than_one_column(fresh_db_and_path):
+    args = [
+        "convert",
+        fresh_db_and_path[1],
+        "example",
+        "records",
+        "othercol",
+        "value",
+        "--multi",
+    ]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 1, result.output
+    assert "Error: Cannot use --multi with more than one column" in result.output
+
+
+def test_multi_with_bad_function(test_db_and_path):
+    args = [
+        "convert",
+        test_db_and_path[1],
+        "example",
+        "dt",
+        "value.upper()",
+        "--multi",
+    ]
+    result = CliRunner().invoke(cli.cli, args)
+    assert result.exit_code == 1, result.output
+    assert "When using --multi code must return a Python dictionary" in result.output
+
+
+def test_convert_where(test_db_and_path):
+    db, db_path = test_db_and_path
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "example",
+            "dt",
+            "str(value).upper()",
+            "--where",
+            "id = :id",
+            "-p",
+            "id",
+            2,
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert list(db["example"].rows) == [
+        {"id": 1, "dt": "5th October 2019 12:04"},
+        {"id": 2, "dt": "6TH OCTOBER 2019 00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ]
+
+
+def test_convert_where_multi(fresh_db_and_path):
+    db, db_path = fresh_db_and_path
+    db["names"].insert_all(
+        [{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}], pk="id"
+    )
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "convert",
+            db_path,
+            "names",
+            "name",
+            '{"upper": value.upper()}',
+            "--where",
+            "id = :id",
+            "-p",
+            "id",
+            2,
+            "--multi",
+        ],
+    )
+    assert 0 == result.exit_code, result.output
+    assert list(db["names"].rows) == [
+        {"id": 1, "name": "Cleo", "upper": None},
+        {"id": 2, "name": "Bants", "upper": "BANTS"},
+    ]

tests/test_cli_memory.py

@@ -0,0 +1,222 @@
+import json
+
+import pytest
+from click.testing import CliRunner
+
+from sqlite_utils import Database, cli
+
+
+def test_memory_basic():
+    result = CliRunner().invoke(cli.cli, ["memory", "select 1 + 1"])
+    assert result.exit_code == 0
+    assert result.output.strip() == '[{"1 + 1": 2}]'
+
+
+@pytest.mark.parametrize("sql_from", ("test", "t", "t1"))
+@pytest.mark.parametrize("use_stdin", (True, False))
+def test_memory_csv(tmpdir, sql_from, use_stdin):
+    content = "id,name\n1,Cleo\n2,Bants"
+    input = None
+    if use_stdin:
+        input = content
+        csv_path = "-"
+        if sql_from == "test":
+            sql_from = "stdin"
+    else:
+        csv_path = str(tmpdir / "test.csv")
+        open(csv_path, "w").write(content)
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", csv_path, "select * from {}".format(sql_from), "--nl"],
+        input=input,
+    )
+    assert result.exit_code == 0
+    assert (
+        result.output.strip() == '{"id": 1, "name": "Cleo"}\n{"id": 2, "name": "Bants"}'
+    )
+
+
+@pytest.mark.parametrize("use_stdin", (True, False))
+def test_memory_tsv(tmpdir, use_stdin):
+    data = "id\tname\n1\tCleo\n2\tBants"
+    if use_stdin:
+        input = data
+        path = "stdin:tsv"
+        sql_from = "stdin"
+    else:
+        input = None
+        path = str(tmpdir / "chickens.tsv")
+        open(path, "w").write(data)
+        path = path + ":tsv"
+        sql_from = "chickens"
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", path, "select * from {}".format(sql_from)],
+        input=input,
+    )
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.output.strip()) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Bants"},
+    ]
+
+
+@pytest.mark.parametrize("use_stdin", (True, False))
+def test_memory_json(tmpdir, use_stdin):
+    data = '[{"name": "Bants"}, {"name": "Dori", "age": 1, "nested": {"nest": 1}}]'
+    if use_stdin:
+        input = data
+        path = "stdin:json"
+        sql_from = "stdin"
+    else:
+        input = None
+        path = str(tmpdir / "chickens.json")
+        open(path, "w").write(data)
+        path = path + ":json"
+        sql_from = "chickens"
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", path, "select * from {}".format(sql_from)],
+        input=input,
+    )
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.output.strip()) == [
+        {"name": "Bants", "age": None, "nested": None},
+        {"name": "Dori", "age": 1, "nested": '{"nest": 1}'},
+    ]
+
+
+@pytest.mark.parametrize("use_stdin", (True, False))
+def test_memory_json_nl(tmpdir, use_stdin):
+    data = '{"name": "Bants"}\n\n{"name": "Dori"}'
+    if use_stdin:
+        input = data
+        path = "stdin:nl"
+        sql_from = "stdin"
+    else:
+        input = None
+        path = str(tmpdir / "chickens.json")
+        open(path, "w").write(data)
+        path = path + ":nl"
+        sql_from = "chickens"
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", path, "select * from {}".format(sql_from)],
+        input=input,
+    )
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.output.strip()) == [
+        {"name": "Bants"},
+        {"name": "Dori"},
+    ]
+
+
+@pytest.mark.parametrize("use_stdin", (True, False))
+def test_memory_csv_encoding(tmpdir, use_stdin):
+    latin1_csv = (
+        b"date,name,latitude,longitude\n" b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n"
+    )
+    input = None
+    if use_stdin:
+        input = latin1_csv
+        csv_path = "-"
+        sql_from = "stdin"
+    else:
+        csv_path = str(tmpdir / "test.csv")
+        with open(csv_path, "wb") as fp:
+            fp.write(latin1_csv)
+        sql_from = "test"
+    # Without --encoding should error:
+    assert (
+        CliRunner()
+        .invoke(
+            cli.cli,
+            ["memory", csv_path, "select * from {}".format(sql_from), "--nl"],
+            input=input,
+        )
+        .exit_code
+        == 1
+    )
+    # With --encoding should work:
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-", "select * from stdin", "--encoding", "latin-1", "--nl"],
+        input=latin1_csv,
+    )
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.output.strip()) == {
+        "date": "2020-03-04",
+        "name": "São Paulo",
+        "latitude": -23.561,
+        "longitude": -46.645,
+    }
+
+
+@pytest.mark.parametrize("extra_args", ([], ["select 1"]))
+def test_memory_dump(extra_args):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-"] + extra_args + ["--dump"],
+        input="id,name\n1,Cleo\n2,Bants",
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == (
+        "BEGIN TRANSACTION;\n"
+        'CREATE TABLE "stdin" (\n'
+        "   [id] INTEGER,\n"
+        "   [name] TEXT\n"
+        ");\n"
+        "INSERT INTO \"stdin\" VALUES(1,'Cleo');\n"
+        "INSERT INTO \"stdin\" VALUES(2,'Bants');\n"
+        "CREATE VIEW t1 AS select * from [stdin];\n"
+        "CREATE VIEW t AS select * from [stdin];\n"
+        "COMMIT;"
+    )
+
+
+@pytest.mark.parametrize("extra_args", ([], ["select 1"]))
+def test_memory_schema(extra_args):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-"] + extra_args + ["--schema"],
+        input="id,name\n1,Cleo\n2,Bants",
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == (
+        'CREATE TABLE "stdin" (\n'
+        "   [id] INTEGER,\n"
+        "   [name] TEXT\n"
+        ");\n"
+        "CREATE VIEW t1 AS select * from [stdin];\n"
+        "CREATE VIEW t AS select * from [stdin];"
+    )
+
+
+@pytest.mark.parametrize("extra_args", ([], ["select 1"]))
+def test_memory_save(tmpdir, extra_args):
+    save_to = str(tmpdir / "save.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-"] + extra_args + ["--save", save_to],
+        input="id,name\n1,Cleo\n2,Bants",
+    )
+    assert result.exit_code == 0
+    db = Database(save_to)
+    assert list(db["stdin"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Bants"},
+    ]
+
+
+@pytest.mark.parametrize("option", ("-n", "--no-detect-types"))
+def test_memory_no_detect_types(option):
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-", "select * from stdin"] + [option],
+        input="id,name,weight\n1,Cleo,45.5\n2,Bants,3.5",
+    )
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.output.strip()) == [
+        {"id": "1", "name": "Cleo", "weight": "45.5"},
+        {"id": "2", "name": "Bants", "weight": "3.5"},
+    ]

tests/test_constructor.py

@@ -1,5 +1,4 @@
 from sqlite_utils import Database
-import pytest
 
 
 def test_recursive_triggers():

tests/test_conversions.py

@@ -1,6 +1,3 @@
-import pytest
-
-
 def test_insert_conversion(fresh_db):
     table = fresh_db["table"]
     table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"})

tests/test_convert.py

@@ -0,0 +1,117 @@
+from sqlite_utils.db import BadMultiValues
+import pytest
+
+
+@pytest.mark.parametrize(
+    "columns,fn,expected",
+    (
+        (
+            "title",
+            lambda value: value.upper(),
+            {"title": "MIXED CASE", "abstract": "Abstract"},
+        ),
+        (
+            ["title", "abstract"],
+            lambda value: value.upper(),
+            {"title": "MIXED CASE", "abstract": "ABSTRACT"},
+        ),
+    ),
+)
+def test_convert(fresh_db, columns, fn, expected):
+    table = fresh_db["table"]
+    table.insert({"title": "Mixed Case", "abstract": "Abstract"})
+    table.convert(columns, fn)
+    assert list(table.rows) == [expected]
+
+
+@pytest.mark.parametrize(
+    "where,where_args", (("id > 1", None), ("id > :id", {"id": 1}), ("id > ?", [1]))
+)
+def test_convert_where(fresh_db, where, where_args):
+    table = fresh_db["table"]
+    table.insert_all(
+        [
+            {"id": 1, "title": "One"},
+            {"id": 2, "title": "Two"},
+        ],
+        pk="id",
+    )
+    table.convert(
+        "title", lambda value: value.upper(), where=where, where_args=where_args
+    )
+    assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}]
+
+
+@pytest.mark.parametrize(
+    "drop,expected",
+    (
+        (False, {"title": "Mixed Case", "other": "MIXED CASE"}),
+        (True, {"other": "MIXED CASE"}),
+    ),
+)
+def test_convert_output(fresh_db, drop, expected):
+    table = fresh_db["table"]
+    table.insert({"title": "Mixed Case"})
+    table.convert("title", lambda v: v.upper(), output="other", drop=drop)
+    assert list(table.rows) == [expected]
+
+
+def test_convert_output_multiple_column_error(fresh_db):
+    table = fresh_db["table"]
+    with pytest.raises(AssertionError) as excinfo:
+        table.convert(["title", "other"], lambda v: v, output="out")
+        assert "output= can only be used with a single column" in str(excinfo.value)
+
+
+@pytest.mark.parametrize(
+    "type,expected",
+    (
+        (int, {"other": 123}),
+        (float, {"other": 123.0}),
+    ),
+)
+def test_convert_output_type(fresh_db, type, expected):
+    table = fresh_db["table"]
+    table.insert({"number": "123"})
+    table.convert("number", lambda v: v, output="other", output_type=type, drop=True)
+    assert list(table.rows) == [expected]
+
+
+def test_convert_multi(fresh_db):
+    table = fresh_db["table"]
+    table.insert({"title": "Mixed Case"})
+    table.convert(
+        "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True
+    )
+    assert list(table.rows) == [
+        {"title": "Mixed Case", "upper": "MIXED CASE", "lower": "mixed case"}
+    ]
+
+
+def test_convert_multi_where(fresh_db):
+    table = fresh_db["table"]
+    table.insert_all(
+        [
+            {"id": 1, "title": "One"},
+            {"id": 2, "title": "Two"},
+        ],
+        pk="id",
+    )
+    table.convert(
+        "title",
+        lambda v: {"upper": v.upper(), "lower": v.lower()},
+        multi=True,
+        where="id > ?",
+        where_args=[1],
+    )
+    assert list(table.rows) == [
+        {"id": 1, "lower": None, "title": "One", "upper": None},
+        {"id": 2, "lower": "two", "title": "Two", "upper": "TWO"},
+    ]
+
+
+def test_convert_multi_exception(fresh_db):
+    table = fresh_db["table"]
+    table.insert({"title": "Mixed Case"})
+    with pytest.raises(BadMultiValues):
+        table.convert("title", lambda v: v.upper(), multi=True)

tests/test_create.py

@@ -1,7 +1,7 @@
 from sqlite_utils.db import (
     Index,
     Database,
-    ForeignKey,
+    DescIndex,
     AlterError,
     NoObviousTable,
     ForeignKey,
@@ -147,8 +147,8 @@ def test_create_table_with_not_null(fresh_db):
 )
 def test_create_table_from_example(fresh_db, example, expected_columns):
     people_table = fresh_db["people"]
-    assert None == people_table.last_rowid
+    assert people_table.last_rowid is None
-    assert None == people_table.last_pk
+    assert people_table.last_pk is None
     people_table.insert(example)
     assert 1 == people_table.last_rowid
     assert 1 == people_table.last_pk
@@ -514,7 +514,7 @@ def test_insert_row_alter_table(
 
 def test_insert_row_alter_table_invalid_column_characters(fresh_db):
     table = fresh_db["table"]
-    rowid = table.insert({"foo": "bar"}).last_pk
+    table.insert({"foo": "bar"}).last_pk
     with pytest.raises(AssertionError):
         table.insert({"foo": "baz", "new_col[abc]": 1.2}, alter=True)
 
@@ -739,6 +739,19 @@ def test_create_index_if_not_exists(fresh_db):
     dogs.create_index(["name"], if_not_exists=True)
 
 
+def test_create_index_desc(fresh_db):
+    dogs = fresh_db["dogs"]
+    dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True})
+    assert [] == dogs.indexes
+    dogs.create_index([DescIndex("age"), "name"])
+    sql = fresh_db.execute(
+        "select sql from sqlite_master where name='idx_dogs_age_name'"
+    ).fetchone()[0]
+    assert sql == (
+        "CREATE INDEX [idx_dogs_age_name]\n" "    ON [dogs] ([age] desc, [name])"
+    )
+
+
 @pytest.mark.parametrize(
     "data_structure",
     (
@@ -748,7 +761,7 @@ def test_create_index_if_not_exists(fresh_db):
         {"dictionary": {"nested": "complex"}},
         collections.OrderedDict(
             [
-                ("key1", {"nested": "complex"}),
+                ("key1", {"nested": ["cømplex"]}),
                 ("key2", "foo"),
             ]
         ),
@@ -762,6 +775,14 @@ def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure):
     assert data_structure == json.loads(row[1])
 
 
+def test_insert_list_nested_unicode(fresh_db):
+    fresh_db["test"].insert(
+        {"id": 1, "data": {"key1": {"nested": ["cømplex"]}}}, pk="id"
+    )
+    row = fresh_db.execute("select id, data from test").fetchone()
+    assert row[1] == '{"key1": {"nested": ["cømplex"]}}'
+
+
 def test_insert_uuid(fresh_db):
     uuid4 = uuid.uuid4()
     fresh_db["test"].insert({"uuid": uuid4})
@@ -805,8 +826,8 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db
         + [{"i": 101, "extra": "Should trigger ALTER"}],
         alter=True,
     )
-    rows = fresh_db.execute_returning_dicts("select * from test where i = 101")
+    rows = list(fresh_db.query("select * from test where i = 101"))
-    assert [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] == rows
+    assert rows == [{"i": 101, "word": None, "extra": "Should trigger ALTER"}]
 
 
 def test_insert_ignore(fresh_db):
@@ -817,8 +838,8 @@ def test_insert_ignore(fresh_db):
     # Using ignore=True should cause our insert to be silently ignored
     fresh_db["test"].insert({"id": 1, "bar": 3}, pk="id", ignore=True)
     # Only one row, and it should be bar=2, not bar=3
-    rows = fresh_db.execute_returning_dicts("select * from test")
+    rows = list(fresh_db.query("select * from test"))
-    assert [{"id": 1, "bar": 2}] == rows
+    assert rows == [{"id": 1, "bar": 2}]
 
 
 def test_insert_hash_id(fresh_db):
@@ -848,8 +869,6 @@ def test_works_with_pathlib_path(tmpdir):
 
 @pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed")
 def test_create_table_numpy(fresh_db):
-    import numpy as np
-
     df = pd.DataFrame({"col 1": range(3), "col 2": range(3)})
     fresh_db["pandas"].insert_all(df.to_dict(orient="records"))
     assert [
@@ -969,6 +988,13 @@ def test_insert_all_empty_list(fresh_db):
     assert 1 == fresh_db["t"].count
 
 
+def test_insert_all_single_column(fresh_db):
+    table = fresh_db["table"]
+    table.insert_all([{"name": "Cleo"}], pk="name")
+    assert [{"name": "Cleo"}] == list(table.rows)
+    assert table.pks == ["name"]
+
+
 def test_create_with_a_null_column(fresh_db):
     record = {"name": "Name", "description": None}
     fresh_db["t"].insert(record)

tests/test_docs.py

@@ -1,10 +1,12 @@
-from sqlite_utils import cli
+from click.testing import CliRunner
+from sqlite_utils import cli, recipes
 from pathlib import Path
 import pytest
 import re
 
 docs_path = Path(__file__).parent.parent / "docs"
 commands_re = re.compile(r"(?:\$ |    )sqlite-utils (\S+) ")
+recipes_re = re.compile(r"r\.(\w+)\(")
 
 
 @pytest.fixture(scope="session")
@@ -17,11 +19,36 @@ def documented_commands():
     }
 
 
+@pytest.fixture(scope="session")
+def documented_recipes():
+    rst = (docs_path / "cli.rst").read_text()
+    return set(recipes_re.findall(rst))
+
+
 @pytest.mark.parametrize("command", cli.cli.commands.keys())
 def test_commands_are_documented(documented_commands, command):
     assert command in documented_commands
 
 
 @pytest.mark.parametrize("command", cli.cli.commands.values())
-def test_commands_have_docstrings(command):
+def test_commands_have_help(command):
-    assert command.__doc__, "{} is missing a docstring".format(command)
+    assert command.help, "{} is missing its help".format(command)
+
+
+def test_convert_help():
+    result = CliRunner().invoke(cli.cli, ["convert", "--help"])
+    assert result.exit_code == 0
+    for expected in (
+        "r.jsonsplit(value, ",
+        "r.parsedate(value, ",
+        "r.parsedatetime(value, ",
+    ):
+        assert expected in result.output
+
+
+@pytest.mark.parametrize(
+    "recipe",
+    [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")],
+)
+def test_recipes_are_documented(documented_recipes, recipe):
+    assert recipe in documented_recipes

tests/test_enable_counts.py

@@ -14,8 +14,31 @@ def test_enable_counts_specific_table(fresh_db):
     # Now enable counts
     foo.enable_counts()
     assert foo.triggers_dict == {
-        "foo_counts_insert": "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\nBEGIN\n    INSERT OR REPLACE INTO [_counts]\n    VALUES (\n        'foo',\n        COALESCE(\n            (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n        0\n        ) + 1\n    );\nEND",
+        "foo_counts_insert": (
-        "foo_counts_delete": "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\nBEGIN\n    INSERT OR REPLACE INTO [_counts]\n    VALUES (\n        'foo',\n        COALESCE(\n            (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n        0\n        ) - 1\n    );\nEND",
+            "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\n"
+            "BEGIN\n"
+            "    INSERT OR REPLACE INTO [_counts]\n"
+            "    VALUES (\n        'foo',\n"
+            "        COALESCE(\n"
+            "            (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n"
+            "        0\n"
+            "        ) + 1\n"
+            "    );\n"
+            "END"
+        ),
+        "foo_counts_delete": (
+            "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\n"
+            "BEGIN\n"
+            "    INSERT OR REPLACE INTO [_counts]\n"
+            "    VALUES (\n"
+            "        'foo',\n"
+            "        COALESCE(\n"
+            "            (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n"
+            "        0\n"
+            "        ) - 1\n"
+            "    );\n"
+            "END"
+        ),
     }
     assert fresh_db.table_names() == ["foo", "_counts"]
     assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}]
@@ -109,7 +132,7 @@ def test_uses_counts_after_enable_counts(counts_db_path):
         assert db["foo"].count == 1
         assert logged == [
             ("select name from sqlite_master where type = 'view'", None),
-            ("select count(*) from [foo]", None),
+            ("select count(*) from [foo]", []),
         ]
         logged.clear()
         assert not db.use_counts_table

tests/test_extract.py

@@ -1,4 +1,4 @@
-from sqlite_utils.db import Index, InvalidColumns
+from sqlite_utils.db import InvalidColumns
 import itertools
 import pytest
 
@@ -126,12 +126,25 @@ def test_extract_rowid_table(fresh_db):
     fresh_db["tree"].extract(["common_name", "latin_name"])
     assert fresh_db["tree"].schema == (
         'CREATE TABLE "tree" (\n'
-        "   [rowid] INTEGER PRIMARY KEY,\n"
         "   [name] TEXT,\n"
         "   [common_name_latin_name_id] INTEGER,\n"
         "   FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n"
         ")"
     )
+    assert (
+        fresh_db.execute(
+            """
+        select
+            tree.name,
+            common_name_latin_name.common_name,
+            common_name_latin_name.latin_name
+        from tree
+            join common_name_latin_name
+            on tree.common_name_latin_name_id = common_name_latin_name.id
+    """
+        ).fetchall()
+        == [("Tree 1", "Palm", "Arecaceae")]
+    )
 
 
 def test_reuse_lookup_table(fresh_db):

tests/test_extracts.py

@@ -1,4 +1,4 @@
-from sqlite_utils.db import Index, ForeignKey
+from sqlite_utils.db import Index
 import pytest
 
 

tests/test_hypothesis.py

@@ -2,6 +2,7 @@ from hypothesis import given
 import hypothesis.strategies as st
 import sqlite_utils
 
+
 # SQLite integers are -(2^63) to 2^63 - 1
 @given(st.integers(-9223372036854775808, 9223372036854775807))
 def test_roundtrip_integers(integer):

tests/test_insert_files.py

@@ -2,9 +2,11 @@ from sqlite_utils import cli, Database
 from click.testing import CliRunner
 import os
 import pathlib
+import pytest
 
 
-def test_insert_files():
+@pytest.mark.parametrize("silent", (False, True))
+def test_insert_files(silent):
     runner = CliRunner()
     with runner.isolated_filesystem():
         tmpdir = pathlib.Path(".")
@@ -34,7 +36,10 @@ def test_insert_files():
             cols += ["-c", "{}:{}".format(coltype, coltype)]
         result = runner.invoke(
             cli.cli,
-            ["insert-files", db_path, "files", str(tmpdir)] + cols + ["--pk", "path"],
+            ["insert-files", db_path, "files", str(tmpdir)]
+            + cols
+            + ["--pk", "path"]
+            + (["--silent"] if silent else []),
             catch_exceptions=False,
         )
         assert result.exit_code == 0, result.stdout

tests/test_introspect.py

@@ -1,4 +1,4 @@
-from sqlite_utils.db import Index, View, Database
+from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn
 import pytest
 
 
@@ -33,7 +33,7 @@ def test_detect_fts(existing_db):
     assert "woo_fts" == existing_db["woo_fts"].detect_fts()
     assert "woo2_fts" == existing_db["woo2"].detect_fts()
     assert "woo2_fts" == existing_db["woo2_fts"].detect_fts()
-    assert None == existing_db["foo"].detect_fts()
+    assert existing_db["foo"].detect_fts() is None
 
 
 def test_tables(existing_db):
@@ -52,7 +52,14 @@ def test_views(fresh_db):
 
 
 def test_count(existing_db):
-    assert 3 == existing_db["foo"].count
+    assert existing_db["foo"].count == 3
+    assert existing_db["foo"].count_where() == 3
+    assert existing_db["foo"].execute_count() == 3
+
+
+def test_count_where(existing_db):
+    assert existing_db["foo"].count_where("text != ?", ["two"]) == 2
+    assert existing_db["foo"].count_where("text != :t", {"t": "two"}) == 2
 
 
 def test_columns(existing_db):
@@ -62,8 +69,12 @@ def test_columns(existing_db):
     ]
 
 
-def test_schema(existing_db):
+def test_table_schema(existing_db):
-    assert "CREATE TABLE foo (text TEXT)" == existing_db["foo"].schema
+    assert existing_db["foo"].schema == "CREATE TABLE foo (text TEXT)"
+
+
+def test_database_schema(existing_db):
+    assert existing_db.schema == "CREATE TABLE foo (text TEXT);"
 
 
 def test_table_repr(fresh_db):
@@ -93,6 +104,33 @@ def test_indexes(fresh_db):
     ] == fresh_db["Gosh"].indexes
 
 
+def test_xindexes(fresh_db):
+    fresh_db.executescript(
+        """
+        create table Gosh (c1 text, c2 text, c3 text);
+        create index Gosh_c1 on Gosh(c1);
+        create index Gosh_c2c3 on Gosh(c2, c3 desc);
+    """
+    )
+    assert fresh_db["Gosh"].xindexes == [
+        XIndex(
+            name="Gosh_c2c3",
+            columns=[
+                XIndexColumn(seqno=0, cid=1, name="c2", desc=0, coll="BINARY", key=1),
+                XIndexColumn(seqno=1, cid=2, name="c3", desc=1, coll="BINARY", key=1),
+                XIndexColumn(seqno=2, cid=-1, name=None, desc=0, coll="BINARY", key=0),
+            ],
+        ),
+        XIndex(
+            name="Gosh_c1",
+            columns=[
+                XIndexColumn(seqno=0, cid=0, name="c1", desc=0, coll="BINARY", key=1),
+                XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll="BINARY", key=0),
+            ],
+        ),
+    ]
+
+
 @pytest.mark.parametrize(
     "column,expected_table_guess",
     (
@@ -144,9 +182,21 @@ def test_triggers_and_triggers_dict(fresh_db):
         (t.name, t.table) for t in fresh_db["authors"].triggers
     }
     expected_triggers = {
-        "authors_ai": "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n  INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND",
+        "authors_ai": (
-        "authors_ad": "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n  INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\nEND",
+            "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n"
-        "authors_au": "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n  INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n  INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND",
+            "  INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\n"
+            "END"
+        ),
+        "authors_ad": (
+            "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n"
+            "  INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n"
+            "END"
+        ),
+        "authors_au": (
+            "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n"
+            "  INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n"
+            "  INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND"
+        ),
     }
     assert authors.triggers_dict == expected_triggers
     assert fresh_db["other"].triggers == []
@@ -206,3 +256,11 @@ def test_virtual_table_using(sql, expected_name, expected_using):
     db = Database(memory=True)
     db.execute(sql)
     assert db[expected_name].virtual_table_using == expected_using
+
+
+def test_use_rowid():
+    db = Database(memory=True)
+    db["rowid_table"].insert({"name": "Cleo"})
+    db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id")
+    assert db["rowid_table"].use_rowid
+    assert not db["regular_table"].use_rowid

tests/test_query.py

@@ -0,0 +1,17 @@
+import types
+
+
+def test_query(fresh_db):
+    fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}])
+    results = fresh_db.query("select * from dogs order by name desc")
+    assert isinstance(results, types.GeneratorType)
+    assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}]
+
+
+def test_execute_returning_dicts(fresh_db):
+    # Like db.query() but returns a list, included for backwards compatibility
+    # see https://github.com/simonw/sqlite-utils/issues/290
+    fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id")
+    assert fresh_db.execute_returning_dicts("select * from test") == [
+        {"id": 1, "bar": 2}
+    ]

tests/test_recipes.py

@@ -0,0 +1,108 @@
+from sqlite_utils import recipes
+import json
+import pytest
+
+
+@pytest.fixture
+def dates_db(fresh_db):
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "dt": "5th October 2019 12:04"},
+            {"id": 2, "dt": "6th October 2019 00:05:06"},
+            {"id": 3, "dt": ""},
+            {"id": 4, "dt": None},
+        ],
+        pk="id",
+    )
+    return fresh_db
+
+
+def test_parsedate(dates_db):
+    dates_db["example"].convert("dt", recipes.parsedate)
+    assert list(dates_db["example"].rows) == [
+        {"id": 1, "dt": "2019-10-05"},
+        {"id": 2, "dt": "2019-10-06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ]
+
+
+def test_parsedatetime(dates_db):
+    dates_db["example"].convert("dt", recipes.parsedatetime)
+    assert list(dates_db["example"].rows) == [
+        {"id": 1, "dt": "2019-10-05T12:04:00"},
+        {"id": 2, "dt": "2019-10-06T00:05:06"},
+        {"id": 3, "dt": ""},
+        {"id": 4, "dt": None},
+    ]
+
+
+@pytest.mark.parametrize(
+    "recipe,kwargs,expected",
+    (
+        ("parsedate", {}, "2005-03-04"),
+        ("parsedate", {"dayfirst": True}, "2005-04-03"),
+        ("parsedatetime", {}, "2005-03-04T00:00:00"),
+        ("parsedatetime", {"dayfirst": True}, "2005-04-03T00:00:00"),
+    ),
+)
+def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected):
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "dt": "03/04/05"},
+        ],
+        pk="id",
+    )
+    fresh_db["example"].convert(
+        "dt", lambda value: getattr(recipes, recipe)(value, **kwargs)
+    )
+    assert list(fresh_db["example"].rows) == [
+        {"id": 1, "dt": expected},
+    ]
+
+
+@pytest.mark.parametrize("delimiter", [None, ";", "-"])
+def test_jsonsplit(fresh_db, delimiter):
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])},
+            {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])},
+        ],
+        pk="id",
+    )
+    fn = recipes.jsonsplit
+    if delimiter is not None:
+
+        def fn(value):
+            return recipes.jsonsplit(value, delimiter=delimiter)
+
+    fresh_db["example"].convert("tags", fn)
+    assert list(fresh_db["example"].rows) == [
+        {"id": 1, "tags": '["foo", "bar"]'},
+        {"id": 2, "tags": '["bar", "baz"]'},
+    ]
+
+
+@pytest.mark.parametrize(
+    "type,expected",
+    (
+        (None, ["1", "2", "3"]),
+        (float, [1.0, 2.0, 3.0]),
+        (int, [1, 2, 3]),
+    ),
+)
+def test_jsonsplit_type(fresh_db, type, expected):
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "records": "1,2,3"},
+        ],
+        pk="id",
+    )
+    fn = recipes.jsonsplit
+    if type is not None:
+
+        def fn(value):
+            return recipes.jsonsplit(value, type=type)
+
+    fresh_db["example"].convert("records", fn)
+    assert json.loads(fresh_db["example"].get(1)["records"]) == expected

tests/test_recreate.py

@@ -15,7 +15,7 @@ def test_recreate_ignored_for_in_memory():
 def test_recreate_not_allowed_for_connection():
     conn = sqlite3.connect(":memory:")
     with pytest.raises(AssertionError):
-        db = Database(conn, recreate=True)
+        Database(conn, recreate=True)
 
 
 @pytest.mark.parametrize(

tests/test_register_function.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 import pytest
 import sys
 from unittest.mock import MagicMock
@@ -55,14 +56,14 @@ def test_register_function_replace(fresh_db):
 
     # This will fail to replace the function:
     @fresh_db.register_function()
-    def one():
+    def one():  # noqa
         return "two"
 
     assert "one" == fresh_db.execute("select one()").fetchone()[0]
 
     # This will replace it
     @fresh_db.register_function(replace=True)
-    def one():
+    def one():  # noqa
         return "two"
 
     assert "two" == fresh_db.execute("select one()").fetchone()[0]

tests/test_rows.py

@@ -1,4 +1,3 @@
-from sqlite_utils.db import Index, View
 import pytest
 
 
@@ -13,6 +12,7 @@ def test_rows(existing_db):
     [
         ("name = ?", ["Pancakes"], {2}),
         ("age > ?", [3], {1}),
+        ("age > :age", {"age": 3}, {1}),
         ("name is not null", [], {1, 2}),
         ("is_good = ?", [True], {1, 2}),
     ],

tests/test_tracer.py

@@ -1,4 +1,3 @@
-import pytest
 from sqlite_utils import Database
 
 
@@ -32,7 +31,9 @@ def test_tracer():
 
 def test_with_tracer():
     collected = []
-    tracer = lambda sql, params: collected.append((sql, params))
+
+    def tracer(sql, params):
+        return collected.append((sql, params))
 
     db = Database(memory=True)
 
@@ -48,13 +49,39 @@ def test_with_tracer():
     assert collected == [
         ("select name from sqlite_master where type = 'view'", None),
         (
-            "SELECT name FROM sqlite_master\n    WHERE rootpage = 0\n    AND (\n        sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n        OR (\n            tbl_name = \"dogs\"\n            AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n        )\n    )",
+            (
+                "SELECT name FROM sqlite_master\n"
+                "    WHERE rootpage = 0\n"
+                "    AND (\n"
+                "        sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n"
+                "        OR (\n"
+                '            tbl_name = "dogs"\n'
+                "            AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n"
+                "        )\n"
+                "    )"
+            ),
             None,
         ),
         ("select name from sqlite_master where type = 'view'", None),
         ("select sql from sqlite_master where name = ?", ("dogs_fts",)),
         (
-            "with original as (\n    select\n        rowid,\n        *\n    from [dogs]\n)\nselect\n    [original].*\nfrom\n    [original]\n    join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n    [dogs_fts] match :query\norder by\n    [dogs_fts].rank",
+            (
+                "with original as (\n"
+                "    select\n"
+                "        rowid,\n"
+                "        *\n"
+                "    from [dogs]\n"
+                ")\n"
+                "select\n"
+                "    [original].*\n"
+                "from\n"
+                "    [original]\n"
+                "    join [dogs_fts] on [original].rowid = [dogs_fts].rowid\n"
+                "where\n"
+                "    [dogs_fts] match :query\n"
+                "order by\n"
+                "    [dogs_fts].rank"
+            ),
             {"query": "Cleopaws"},
         ),
     ]

tests/test_transform.py

@@ -89,9 +89,14 @@ import pytest
     ],
 )
 @pytest.mark.parametrize("use_pragma_foreign_keys", [False, True])
-def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys):
+def test_transform_sql_table_with_primary_key(
+    fresh_db, params, expected_sql, use_pragma_foreign_keys
+):
     captured = []
-    tracer = lambda sql, params: captured.append((sql, params))
+
+    def tracer(sql, params):
+        return captured.append((sql, params))
+
     dogs = fresh_db["dogs"]
     if use_pragma_foreign_keys:
         fresh_db.conn.execute("PRAGMA foreign_keys=ON")
@@ -111,7 +116,80 @@ def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys):
         assert ("PRAGMA foreign_keys=1;", None) not in captured
 
 
-def test_transform_sql_rowid_to_id(fresh_db):
+@pytest.mark.parametrize(
+    "params,expected_sql",
+    [
+        # Identity transform - nothing changes
+        (
+            {},
+            [
+                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] TEXT\n);",
+                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                "DROP TABLE [dogs];",
+                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+            ],
+        ),
+        # Change column type
+        (
+            {"types": {"age": int}},
+            [
+                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [age] INTEGER\n);",
+                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                "DROP TABLE [dogs];",
+                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+            ],
+        ),
+        # Rename a column
+        (
+            {"rename": {"age": "dog_age"}},
+            [
+                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER,\n   [name] TEXT,\n   [dog_age] TEXT\n);",
+                "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                "DROP TABLE [dogs];",
+                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+            ],
+        ),
+        # Make ID a primary key
+        (
+            {"pk": "id"},
+            [
+                "CREATE TABLE [dogs_new_suffix] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] TEXT\n);",
+                "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n   SELECT [id], [name], [age] FROM [dogs];",
+                "DROP TABLE [dogs];",
+                "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
+            ],
+        ),
+    ],
+)
+@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True])
+def test_transform_sql_table_with_no_primary_key(
+    fresh_db, params, expected_sql, use_pragma_foreign_keys
+):
+    captured = []
+
+    def tracer(sql, params):
+        return captured.append((sql, params))
+
+    dogs = fresh_db["dogs"]
+    if use_pragma_foreign_keys:
+        fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    dogs.insert({"id": 1, "name": "Cleo", "age": "5"})
+    sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}})
+    assert sql == expected_sql
+    # Check that .transform() runs without exceptions:
+    with fresh_db.tracer(tracer):
+        dogs.transform(**params)
+    # If use_pragma_foreign_keys, check that we did the right thing
+    if use_pragma_foreign_keys:
+        assert ("PRAGMA foreign_keys=0;", None) in captured
+        assert captured[-2] == ("PRAGMA foreign_key_check;", None)
+        assert captured[-1] == ("PRAGMA foreign_keys=1;", None)
+    else:
+        assert ("PRAGMA foreign_keys=0;", None) not in captured
+        assert ("PRAGMA foreign_keys=1;", None) not in captured
+
+
+def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db):
     dogs = fresh_db["dogs"]
     dogs.insert({"id": 1, "name": "Cleo", "age": "5"})
     assert (

tests/test_upsert.py

@@ -21,6 +21,13 @@ def test_upsert_all(fresh_db):
     assert table.last_pk is None
 
 
+def test_upsert_all_single_column(fresh_db):
+    table = fresh_db["table"]
+    table.upsert_all([{"name": "Cleo"}], pk="name")
+    assert [{"name": "Cleo"}] == list(table.rows)
+    assert table.pks == ["name"]
+
+
 def test_upsert_error_if_no_pk(fresh_db):
     table = fresh_db["table"]
     with pytest.raises(PrimaryKeyRequired):
@@ -47,7 +54,7 @@ def test_upsert_compound_primary_key(fresh_db):
         ],
         pk=("species", "id"),
     )
-    assert None == table.last_pk
+    assert table.last_pk is None
     table.upsert({"species": "dog", "id": 1, "age": 5}, pk=("species", "id"))
     assert ("dog", 1) == table.last_pk
     assert [

tests/test_wal.py

@@ -1,6 +1,5 @@
 import pytest
 from sqlite_utils import Database
-import sqlite3
 
 
 @pytest.fixture